From a9e539a3b4d8706a2c26bc23c69a677fb77222c0 Mon Sep 17 00:00:00 2001 From: Yuriy Tyshetskiy Date: Wed, 22 Jan 2020 23:30:55 +1100 Subject: [PATCH 1/5] bugfix: saving DeepSAD model with pretrain=False --- src/DeepSAD.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DeepSAD.py b/src/DeepSAD.py index 1f002040e..4de441d1f 100644 --- a/src/DeepSAD.py +++ b/src/DeepSAD.py @@ -130,7 +130,7 @@ def save_model(self, export_model, save_ae=True): """Save Deep SAD model to export_model.""" net_dict = self.net.state_dict() - ae_net_dict = self.ae_net.state_dict() if save_ae else None + ae_net_dict = self.ae_net.state_dict() if (save_ae and self.ae_net is not None) else None torch.save({'c': self.c, 'net_dict': net_dict, From e7a5194aeb00c0bf272db7d086c1cf391ad033ab Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Mueller Date: Tue, 7 Apr 2020 11:34:01 +1000 Subject: [PATCH 2/5] Allow newer versions of torch and torchvision --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7df715fc9..28cf6dd78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,5 +14,5 @@ scikit-learn==0.21.2 scipy==1.3.0 seaborn==0.9.0 six==1.12.0 -torch==1.1.0 -torchvision==0.3.0 +torch>=1.1.0 +torchvision>=0.3.0 From 02d2a571bb8c87066233dd7439d253f70c7e87e0 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Mueller Date: Wed, 22 Apr 2020 17:35:49 +1000 Subject: [PATCH 3/5] Return train and validation loss --- src/DeepSAD.py | 5 +++-- src/optim/DeepSAD_trainer.py | 40 ++++++++++++++++++++++++++++++------ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/DeepSAD.py b/src/DeepSAD.py index 4de441d1f..d1823f350 100644 --- a/src/DeepSAD.py +++ b/src/DeepSAD.py @@ -60,7 +60,7 @@ def set_network(self, net_name): def train(self, dataset: BaseADDataset, optimizer_name: str = 'adam', lr: float = 0.001, n_epochs: int = 50, lr_milestones: tuple = (), batch_size: int = 128, weight_decay: float = 1e-6, device: str = 'cuda', - n_jobs_dataloader: int = 0): + n_jobs_dataloader: int = 0, validate : bool = False): """Trains the Deep SAD model on the training data.""" self.optimizer_name = optimizer_name @@ -68,8 +68,9 @@ def train(self, dataset: BaseADDataset, optimizer_name: str = 'adam', lr: float lr_milestones=lr_milestones, batch_size=batch_size, weight_decay=weight_decay, device=device, n_jobs_dataloader=n_jobs_dataloader) # Get the model - self.net = self.trainer.train(dataset, self.net) + self.net = self.trainer.train(dataset, self.net, validate=validate) self.results['train_time'] = self.trainer.train_time + self.train_loss = self.trainer.train_loss self.c = self.trainer.c.cpu().data.numpy().tolist() # get as list def test(self, dataset: BaseADDataset, device: str = 'cuda', n_jobs_dataloader: int = 0): diff --git a/src/optim/DeepSAD_trainer.py b/src/optim/DeepSAD_trainer.py index 44b1118de..720a03f79 100644 --- a/src/optim/DeepSAD_trainer.py +++ b/src/optim/DeepSAD_trainer.py @@ -28,15 +28,16 @@ def __init__(self, c, eta: float, optimizer_name: str = 'adam', lr: float = 0.00 # Results self.train_time = None + self.train_loss = None self.test_auc = None self.test_time = None self.test_scores = None - def train(self, dataset: BaseADDataset, net: BaseNet): + def train(self, dataset: BaseADDataset, net: BaseNet, validate: bool = False): logger = logging.getLogger() # Get train data loader - train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) + train_loader, test_loader = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) # Set device for network net = net.to(self.device) @@ -57,13 +58,14 @@ def train(self, dataset: BaseADDataset, net: BaseNet): logger.info('Starting training...') start_time = time.time() net.train() + self.train_loss = [] for epoch in range(self.n_epochs): scheduler.step() if epoch in self.lr_milestones: logger.info(' LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0])) - epoch_loss = 0.0 + train_epoch_loss = 0.0 n_batches = 0 epoch_start_time = time.time() for data in train_loader: @@ -81,13 +83,39 @@ def train(self, dataset: BaseADDataset, net: BaseNet): loss.backward() optimizer.step() - epoch_loss += loss.item() + train_epoch_loss += loss.item() n_batches += 1 + train_loss = train_epoch_loss/n_batches + epoch_loss_history = (epoch + 1, train_loss) + + if validate: + n_batches = 0 + test_epoch_loss = 0.0 + with torch.set_grad_enabled(False): + for data in test_loader: + inputs, _, semi_targets, _ = data + inputs, semi_targets = inputs.to(self.device), semi_targets.to(self.device) + + outputs = net(inputs) + dist = torch.sum((outputs - self.c) ** 2, dim=1) + losses = torch.where(semi_targets == 0, dist, self.eta * ((dist + self.eps) ** semi_targets.float())) + loss = torch.mean(losses) + + test_epoch_loss += loss.item() + n_batches += 1 + test_loss = test_epoch_loss/n_batches + epoch_loss_history = (epoch + 1, train_loss, test_loss) + + self.train_loss.append(epoch_loss_history) # log epoch statistics epoch_train_time = time.time() - epoch_start_time - logger.info(f'| Epoch: {epoch + 1:03}/{self.n_epochs:03} | Train Time: {epoch_train_time:.3f}s ' - f'| Train Loss: {epoch_loss / n_batches:.6f} |') + + stats = f'| Epoch: {epoch + 1:03}/{self.n_epochs:03} | Train Time: {epoch_train_time:.3f}s ' \ + f'| Train Loss: {train_loss:.6f}' + if validate: + stats = stats + f' | Validation Loss: {test_loss:.6f}' + logger.info(stats) self.train_time = time.time() - start_time logger.info('Training Time: {:.3f}s'.format(self.train_time)) From 3c9f9acb8c00cdb88e368ee4edc62a5fee0babd7 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Mueller Date: Wed, 6 May 2020 16:48:11 +1000 Subject: [PATCH 4/5] Use validation loader for validation --- src/optim/DeepSAD_trainer.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/optim/DeepSAD_trainer.py b/src/optim/DeepSAD_trainer.py index 720a03f79..673956c1e 100644 --- a/src/optim/DeepSAD_trainer.py +++ b/src/optim/DeepSAD_trainer.py @@ -37,7 +37,7 @@ def train(self, dataset: BaseADDataset, net: BaseNet, validate: bool = False): logger = logging.getLogger() # Get train data loader - train_loader, test_loader = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) + train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) # Set device for network net = net.to(self.device) @@ -91,9 +91,10 @@ def train(self, dataset: BaseADDataset, net: BaseNet, validate: bool = False): if validate: n_batches = 0 - test_epoch_loss = 0.0 + valid_epoch_loss = 0.0 + valid_loader = dataset.validation_loader(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) with torch.set_grad_enabled(False): - for data in test_loader: + for data in valid_loader: inputs, _, semi_targets, _ = data inputs, semi_targets = inputs.to(self.device), semi_targets.to(self.device) @@ -102,10 +103,10 @@ def train(self, dataset: BaseADDataset, net: BaseNet, validate: bool = False): losses = torch.where(semi_targets == 0, dist, self.eta * ((dist + self.eps) ** semi_targets.float())) loss = torch.mean(losses) - test_epoch_loss += loss.item() + valid_epoch_loss += loss.item() n_batches += 1 - test_loss = test_epoch_loss/n_batches - epoch_loss_history = (epoch + 1, train_loss, test_loss) + valid_loss = valid_epoch_loss/n_batches + epoch_loss_history = (epoch + 1, train_loss, valid_loss) self.train_loss.append(epoch_loss_history) # log epoch statistics @@ -114,7 +115,7 @@ def train(self, dataset: BaseADDataset, net: BaseNet, validate: bool = False): stats = f'| Epoch: {epoch + 1:03}/{self.n_epochs:03} | Train Time: {epoch_train_time:.3f}s ' \ f'| Train Loss: {train_loss:.6f}' if validate: - stats = stats + f' | Validation Loss: {test_loss:.6f}' + stats = stats + f' | Valid Loss: {valid_loss:.6f}' logger.info(stats) self.train_time = time.time() - start_time From b7b468c6ffc72f1bf461487aa520e559a6bec3b3 Mon Sep 17 00:00:00 2001 From: Yuriy Tyshetskiy Date: Fri, 5 Jun 2020 13:35:17 +1000 Subject: [PATCH 5/5] Changed lr scheduler to self.scheduler in DeepSADtrainer --- src/optim/DeepSAD_trainer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/optim/DeepSAD_trainer.py b/src/optim/DeepSAD_trainer.py index 673956c1e..486303b9d 100644 --- a/src/optim/DeepSAD_trainer.py +++ b/src/optim/DeepSAD_trainer.py @@ -46,13 +46,13 @@ def train(self, dataset: BaseADDataset, net: BaseNet, validate: bool = False): optimizer = optim.Adam(net.parameters(), lr=self.lr, weight_decay=self.weight_decay) # Set learning rate scheduler - scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=self.lr_milestones, gamma=0.1) + self.scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=self.lr_milestones, gamma=0.1) # Initialize hypersphere center c (if c not loaded) if self.c is None: logger.info('Initializing center c...') self.c = self.init_center_c(train_loader, net) - logger.info('Center c initialized.') + logger.info('Center c initialized to {}.'.format(self.c)) # Training logger.info('Starting training...') @@ -61,9 +61,9 @@ def train(self, dataset: BaseADDataset, net: BaseNet, validate: bool = False): self.train_loss = [] for epoch in range(self.n_epochs): - scheduler.step() + self.scheduler.step() if epoch in self.lr_milestones: - logger.info(' LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0])) + logger.info(' LR scheduler: new learning rate is %g' % float(self.scheduler.get_lr()[0])) train_epoch_loss = 0.0 n_batches = 0