diff --git a/dataset/transform_cfg.py b/dataset/transform_cfg.py index ed9788d..ab74703 100644 --- a/dataset/transform_cfg.py +++ b/dataset/transform_cfg.py @@ -28,49 +28,62 @@ ]) ] -transform_A_test = [ + +transform_B = [ transforms.Compose([ lambda x: Image.fromarray(x), - transforms.RandomCrop(84, padding=8), + transforms.RandomResizedCrop(84, scale=(0.2, 1.0)), transforms.RandomHorizontalFlip(), lambda x: np.asarray(x), transforms.ToTensor(), - normalize + transforms.Normalize(np.array([x / 255.0 for x in [125.3, 123.0, 113.9]]), + np.array([x / 255.0 for x in [63.0, 62.1, 66.7]])) ]), transforms.Compose([ lambda x: Image.fromarray(x), + transforms.Resize(92), + transforms.CenterCrop(84), transforms.ToTensor(), - normalize + transforms.Normalize(np.array([x / 255.0 for x in [125.3, 123.0, 113.9]]), + np.array([x / 255.0 for x in [63.0, 62.1, 66.7]])) ]) ] -# CIFAR style transformation -mean = [0.5071, 0.4867, 0.4408] -std = [0.2675, 0.2565, 0.2761] -normalize_cifar100 = transforms.Normalize(mean=mean, std=std) -transform_D = [ +transform_C = [ transforms.Compose([ lambda x: Image.fromarray(x), - transforms.RandomCrop(32, padding=4), - transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), + # transforms.Resize(92, interpolation = PIL.Image.BICUBIC), + transforms.RandomResizedCrop(80), + # transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), transforms.RandomHorizontalFlip(), - lambda x: np.asarray(x), transforms.ToTensor(), - normalize_cifar100 + # Lighting(0.1, imagenet_pca['eigval'], imagenet_pca['eigvec']), + # normalize + transforms.Normalize(np.array([x / 255.0 for x in [125.3, 123.0, 113.9]]), + np.array([x / 255.0 for x in [63.0, 62.1, 66.7]])) ]), transforms.Compose([ lambda x: Image.fromarray(x), + transforms.Resize(92), + transforms.CenterCrop(80), transforms.ToTensor(), - normalize_cifar100 + # normalize + transforms.Normalize(np.array([x / 255.0 for x in [125.3, 123.0, 113.9]]), + np.array([x / 255.0 for x in [63.0, 62.1, 66.7]])) ]) ] -transform_D_test = [ +# CIFAR style transformation +mean = [0.5071, 0.4867, 0.4408] +std = [0.2675, 0.2565, 0.2761] +normalize_cifar100 = transforms.Normalize(mean=mean, std=std) +transform_D = [ transforms.Compose([ lambda x: Image.fromarray(x), transforms.RandomCrop(32, padding=4), + transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), transforms.RandomHorizontalFlip(), lambda x: np.asarray(x), transforms.ToTensor(), @@ -85,15 +98,12 @@ ] -transforms_list = ['A', 'D'] +transforms_list = ['A', 'B', 'C', 'D'] transforms_options = { 'A': transform_A, + 'B': transform_B, + 'C': transform_C, 'D': transform_D, } - -transforms_test_options = { - 'A': transform_A_test, - 'D': transform_D_test, -} diff --git a/eval/meta_eval.py b/eval/meta_eval.py index 544f352..c5244ac 100644 --- a/eval/meta_eval.py +++ b/eval/meta_eval.py @@ -7,11 +7,14 @@ import torch from sklearn import metrics -from sklearn.svm import SVC +from sklearn.svm import SVC, LinearSVC from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.ensemble import RandomForestClassifier +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + def mean_confidence_interval(data, confidence=0.95): a = 1.0 * np.array(data) @@ -27,7 +30,7 @@ def normalize(x): return out -def meta_test(net, testloader, use_logit=True, is_norm=True, classifier='LR'): +def meta_test(net, testloader, use_logit=True, is_norm=True, classifier='LR', opt=None): net = net.eval() acc = [] @@ -36,9 +39,9 @@ def meta_test(net, testloader, use_logit=True, is_norm=True, classifier='LR'): support_xs, support_ys, query_xs, query_ys = data support_xs = support_xs.cuda() query_xs = query_xs.cuda() - batch_size, _, height, width, channel = support_xs.size() - support_xs = support_xs.view(-1, height, width, channel) - query_xs = query_xs.view(-1, height, width, channel) + batch_size, _, channel, height, width = support_xs.size() + support_xs = support_xs.view(-1, channel, height, width) + query_xs = query_xs.view(-1, channel, height, width) if use_logit: support_features = net(support_xs).view(support_xs.size(0), -1) @@ -59,20 +62,29 @@ def meta_test(net, testloader, use_logit=True, is_norm=True, classifier='LR'): support_ys = support_ys.view(-1).numpy() query_ys = query_ys.view(-1).numpy() + # clf = SVC(gamma='auto', C=0.1) if classifier == 'LR': clf = LogisticRegression(penalty='l2', random_state=0, - C=10, + C=1.0, solver='lbfgs', max_iter=1000, - class_weight='balanced', multi_class='multinomial') clf.fit(support_features, support_ys) query_ys_pred = clf.predict(query_features) + elif classifier == 'SVM': + clf = make_pipeline(StandardScaler(), SVC(gamma='auto', + C=1, + kernel='linear', + decision_function_shape='ovr')) + clf.fit(support_features, support_ys) + query_ys_pred = clf.predict(query_features) elif classifier == 'NN': query_ys_pred = NN(support_features, support_ys, query_features) elif classifier == 'Cosine': query_ys_pred = Cosine(support_features, support_ys, query_features) + elif classifier == 'Proto': + query_ys_pred = Proto(support_features, support_ys, query_features, opt) else: raise NotImplementedError('classifier not supported: {}'.format(classifier)) @@ -81,6 +93,19 @@ def meta_test(net, testloader, use_logit=True, is_norm=True, classifier='LR'): return mean_confidence_interval(acc) +def Proto(support, support_ys, query, opt): + """Protonet classifier""" + nc = support.shape[-1] + support = np.reshape(support, (-1, 1, opt.n_ways, opt.n_shots, nc)) + support = support.mean(axis=3) + batch_size = support.shape[0] + query = np.reshape(query, (batch_size, -1, 1, nc)) + logits = - ((query - support)**2).sum(-1) + pred = np.argmax(logits, axis=-1) + pred = np.reshape(pred, (-1,)) + return pred + + def NN(support, support_ys, query): """nearest classifier""" support = np.expand_dims(support.transpose(), 0) diff --git a/eval_fewshot.py b/eval_fewshot.py index a56aeba..438fae2 100644 --- a/eval_fewshot.py +++ b/eval_fewshot.py @@ -1,26 +1,30 @@ from __future__ import print_function +import os import argparse import socket import time +import sys import torch import torch.backends.cudnn as cudnn from torch.utils.data import DataLoader -from models import model_pool +from models import model_dict, model_pool from models.util import create_model -from dataset.mini_imagenet import MetaImageNet -from dataset.tiered_imagenet import MetaTieredImageNet -from dataset.cifar import MetaCIFAR100 -from dataset.transform_cfg import transforms_test_options, transforms_list +from dataset.mini_imagenet import ImageNet, MetaImageNet +from dataset.tiered_imagenet import TieredImageNet, MetaTieredImageNet +from dataset.cifar import CIFAR100, MetaCIFAR100 +from dataset.transform_cfg import transforms_options, transforms_list from eval.meta_eval import meta_test def parse_option(): + hostname = socket.gethostname() + parser = argparse.ArgumentParser('argument for training') # load pretrained model @@ -32,9 +36,6 @@ def parse_option(): 'CIFAR-FS', 'FC100']) parser.add_argument('--transform', type=str, default='A', choices=transforms_list) - # specify data_root - parser.add_argument('--data_root', type=str, default='', help='path to data root') - # meta setting parser.add_argument('--n_test_runs', type=int, default=600, metavar='N', help='Number of test runs') @@ -46,6 +47,8 @@ def parse_option(): help='Number of query in test') parser.add_argument('--n_aug_support_samples', default=5, type=int, help='The number of augmented samples for each meta test sample') + parser.add_argument('--data_root', type=str, default='data', metavar='N', + help='Root dataset') parser.add_argument('--num_workers', type=int, default=3, metavar='N', help='Number of workers for dataloader') parser.add_argument('--test_batch_size', type=int, default=1, metavar='test_batch_size', @@ -59,11 +62,16 @@ def parse_option(): opt.use_trainval = False # set the path according to the environment - if not opt.data_root: - opt.data_root = './data/{}'.format(opt.dataset) + if hostname.startswith('visiongpu'): + opt.data_root = '/data/vision/phillipi/rep-learn/{}'.format(opt.dataset) + opt.data_aug = True + elif hostname.startswith('instance'): + opt.data_root = '/mnt/globalssd/fewshot/{}'.format(opt.dataset) + opt.data_aug = True + elif opt.data_root != 'data': + opt.data_aug = True else: - opt.data_root = '{}/{}'.format(opt.data_root, opt.dataset) - opt.data_aug = True + raise NotImplementedError('server invalid: {}'.format(hostname)) return opt @@ -74,9 +82,11 @@ def main(): # test loader args = opt + args.batch_size = args.test_batch_size + # args.n_aug_support_samples = 1 if opt.dataset == 'miniImageNet': - train_trans, test_trans = transforms_test_options[opt.transform] + train_trans, test_trans = transforms_options[opt.transform] meta_testloader = DataLoader(MetaImageNet(args=opt, partition='test', train_transform=train_trans, test_transform=test_trans, @@ -94,7 +104,7 @@ def main(): else: n_cls = 64 elif opt.dataset == 'tieredImageNet': - train_trans, test_trans = transforms_test_options[opt.transform] + train_trans, test_trans = transforms_options[opt.transform] meta_testloader = DataLoader(MetaTieredImageNet(args=opt, partition='test', train_transform=train_trans, test_transform=test_trans, @@ -112,7 +122,7 @@ def main(): else: n_cls = 351 elif opt.dataset == 'CIFAR-FS' or opt.dataset == 'FC100': - train_trans, test_trans = transforms_test_options['D'] + train_trans, test_trans = transforms_options['D'] meta_testloader = DataLoader(MetaCIFAR100(args=opt, partition='test', train_transform=train_trans, test_transform=test_trans, @@ -150,22 +160,28 @@ def main(): start = time.time() val_acc, val_std = meta_test(model, meta_valloader) val_time = time.time() - start - print('val_acc: {:.4f}, val_std: {:.4f}, time: {:.1f}'.format(val_acc, val_std, val_time)) + print('val_acc: {:.4f}, val_std: {:.4f}, time: {:.1f}'.format(val_acc, val_std, + val_time)) start = time.time() val_acc_feat, val_std_feat = meta_test(model, meta_valloader, use_logit=False) val_time = time.time() - start - print('val_acc_feat: {:.4f}, val_std: {:.4f}, time: {:.1f}'.format(val_acc_feat, val_std_feat, val_time)) + print('val_acc_feat: {:.4f}, val_std: {:.4f}, time: {:.1f}'.format(val_acc_feat, + val_std_feat, + val_time)) start = time.time() test_acc, test_std = meta_test(model, meta_testloader) test_time = time.time() - start - print('test_acc: {:.4f}, test_std: {:.4f}, time: {:.1f}'.format(test_acc, test_std, test_time)) + print('test_acc: {:.4f}, test_std: {:.4f}, time: {:.1f}'.format(test_acc, test_std, + test_time)) start = time.time() test_acc_feat, test_std_feat = meta_test(model, meta_testloader, use_logit=False) test_time = time.time() - start - print('test_acc_feat: {:.4f}, test_std: {:.4f}, time: {:.1f}'.format(test_acc_feat, test_std_feat, test_time)) + print('test_acc_feat: {:.4f}, test_std: {:.4f}, time: {:.1f}'.format(test_acc_feat, + test_std_feat, + test_time)) if __name__ == '__main__':