inference with shadow model & attack model to target model

.gitignore

@@ -8,4 +8,5 @@ logs
 *.joblib
 *.tsv
 *.tfevents
-*.json
+*.json
+attack/*

config.yaml

@@ -24,4 +24,5 @@ CFG_ATTACK:
   attack_dset_path: ./attack/ResNet_pretrained_False_num_shadow_100.csv
   test_size: 0.2
   n_estimators: 100
-  attack_model_path: ./attack/XGBClassifier.joblib
+  target_model_path: ./ckpt/target_loss_ 2.4_acc5_93.68000030517578.ckpt
+  attack_model_path: ./attack/CatBoostClassifier_0.57078

inference_attack.py

@@ -1,16 +1,27 @@
+# -*- coding: utf-8 -*-
+from shadow.make_data import make_member_nonmember
+import torch
+from torch import nn
+import torchvision
+import torchvision.transforms as transforms
 from utils.seed import seed_everything
 import pandas as pd
 import numpy as np
 import yaml
 from easydict import EasyDict
 from joblib import dump, load
+import importlib
+
+# get metric and train, test support
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, precision_recall_fscore_support
 
 # get classifier models
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.linear_model import RidgeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, precision_recall_fscore_support
 import xgboost as xgb
+import lightgbm as lgb
+from catboost import CatBoostClassifier
 
 # Read config.yaml file
 with open("config.yaml") as infile:
@@ -21,5 +32,76 @@
 # seed for future replication
 seed_everything(CFG.seed)
 
+# Load the CIFAR dataset
+# CIFAR train is used for shadow model train & evaluation
+# CIFAR test is used for target model train & evaluation
+if CFG.num_classes == 10:
+    DSET_CLASS = torchvision.datasets.CIFAR10
+elif CFG.num_classes == 100:
+    DSET_CLASS = torchvision.datasets.CIFAR100
+
+transform = transforms.Compose(
+    [
+        transforms.Resize((CFG.input_resolution, CFG.input_resolution)),
+        transforms.ToTensor(),
+        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
+    ]
+)
+
+testset = DSET_CLASS(root="./data", train=False, download=True, transform=transform)
+trainset = DSET_CLASS(root="./data", train=True, download=True, transform=transform)
+
+print("mapped classes to ids:", testset.class_to_idx)
+columns_attack_sdet = [f"top_{index}_prob" for index in range(CFG.topk_num_accessible_probs)]
+
+# load member data
+list_nonmember_indices = pd.read_csv("./attack/train_indices.csv")["index"].to_list()
+list_member_indices = np.random.choice(len(testset), len(list_nonmember_indices), replace=False)
+
+subset_nonmember = torch.utils.data.Subset(trainset, list_nonmember_indices)
+subset_member = torch.utils.data.Subset(testset, list_member_indices)
+
+subset_nonmember_loader = torch.utils.data.DataLoader(
+    subset_nonmember, batch_size=CFG.train_batch_size, shuffle=True, num_workers=2
+)
+
+subset_member_loader = torch.utils.data.DataLoader(
+    subset_member, batch_size=CFG.train_batch_size, shuffle=True, num_workers=2
+)
+
+# target model loading (equivalent to API model that yields the prediction and logit)
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+model_architecture = importlib.import_module("torchvision.models")
+model_class = getattr(model_architecture, CFG.model_architecture)
+target_model = model_class(pretrained=CFG.bool_pretrained)
+target_model.fc = nn.Linear(in_features=target_model.fc.in_features, out_features=CFG.num_classes)
+target_model.to(device)
+target_model.load_state_dict(torch.load(CFG_ATTACK.target_model_path))
+target_model.eval()
+
+# target model request with member and nonmember data
+member_dset, non_member_dset = make_member_nonmember(
+    target_model, subset_member_loader, subset_nonmember_loader, nn.CrossEntropyLoss(), device
+)
+df_member = pd.DataFrame(member_dset, columns=columns_attack_sdet)
+df_member["is_member"] = 1
+df_non_member = pd.DataFrame(non_member_dset, columns=columns_attack_sdet)
+df_non_member["is_member"] = 0
+df_target_inference = pd.concat([df_member, df_non_member])
+
 # load model from the path
-model_loaded = load(CFG_ATTACK.attack_model_path)
+attack_model = CatBoostClassifier()
+attack_model.load_model(CFG_ATTACK.attack_model_path)
+X_test = df_target_inference[columns_attack_sdet].to_numpy()
+y_true = df_target_inference["is_member"].to_numpy()
+y_pred = attack_model.predict(X_test)
+
+# get accuracy, precision, recall, f1-score
+precision = precision_recall_fscore_support(y_true, y_pred, average="macro")[0]
+recall = precision_recall_fscore_support(y_true, y_pred, average="macro")[1]
+f1_score = precision_recall_fscore_support(y_true, y_pred, average="macro")[2]
+accuracy = accuracy_score(y_true, y_pred)
+print("precision:", precision)
+print("recall:", recall)
+print("f1-score:", f1_score)
+print("accuracy:", accuracy)

train_attack.py

@@ -39,13 +39,16 @@
 # fit model: https://github.com/snoop2head/ml_classification_tutorial/blob/main/ML_Classification.ipynb
 # model = xgb.XGBClassifier(n_estimators=CFG_ATTACK.n_estimators, n_jobs=-1, random_state=CFG.seed)
 # model = lgb.LGBMClassifier(n_estimators=CFG_ATTACK.n_estimators, n_jobs=-1, random_state=CFG.seed)
+
+# https://catboost.ai/en/docs/concepts/loss-functions-classification
 model = CatBoostClassifier(
-    iterations=100, depth=2, learning_rate=1, loss_function="Logloss", verbose=True
+    iterations=100, depth=2, learning_rate=0.25, loss_function="Logloss", verbose=True
 )
 
 model.fit(X_train, y_train)
 accuracy = model.score(X_test, y_test)
 print(accuracy)
-save_path = f"./attack/{model.__class__.__name__}_{accuracy}.joblib"
-dump(model, save_path)
+save_path = f"./attack/{model.__class__.__name__}_{accuracy}"
+# dump(model, save_path)
+model.save_model(save_path)
 

train_target.py

@@ -63,6 +63,10 @@
 
 target_train_indices = np.random.choice(len(testset), CFG.target_train_size, replace=False)
 target_eval_indices = np.setdiff1d(np.arange(len(testset)), target_train_indices)
+# save target_train_indices as dataframe
+pd.DataFrame(target_train_indices, columns=["index"]).to_csv(
+    "./attack/train_indices.csv", index=False
+)
 
 subset_tgt_train = torch.utils.data.Subset(testset, target_train_indices)
 subset_tgt_eval = torch.utils.data.Subset(testset, target_eval_indices)