update: fpr and tpr metric for Carlini et al(2021)

inference_attack.py

@@ -5,6 +5,7 @@
 import torchvision
 import torchvision.transforms as transforms
 from utils.seed import seed_everything
+from utils.load_config import load_config
 import pandas as pd
 import numpy as np
 import yaml
@@ -23,11 +24,10 @@
 import lightgbm as lgb
 from catboost import CatBoostClassifier
 
-# Read config.yaml file
-with open("config.yaml") as infile:
-    SAVED_CFG = yaml.load(infile, Loader=yaml.FullLoader)
-    CFG = EasyDict(SAVED_CFG["CFG"])
-    CFG_ATTACK = EasyDict(SAVED_CFG["CFG_ATTACK"])
+
+# load config
+CFG = load_config("CFG")
+CFG_ATTACK = load_config("CFG_ATTACK")
 
 # seed for future replication
 seed_everything(CFG.seed)

train_attack.py

@@ -1,13 +1,11 @@
 from utils.seed import seed_everything
+from utils.load_config import load_config
 import pandas as pd
 import numpy as np
-import yaml
-from easydict import EasyDict
-from joblib import dump, load
 
 # get metric and train, test support
 from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, precision_recall_fscore_support
+from sklearn.metrics import precision_recall_fscore_support, roc_curve
 
 # get classifier models
 from sklearn.ensemble import RandomForestClassifier
@@ -16,11 +14,9 @@
 import lightgbm as lgb
 from catboost import CatBoostClassifier
 
-# Read config.yaml file
-with open("config.yaml") as infile:
-    SAVED_CFG = yaml.load(infile, Loader=yaml.FullLoader)
-    CFG = EasyDict(SAVED_CFG["CFG"])
-    CFG_ATTACK = EasyDict(SAVED_CFG["CFG_ATTACK"])
+# load config
+CFG = load_config("CFG")
+CFG_ATTACK = load_config("CFG_ATTACK")
 
 # seed for future replication
 seed_everything(CFG.seed)
@@ -39,14 +35,12 @@
 )
 
 
-# fit model: https://github.com/snoop2head/ml_classification_tutorial/blob/main/ML_Classification.ipynb
+# fit attack model: https://github.com/snoop2head/ml_classification_tutorial/blob/main/ML_Classification.ipynb
 # model = xgb.XGBClassifier(n_estimators=CFG_ATTACK.n_estimators, n_jobs=-1, random_state=CFG.seed)
 # model = lgb.LGBMClassifier(n_estimators=CFG_ATTACK.n_estimators, n_jobs=-1, random_state=CFG.seed)
-
-# https://catboost.ai/en/docs/concepts/loss-functions-classification
 model = CatBoostClassifier(
     iterations=200, depth=2, learning_rate=0.25, loss_function="Logloss", verbose=True
-)
+)  # https://catboost.ai/en/docs/concepts/loss-functions-classification
 
 model.fit(X_train, y_train)
 accuracy = model.score(X_test, y_test)
@@ -57,6 +51,13 @@
 print("precision:", precision)
 print("recall:", recall)
 print("f1_score:", f1_score)
+
+fpr, tpr, thresholds = roc_curve(y_test, model.predict_proba(X_test)[:, 1])
+print("fpr:", np.mean(fpr))
+print("tpr:", np.mean(tpr))
+print("thresholds:", thresholds)
+#
+
 save_path = f"./attack/{model.__class__.__name__}_{accuracy}"
 # dump(model, save_path)
 model.save_model(save_path)

train_shadow.py

@@ -1,6 +1,7 @@
 from shadow.trainer import train
 from shadow.make_data import make_member_nonmember
 from utils.seed import seed_everything
+from utils.load_config import load_config
 import os
 import torch
 import torchvision
@@ -17,10 +18,8 @@
 import wandb
 import importlib
 
-# Read config.yaml file
-with open("config.yaml") as infile:
-    SAVED_CFG = yaml.load(infile, Loader=yaml.FullLoader)
-    CFG = EasyDict(SAVED_CFG["CFG"])
+# load config
+CFG = load_config("CFG")
 
 # conduct training
 if not os.path.exists(CFG.save_path):

train_target.py

@@ -1,6 +1,7 @@
 from shadow.trainer import train
 from shadow.make_data import make_member_nonmember
 from utils.seed import seed_everything
+from utils.load_config import load_config
 import os
 import torch
 import torchvision
@@ -17,10 +18,9 @@
 import wandb
 import importlib
 
-# Read config.yaml file
-with open("config.yaml") as infile:
-    SAVED_CFG = yaml.load(infile, Loader=yaml.FullLoader)
-    CFG = EasyDict(SAVED_CFG["CFG"])
+# load config
+CFG = load_config("CFG")
+
 
 # seed for future replication
 seed_everything(CFG.seed)

utils/load_config.py

@@ -0,0 +1,10 @@
+from easydict import EasyDict
+import yaml
+
+
+def load_config(config_name, config_path="./config.yaml"):
+    # Read config.yaml file
+    with open(config_path) as infile:
+        SAVED_CFG = yaml.load(infile, Loader=yaml.FullLoader)
+        CFG = EasyDict(SAVED_CFG[config_name])
+    return CFG