diff --git a/use_case_examples/deployment/README.md b/use_case_examples/deployment/README.md new file mode 100644 index 000000000..38baac25b --- /dev/null +++ b/use_case_examples/deployment/README.md @@ -0,0 +1,42 @@ +# Deployment Examples + +This folder contains examples of how to deploy Concrete ML models using Fully Homomorphic Encryption (FHE). These examples demonstrate the process of training, compiling, and deploying models for various use cases. + +## Overview + +The deployment process generally follows these steps: + +1. Train the model (optional, depending on the use case) +2. Compile the model to an FHE circuit +3. Deploy the model using Docker +4. Run inference using a client (locally or in Docker) + +## Available Examples + +We provide three different use cases to demonstrate the deployment process: + +1. [Breast Cancer Classification](./breast_cancer/README.md) +2. [Sentiment Analysis](./sentiment_analysis/README.md) +3. [CIFAR-10 Image Classification](./cifar/README.md) + +## Getting Started + +Each example folder contains its own README with specific instructions. However, the general process is similar: + +1. Train or compile the model using the provided scripts +2. Deploy the model using `deploy_to_docker.py` from the `server` folder +3. Build the client Docker image +4. Run the client to interact with the deployed model + +For detailed instructions, please refer to the README in each example folder. + +## Requirements + +- Docker +- Python 3.8 or later +- Concrete ML library installed + +## Additional Resources + +- [Client-Server Guide](../../docs/guides/client_server.md) +- [Server Deployment Scripts](./server/README.md) diff --git a/use_case_examples/deployment/cifar/models/__init__.py b/use_case_examples/deployment/cifar/models/__init__.py new file mode 100644 index 000000000..16fb2cc19 --- /dev/null +++ b/use_case_examples/deployment/cifar/models/__init__.py @@ -0,0 +1,64 @@ +# MIT License +# +# Copyright (c) 2019 Xilinx +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# Original file can be found at https://github.com/Xilinx/brevitas/blob/8c3d9de0113528cf6693c6474a13d802a66682c6/src/brevitas_examples/bnn_pynq/models/__init__.py + +import os +from configparser import ConfigParser + +import torch +from torch import hub + +__all__ = ["cnv_2w2a"] + +from .model import cnv + +model_impl = { + "CNV": cnv, +} + + +def get_model_cfg(name): + cfg = ConfigParser() + current_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(current_dir, name.lower() + ".ini") + assert os.path.exists(config_path) + cfg.read(config_path) + return cfg + + +def model_with_cfg(name, pre_trained): + cfg = get_model_cfg(name) + arch = cfg.get("MODEL", "ARCH") + model = model_impl[arch](cfg) + if pre_trained: + checkpoint = cfg.get("MODEL", "PRETRAINED_URL") + state_dict = hub.load_state_dict_from_url(checkpoint, progress=True, map_location="cpu") + model.load_state_dict(state_dict, strict=True) + return model, cfg + + +def cnv_2w2a(pre_trained=False): + assert ( + pre_trained == False + ), "No online pre-trained network are available. Use --resume instead with a valid checkpoint." + model, _ = model_with_cfg("cnv_2w2a", pre_trained) + return model diff --git a/use_case_examples/deployment/cifar/models/cnv_2w2a.ini b/use_case_examples/deployment/cifar/models/cnv_2w2a.ini new file mode 100644 index 000000000..5735e3a90 --- /dev/null +++ b/use_case_examples/deployment/cifar/models/cnv_2w2a.ini @@ -0,0 +1,13 @@ +[MODEL] +ARCH: CNV +PRETRAINED_URL: https://github.com/Xilinx/brevitas/releases/download/bnn_pynq-r0/cnv_2w2a-0702987f.pth +EVAL_LOG: https://github.com/Xilinx/brevitas/releases/download/cnv_test_ref-r0/cnv_2w2a_eval-5aaca4c6.txt +DATASET: CIFAR10 +IN_CHANNELS: 3 +NUM_CLASSES: 10 + +[QUANT] +WEIGHT_BIT_WIDTH: 2 +ACT_BIT_WIDTH: 2 +IN_BIT_WIDTH: 8 + diff --git a/use_case_examples/deployment/cifar/models/common.py b/use_case_examples/deployment/cifar/models/common.py new file mode 100644 index 000000000..bcc5fff4c --- /dev/null +++ b/use_case_examples/deployment/cifar/models/common.py @@ -0,0 +1,79 @@ +# MIT License +# +# Copyright (c) 2019 Xilinx +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# Original file can be found at https://github.com/Xilinx/brevitas/blob/8c3d9de0113528cf6693c6474a13d802a66682c6/src/brevitas_examples/bnn_pynq/models/common.py +from brevitas.core.bit_width import BitWidthImplType +from brevitas.core.quant import QuantType +from brevitas.core.restrict_val import FloatToIntImplType, RestrictValueType +from brevitas.core.scaling import ScalingImplType +from brevitas.core.zero_point import ZeroZeroPoint +from brevitas.inject import ExtendedInjector +from brevitas.quant import Int8WeightPerTensorFloat +from brevitas.quant.solver import ActQuantSolver, WeightQuantSolver +from dependencies import value + + +class CommonQuant(ExtendedInjector): + bit_width_impl_type = BitWidthImplType.CONST + scaling_impl_type = ScalingImplType.CONST + restrict_scaling_type = RestrictValueType.FP + zero_point_impl = ZeroZeroPoint + float_to_int_impl_type = FloatToIntImplType.ROUND + scaling_per_output_channel = False + narrow_range = True + signed = True + + @value + def quant_type(bit_width): + if bit_width is None: + return QuantType.FP + elif bit_width == 1: + return QuantType.BINARY + else: + return QuantType.INT + + +class CommonIntWeightPerTensorQuant(Int8WeightPerTensorFloat): + """ + Common per-tensor weight quantizer with bit-width set to None so that it is forced to be + specified by each layer. + """ + + scaling_min_val = 2e-16 + bit_width = None + + +class CommonIntWeightPerChannelQuant(CommonIntWeightPerTensorQuant): + """ + Common per-channel weight quantizer with bit-width set to None so that it is forced to be + specified by each layer. + """ + + scaling_per_output_channel = True + + +class CommonWeightQuant(CommonQuant, WeightQuantSolver): + scaling_const = 1.0 + + +class CommonActQuant(CommonQuant, ActQuantSolver): + min_val = -1.0 + max_val = 1.0 diff --git a/use_case_examples/deployment/cifar/models/model.py b/use_case_examples/deployment/cifar/models/model.py new file mode 100644 index 000000000..951703826 --- /dev/null +++ b/use_case_examples/deployment/cifar/models/model.py @@ -0,0 +1,141 @@ +# MIT License +# +# Copyright (c) 2019 Xilinx +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# Original file can be found at https://github.com/Xilinx/brevitas/blob/8c3d9de0113528cf6693c6474a13d802a66682c6/src/brevitas_examples/bnn_pynq/models/CNV.py + +import torch +from brevitas.core.restrict_val import RestrictValueType +from brevitas.nn import QuantConv2d, QuantIdentity, QuantLinear +from torch.nn import AvgPool2d, BatchNorm1d, BatchNorm2d, Module, ModuleList + +from .common import CommonActQuant, CommonWeightQuant +from .tensor_norm import TensorNorm + +CNV_OUT_CH_POOL = [(64, False), (64, True), (128, False), (128, True), (256, False), (256, False)] +INTERMEDIATE_FC_FEATURES = [(256, 512), (512, 512)] +LAST_FC_IN_FEATURES = 512 +LAST_FC_PER_OUT_CH_SCALING = False +POOL_SIZE = 2 +KERNEL_SIZE = 3 + + +class CNV(Module): + def __init__(self, num_classes, weight_bit_width, act_bit_width, in_bit_width, in_ch): + super(CNV, self).__init__() + + self.conv_features = ModuleList() + self.linear_features = ModuleList() + + self.conv_features.append( + QuantIdentity( # for Q1.7 input format + act_quant=CommonActQuant, + return_quant_tensor=True, + bit_width=in_bit_width, + min_val=-1.0, + max_val=1.0 - 2.0 ** (-7), + narrow_range=False, + restrict_scaling_type=RestrictValueType.POWER_OF_TWO, + ) + ) + + for out_ch, is_pool_enabled in CNV_OUT_CH_POOL: + self.conv_features.append( + QuantConv2d( + kernel_size=KERNEL_SIZE, + in_channels=in_ch, + out_channels=out_ch, + bias=False, + weight_quant=CommonWeightQuant, + weight_bit_width=weight_bit_width, + ) + ) + in_ch = out_ch + self.conv_features.append(BatchNorm2d(in_ch, eps=1e-4)) + self.conv_features.append( + QuantIdentity(act_quant=CommonActQuant, bit_width=act_bit_width) + ) + if is_pool_enabled: + self.conv_features.append(AvgPool2d(kernel_size=2)) + self.conv_features.append( + QuantIdentity(act_quant=CommonActQuant, bit_width=act_bit_width) + ) + + for in_features, out_features in INTERMEDIATE_FC_FEATURES: + self.linear_features.append( + QuantLinear( + in_features=in_features, + out_features=out_features, + bias=False, + weight_quant=CommonWeightQuant, + weight_bit_width=weight_bit_width, + ) + ) + self.linear_features.append(BatchNorm1d(out_features, eps=1e-4)) + self.linear_features.append( + QuantIdentity(act_quant=CommonActQuant, bit_width=act_bit_width) + ) + + self.linear_features.append( + QuantLinear( + in_features=LAST_FC_IN_FEATURES, + out_features=num_classes, + bias=False, + weight_quant=CommonWeightQuant, + weight_bit_width=weight_bit_width, + ) + ) + self.linear_features.append(TensorNorm()) + + for m in self.modules(): + if isinstance(m, QuantConv2d) or isinstance(m, QuantLinear): + torch.nn.init.uniform_(m.weight.data, -1, 1) + + def clip_weights(self, min_val, max_val): + for mod in self.conv_features: + if isinstance(mod, QuantConv2d): + mod.weight.data.clamp_(min_val, max_val) + for mod in self.linear_features: + if isinstance(mod, QuantLinear): + mod.weight.data.clamp_(min_val, max_val) + + def forward(self, x): + for mod in self.conv_features: + x = mod(x) + x = torch.flatten(x, 1) + for mod in self.linear_features: + x = mod(x) + return x + + +def cnv(cfg): + weight_bit_width = cfg.getint("QUANT", "WEIGHT_BIT_WIDTH") + act_bit_width = cfg.getint("QUANT", "ACT_BIT_WIDTH") + in_bit_width = cfg.getint("QUANT", "IN_BIT_WIDTH") + num_classes = cfg.getint("MODEL", "NUM_CLASSES") + in_channels = cfg.getint("MODEL", "IN_CHANNELS") + net = CNV( + weight_bit_width=weight_bit_width, + act_bit_width=act_bit_width, + in_bit_width=in_bit_width, + num_classes=num_classes, + in_ch=in_channels, + ) + return net diff --git a/use_case_examples/deployment/cifar/models/tensor_norm.py b/use_case_examples/deployment/cifar/models/tensor_norm.py new file mode 100644 index 000000000..1a13288b0 --- /dev/null +++ b/use_case_examples/deployment/cifar/models/tensor_norm.py @@ -0,0 +1,63 @@ +# MIT License +# +# Copyright (c) 2019 Xilinx +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# Original file can be found at https://github.com/Xilinx/brevitas/blob/8c3d9de0113528cf6693c6474a13d802a66682c6/src/brevitas_examples/bnn_pynq/models/tensor_norm.py + +import torch +import torch.nn as nn +import torch.nn.init as init + + +class TensorNorm(nn.Module): + def __init__(self, eps=1e-4, momentum=0.1): + super().__init__() + + self.eps = eps + self.momentum = momentum + self.weight = nn.Parameter(torch.rand(1)) + self.bias = nn.Parameter(torch.rand(1)) + self.register_buffer("running_mean", torch.zeros(1)) + self.register_buffer("running_var", torch.ones(1)) + self.reset_running_stats() + + def reset_running_stats(self): + self.running_mean.zero_() + self.running_var.fill_(1) + init.ones_(self.weight) + init.zeros_(self.bias) + + def forward(self, x): + if self.training: + mean = x.mean() + unbias_var = x.var(unbiased=True) + biased_var = x.var(unbiased=False) + self.running_mean = ( + 1 - self.momentum + ) * self.running_mean + self.momentum * mean.detach() + self.running_var = ( + 1 - self.momentum + ) * self.running_var + self.momentum * unbias_var.detach() + inv_std = 1 / (biased_var + self.eps).pow(0.5) + return (x - mean) * inv_std * self.weight + self.bias + else: + return ( + (x - self.running_mean) / (self.running_var + self.eps).pow(0.5) + ) * self.weight + self.bias diff --git a/use_case_examples/deployment/cifar/requirements.txt b/use_case_examples/deployment/cifar/requirements.txt index d5fa2d7c9..5f6747435 100644 --- a/use_case_examples/deployment/cifar/requirements.txt +++ b/use_case_examples/deployment/cifar/requirements.txt @@ -1,2 +1,3 @@ torchvision==0.14.1 Pillow +grequests \ No newline at end of file diff --git a/use_case_examples/deployment/server/README.md b/use_case_examples/deployment/server/README.md index cd6072365..bc90eec5e 100644 --- a/use_case_examples/deployment/server/README.md +++ b/use_case_examples/deployment/server/README.md @@ -5,15 +5,15 @@ We show-case how to do this on 3 examples: - Breast cancer classification using a simple XGBoost model - Sentiment analysis by running a XGBoost model on top of a Transformer model -- CIFAR-10 classification using a VGG model split in two parts. +- CIFAR-10 classification using a VGG model. -You can run these example locally using Docker, or on AWS if you have your credentials set up. +You can run these example locally using Docker. For all of them the workflow is the same: 1. Optional: Train the model 1. Compile the model to an FHE circuit -1. Deploy to AWS, Docker or localhost +1. Deploy to Docker or localhost 1. Run the inference using the client (locally or in Docker) The script to deploy the model compiled to an FHE circuit is the same for all. The main difference between them is the client. Each use-case needs its own client.