Worker key refresh policy implemenation

This feature initiates refresh of worker encryption key pair based on
number of work orders processed in case of Singleton worker or number of
pre-processed work orders in case of KME worker.
A new pair of encryption key is generated in the enclave and the updated
enclave signup details are stored in the KvStorage in workers table.

Worker encryption key signature is re-computed when encryption key gets refreshed.

When a worker key gets refreshed during the work order submission,
a specific error code is returned to client to indicate worker key refresh.
On receiving this error code, client retrieves the updated worker details and
does work order submission again.

Signed-off-by: manju956 <[email protected]>

common/cpp/error.h

@@ -52,6 +52,13 @@ namespace tcf {
                 ) : Error(TCF_ERR_CRYPTO, msg) {}
         }; // class CryptoError
 
+        // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+        class KeyRefreshError : public Error {
+        public:
+            explicit KeyRefreshError(
+                const std::string& msg
+                ) : Error(TCF_ERR_ENCRYPT_KEY_REFRESH, msg) {}
+        }; // class KeyRefreshError
 
         // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
         class MemoryError : public Error {

common/cpp/tcf_error.h

@@ -42,7 +42,9 @@ typedef enum {
     TCF_ERR_SYSTEM_BUSY = -10,
     TCF_ERR_CRYPTO = -11,
     /** Invalid workload ID */
-    TCF_ERR_INVALID_WORKLOAD = -12
+    TCF_ERR_INVALID_WORKLOAD = -12,
+    /* Enclave key refresh error */
+    TCF_ERR_ENCRYPT_KEY_REFRESH = -13
 } tcf_err_t;
 
 typedef enum {

common/python/error_code/enclave_error.py

@@ -31,4 +31,7 @@ class EnclaveError(IntEnum):
     # this should be converted to ENCLAVE_ERR_SYSTEM for reporting.
     ENCLAVE_ERR_SYSTEM_BUSY = -10,
     ENCLAVE_ERR_CRYPTO = -11,
-    ENCLAVE_ERR_INVALID_WORKLOAD = -12  # Invalid workload ID
+    # Invalid workload ID
+    ENCLAVE_ERR_INVALID_WORKLOAD = -12,
+    # Worker encryption key refresh error
+    ENCLAVE_ERR_ENCRYPT_KEY_REFRESH = -13

common/python/error_code/error_status.py

@@ -33,7 +33,8 @@ class WorkOrderStatus(IntEnum):
     PROCESSING = 7
     BUSY = 8
     INVALID_WORKLOAD = 9
-    UNKNOWN_ERROR = 10
+    WORKER_ENCRYPT_KEY_REFRESHED = 10
+    UNKNOWN_ERROR = 11
 
 
 @unique

config/kme_config.toml

@@ -105,3 +105,8 @@ DataEncryptionAlgorithm = "AES-GCM-256"
 # Supported work order formats are JSON-RPC, JSON-RPC-JWT, and Custom format
 # starting with tilde "~"
 workOrderPayloadFormats = "JSON-RPC"
+
+[WorkerKeyRefresh]
+# Configure key refresh interval based on number of preprocessed work orders.
+# By default, key refresh feature is disabled.
+work_orders_count = 0

config/singleton_enclave_config.toml

@@ -116,3 +116,7 @@ DataEncryptionAlgorithm = "AES-GCM-256"
 # starting with tilde "~"
 workOrderPayloadFormats = "JSON-RPC"
 
+[WorkerKeyRefresh]
+# Configure key refresh interval based on number of processed work orders.
+# By default, key refresh feature is disabled.
+work_orders_count = 0

enclave_manager/avalon_enclave_manager/base_enclave_info.py

@@ -316,5 +316,3 @@ def _get_sealed_data_file_name(self, relative_path, worker_id):
             @returns file_name - Fully qualified file name for sealed data
         """
         return os.path.join(TCF_HOME, relative_path + "." + worker_id)
-
-    # -----------------------------------------------------------------

enclave_manager/avalon_enclave_manager/base_enclave_manager.py

@@ -18,9 +18,11 @@
 import json
 import logging
 import sys
-import utility.hex_utils as hex_utils
 from abc import ABC, abstractmethod
 
+import utility.hex_utils as hex_utils
+import utility.file_utils as file_utils
+
 from database import connector
 from avalon_enclave_manager.worker_kv_delegate import WorkerKVDelegate
 from avalon_enclave_manager.work_order_kv_delegate import WorkOrderKVDelegate
@@ -34,6 +36,8 @@ class EnclaveManager(ABC):
     Abstract base class for Enclave Manager
     """
 
+    signup_data = None
+
     def __init__(self, config):
 
         super().__init__()
@@ -143,10 +147,11 @@ def _setup_enclave(self):
             if signup_data is None:
                 logger.error("Failed to create signup data")
                 return None
+            EnclaveManager.signup_data = signup_data
         except Exception as e:
             logger.exception("failed to initialize/signup enclave; %s", str(e))
             sys.exit(-1)
-        return self._get_JSON_from_signup_object(signup_data)
+        return self._get_JSON_from_signup_object(EnclaveManager.signup_data)
 
     # -----------------------------------------------------------------
 
@@ -204,7 +209,8 @@ def create_json_worker(enclave_data, config):
         worker_type_data["verificationKey"] = enclave_data.verifying_key
         worker_type_data["extendedMeasurements"] = \
             enclave_data.extended_measurements
-        worker_type_data["proofDataType"] = enclave_data.proof_data_type
+        worker_type_data["proofDataType"] = \
+            config.get("WorkerConfig")["ProofDataType"]
         worker_type_data["proofData"] = enclave_data.proof_data
         worker_type_data["encryptionKey"] = enclave_data.encryption_key
         worker_type_data["encryptionKeySignature"] = \

enclave_manager/avalon_enclave_manager/kme/kme_enclave_info.py

@@ -25,6 +25,7 @@
 import utility.file_utils as file_utils
 import avalon_enclave_manager.kme.kme_enclave as enclave
 import avalon_enclave_manager.base_enclave_info as enclave_info
+import avalon_enclave_manager.worker_key_refresh as key_refresh
 
 logger = logging.getLogger(__name__)
 
@@ -41,9 +42,9 @@ def __init__(self, config, worker_id):
         enclave._SetLogger(logger)
         super().__init__(enclave.is_sgx_simulator())
 
-        self._config = config
+        self._config = config["EnclaveModule"]
         self._worker_id = worker_id
-        self._initialize_enclave(config)
+        self._initialize_enclave(self._config)
         enclave_info = self._create_enclave_signup_data()
         try:
             self.ias_nonce = enclave_info['ias_nonce']
@@ -58,6 +59,8 @@ def __init__(self, config, worker_id):
         except KeyError as ke:
             raise Exception("missing enclave initialization parameter; {}"
                             .format(str(ke)))
+        self.worker_key_refresh = key_refresh.WorkerKeyRefresh(
+            self, config, "kme")
 
     # -------------------------------------------------------
 

enclave_manager/avalon_enclave_manager/kme/kme_enclave_manager.py

@@ -42,6 +42,7 @@ def __init__(self, config):
 
         super().__init__(config)
         self.proof_data_type = config.get("WorkerConfig")["ProofDataType"]
+        self.preprocessed_wo_count = 0
 
 # -------------------------------------------------------------------------
 
@@ -54,8 +55,7 @@ def _create_signup_data(self):
                           enclave
         """
         return enclave_info.\
-            KeyManagementEnclaveInfo(self._config["EnclaveModule"],
-                                     self._worker_id)
+            KeyManagementEnclaveInfo(self._config, self._worker_id)
 
 # -------------------------------------------------------------------------
 
@@ -130,10 +130,8 @@ def start_enclave_manager(self):
             logger.error("Failed to execute boot time flow; " +
                          "exiting Intel SGX Enclave manager: {}".format(err))
             exit(1)
-
         self._start_kme_listener()
 
-
 # -------------------------------------------------------------------------
 
     def _start_kme_listener(self):
@@ -155,7 +153,6 @@ def _start_kme_listener(self):
         kme_listener = KMEListener(rpc_methods)
         kme_listener.start(host_name, port)
 
-
 # -----------------------------------------------------------------
 
     def GetUniqueVerificationKey(self, **params):
@@ -207,11 +204,30 @@ def RegisterWorkOrderProcessor(self, **params):
     def PreProcessWorkOrder(self, **params):
         """
         """
+        try:
+            wo_threshold = \
+                int(self._config["WorkerKeyRefresh"]["work_orders_count"])
+        except Exception as err:
+            logger.warning("Failed to get work order count from config file." +
+                           " Setting work orders threshold to 0: %s", str(err))
+            wo_threshold = 0
+
         wo_request = self._get_request_json("PreProcessWorkOrder")
         wo_request["params"] = params
         wo_response = self._execute_work_order(json.dumps(wo_request), "")
         wo_response_json = json.loads(wo_response)
 
+        self.preprocessed_wo_count += 1
+        if wo_threshold > 0 and self.preprocessed_wo_count == wo_threshold:
+            try:
+                enclave_info = EnclaveManager.signup_data
+                enclave_info.worker_key_refresh._initiate_key_refresh()
+                # Set preprocessed_wo_count to 0
+                self.preprocessed_wo_count = 0
+            except Exception as e:
+                logger.error("failed to get signup data after key refresh: %s",
+                             str(e))
+
         if "result" in wo_response_json:
             return wo_response_json["result"]
         else:

enclave_manager/avalon_enclave_manager/singleton/singleton_enclave_info.py

@@ -23,6 +23,7 @@
 from requests.exceptions import HTTPError
 import utility.file_utils as file_utils
 import avalon_enclave_manager.singleton.singleton_enclave as enclave
+import avalon_enclave_manager.worker_key_refresh as key_refresh
 from avalon_enclave_manager.base_enclave_info import BaseEnclaveInfo
 
 logger = logging.getLogger(__name__)
@@ -40,11 +41,11 @@ def __init__(self, config, worker_id):
         # Initialize the keys that can be used later to
         # register the enclave
         enclave._SetLogger(logger)
-        self._config = config
+        self._config = config["EnclaveModule"]
         self._worker_id = worker_id
         super().__init__(enclave.is_sgx_simulator())
 
-        self._initialize_enclave(config)
+        self._initialize_enclave(self._config)
         enclave_info = self._create_enclave_signup_data()
         try:
             self.ias_nonce = enclave_info['ias_nonce']
@@ -59,6 +60,8 @@ def __init__(self, config, worker_id):
         except KeyError as ke:
             raise Exception("missing enclave initialization parameter; {}"
                             .format(str(ke)))
+        self.worker_key_refresh = key_refresh.WorkerKeyRefresh(
+            self, config, "singleton")
 
     # -------------------------------------------------------
 

enclave_manager/avalon_enclave_manager/singleton/singleton_enclave_manager.py

@@ -73,8 +73,7 @@ def _create_signup_data(self):
                           enclave
         """
         return enclave_info.\
-            SingletonEnclaveInfo(self._config.get("EnclaveModule"),
-                                 self._worker_id)
+            SingletonEnclaveInfo(self._config, self._worker_id)
 
 # -------------------------------------------------------------------------
 
@@ -101,7 +100,6 @@ def _execute_wo_in_trusted_enclave(self, input_json_str):
 
 # -----------------------------------------------------------------
 
-
 def main(args=None):
     import config.config as pconfig
     import utility.logger as plogger

enclave_manager/avalon_enclave_manager/work_order_processor_manager.py

@@ -36,6 +36,7 @@ class WOProcessorManager(EnclaveManager):
     def __init__(self, config):
         super().__init__(config)
         self._identity = None
+        self.wo_processed_count = 0
 
 # -------------------------------------------------------------------------
 
@@ -76,6 +77,14 @@ def _process_work_orders(self):
             "About to process work orders found in wo-worker-scheduled table.")
 
         wo_id = self._kv_helper.csv_pop("wo-worker-scheduled", self._worker_id)
+        try:
+            wo_threshold = \
+                int(self._config["WorkerKeyRefresh"]["work_orders_count"])
+        except Exception as err:
+            logger.warning("Failed to get work order count from config file." +
+                           " Setting work orders threshold to 0: %s", str(err))
+            wo_threshold = 0
+
         while wo_id is not None:
 
             self._process_work_order_by_id(wo_id)
@@ -84,6 +93,18 @@ def _process_work_orders(self):
 
             wo_id = self._kv_helper.csv_pop("wo-worker-scheduled",
                                             self._worker_id)
+            self.wo_processed_count += 1
+            if wo_threshold > 0 and self.wo_processed_count == wo_threshold:
+                try:
+                    enclave_info = EnclaveManager.signup_data
+                    enclave_info.worker_key_refresh._initiate_key_refresh()
+                    # Set processed work orders count to 0
+                    self.wo_processed_count = 0
+                except Exception as e:
+                    logger.error(
+                        "failed to get signup data after key refresh: %s",
+                        str(e))
+
         # end of loop
         logger.info("No more worker orders in wo-worker-scheduled table.")
 
@@ -248,12 +269,13 @@ def _validate_request(self, wo_id, wo_request):
             return False
         return True
 
-    # -------------------------------------------------------------------------
+    # -----------------------------------------------------------------
 
     def start_enclave_manager(self):
         """
-        Execute boot flow and run time flow
+        Execute run time flow either by polling or by listening on ZMQ socket
         """
+
         try:
             logger.info(
                 "--------------- Starting Boot time flow ----------------")
@@ -277,7 +299,7 @@ def start_enclave_manager(self):
 
     def _start_polling_kvstore(self):
         """
-        This function is runs indefinitely polling the KV Storage
+        This function is run indefinitely polling the KV Storage
         for new work-order request and processing them. The poll
         is interleaved with sleeps hence avoiding busy waits. It
         terminates only when an exception occurs.
@@ -347,7 +369,7 @@ def _start_zmq_listener(self):
                 wo_id = socket.recv()
                 wo_id = wo_id.decode()
                 logger.info("Received request at enclave manager: %s" % wo_id)
-                result = self._process_work_order_sync(wo_id)
+                result = await self._process_work_order_sync(wo_id)
                 if result is None:
                     socket.send_string("Error while processing work order: " +
                                        str(wo_id))