add read only fields to memory, clean record (#23)

target_salesforce_v3/sinks.py

@@ -928,6 +928,10 @@ def preprocess_record(self, record, context):
         for key in record:
             if isinstance(record.get(key), datetime):
                 record[key] = record[key].isoformat()
+
+        # clean any read only fields
+        for field in self._target.read_only_fields.get(self.stream_name, []):
+            record.pop(field, None)
         return record
 
     def upsert_record(self, record, context):
@@ -1008,8 +1012,17 @@ def upsert_record(self, record, context):
             return id, True, state_updates
         except Exception as e:
             if "INVALID_FIELD_FOR_INSERT_UPDATE" in str(e):
-                fields = json.loads(str(e))[0]['fields']
+                try:
+                    fields = json.loads(str(e))[0]['fields']
+                except:
+                    raise Exception(f"Attempted to write read-only fields. Unable to extract read-only fields to retry request: {str(e)}")
+
                 self.logger.warning(f"Attempted to write read-only fields: {fields}. Removing them and retrying.")
+                # append read-only field to a list
+                if not self._target.read_only_fields.get(self.stream_name):
+                    self._target.read_only_fields[self.stream_name] = []
+                self._target.read_only_fields[self.stream_name].extend(fields)
+                # remove read-only fields from record
                 for f in fields:
                     record.pop(f, None)
                 # retry

target_salesforce_v3/target.py

@@ -34,6 +34,8 @@ class TargetSalesforceV3(TargetHotglue):
     name = "target-salesforce-v3"
     MAX_PARALLELISM = 10
     SINK_TYPES = SINK_TYPES
+    read_only_fields = {}
+
     def get_sink_class(self, stream_name: str):
         """Get sink for a stream."""
         for sink_class in SINK_TYPES: