Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix UAV migration bug: look up editableMappingsInfo versions in correct list #8378

Merged
merged 2 commits into from
Feb 7, 2025
Merged

Fix UAV migration bug: look up editableMappingsInfo versions in correct list #8378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
49b70b7
Fix UAV migration bug: look up editableMappingsInfo versions in corre…
fm3 Feb 5, 2025
23a4c0e
Merge branch 'master' into migration-right-editable-mapping-versions
fm3 Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions tools/migration-unified-annotation-versioning/add_editable_mapping_info.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import logging
from utils import setup_logging, log_since
import argparse
from connections import connect_to_fossildb, assert_grpc_success
import time
import fossildbapi_pb2 as proto


logger = logging.getLogger("migration-logs")


def main():
logger.info("Hello from add_editable_mapping_info")
setup_logging()
parser = argparse.ArgumentParser()
parser.add_argument("--fossil", type=str, help="Fossildb host and port. Example: localhost:7155", required=True)
parser.add_argument("--editable_mapping_info_file", type=str, help="path to binary input file to put", required=True)
parser.add_argument("--tracingId", type=str, help="tracingId to put the entry to", required=True)
parser.add_argument("--version", type=int, help="version number to put the entry to", required=True)
args = parser.parse_args()
before = time.time()
stub = connect_to_fossildb(args.fossil, "target")
with open(args.editable_mapping_info_file, 'rb') as infile:
bytes_to_put = infile.read()

logger.info(f"putting {len(bytes_to_put)} bytes of type {type(bytes_to_put)} at {args.tracingId} v{args.version}")
reply = stub.Put(proto.PutRequest(collection="editableMappingsInfo", key=args.tracingId, version=args.version, value=bytes_to_put))
assert_grpc_success(reply)
log_since(before, "Inserting one editable mapping info entry")
Comment on lines +12 to +29
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Add error handling for file operations and gRPC calls.

The script needs better error handling for:

  1. File operations when reading the binary input file
  2. gRPC connection failures
  3. Invalid input validation

Apply this diff to add error handling:

 def main():
     logger.info("Hello from add_editable_mapping_info")
     setup_logging()
     parser = argparse.ArgumentParser()
     parser.add_argument("--fossil", type=str, help="Fossildb host and port. Example: localhost:7155", required=True)
     parser.add_argument("--editable_mapping_info_file", type=str, help="path to binary input file to put", required=True)
     parser.add_argument("--tracingId", type=str, help="tracingId to put the entry to", required=True)
     parser.add_argument("--version", type=int, help="version number to put the entry to", required=True)
     args = parser.parse_args()
     before = time.time()
-    stub = connect_to_fossildb(args.fossil, "target")
-    with open(args.editable_mapping_info_file, 'rb') as infile:
-        bytes_to_put = infile.read()
+    try:
+        if args.version < 0:
+            raise ValueError("Version number must be non-negative")
+        
+        stub = connect_to_fossildb(args.fossil, "target")
+        
+        try:
+            with open(args.editable_mapping_info_file, 'rb') as infile:
+                bytes_to_put = infile.read()
+        except IOError as e:
+            logger.error(f"Failed to read input file: {e}")
+            raise
 
-    logger.info(f"putting {len(bytes_to_put)} bytes of type {type(bytes_to_put)} at {args.tracingId} v{args.version}")
-    reply = stub.Put(proto.PutRequest(collection="editableMappingsInfo", key=args.tracingId, version=args.version, value=bytes_to_put))
-    assert_grpc_success(reply)
-    log_since(before, "Inserting one editable mapping info entry")
+        if not bytes_to_put:
+            raise ValueError("Input file is empty")
+            
+        logger.info(f"putting {len(bytes_to_put)} bytes of type {type(bytes_to_put)} at {args.tracingId} v{args.version}")
+        try:
+            reply = stub.Put(proto.PutRequest(collection="editableMappingsInfo", key=args.tracingId, version=args.version, value=bytes_to_put))
+            assert_grpc_success(reply)
+            log_since(before, "Inserting one editable mapping info entry")
+        except Exception as e:
+            logger.error(f"Failed to put data to FossilDB: {e}")
+            raise
+    except Exception as e:
+        logger.error(f"Script failed: {e}")
+        sys.exit(1)
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
def main():
logger.info("Hello from add_editable_mapping_info")
setup_logging()
parser = argparse.ArgumentParser()
parser.add_argument("--fossil", type=str, help="Fossildb host and port. Example: localhost:7155", required=True)
parser.add_argument("--editable_mapping_info_file", type=str, help="path to binary input file to put", required=True)
parser.add_argument("--tracingId", type=str, help="tracingId to put the entry to", required=True)
parser.add_argument("--version", type=int, help="version number to put the entry to", required=True)
args = parser.parse_args()
before = time.time()
stub = connect_to_fossildb(args.fossil, "target")
with open(args.editable_mapping_info_file, 'rb') as infile:
bytes_to_put = infile.read()
logger.info(f"putting {len(bytes_to_put)} bytes of type {type(bytes_to_put)} at {args.tracingId} v{args.version}")
reply = stub.Put(proto.PutRequest(collection="editableMappingsInfo", key=args.tracingId, version=args.version, value=bytes_to_put))
assert_grpc_success(reply)
log_since(before, "Inserting one editable mapping info entry")
def main():
logger.info("Hello from add_editable_mapping_info")
setup_logging()
parser = argparse.ArgumentParser()
parser.add_argument("--fossil", type=str, help="Fossildb host and port. Example: localhost:7155", required=True)
parser.add_argument("--editable_mapping_info_file", type=str, help="path to binary input file to put", required=True)
parser.add_argument("--tracingId", type=str, help="tracingId to put the entry to", required=True)
parser.add_argument("--version", type=int, help="version number to put the entry to", required=True)
args = parser.parse_args()
before = time.time()
try:
if args.version < 0:
raise ValueError("Version number must be non-negative")
stub = connect_to_fossildb(args.fossil, "target")
try:
with open(args.editable_mapping_info_file, 'rb') as infile:
bytes_to_put = infile.read()
except IOError as e:
logger.error(f"Failed to read input file: {e}")
raise
if not bytes_to_put:
raise ValueError("Input file is empty")
logger.info(f"putting {len(bytes_to_put)} bytes of type {type(bytes_to_put)} at {args.tracingId} v{args.version}")
try:
reply = stub.Put(proto.PutRequest(collection="editableMappingsInfo", key=args.tracingId, version=args.version, value=bytes_to_put))
assert_grpc_success(reply)
log_since(before, "Inserting one editable mapping info entry")
except Exception as e:
logger.error(f"Failed to put data to FossilDB: {e}")
raise
except Exception as e:
logger.error(f"Script failed: {e}")
sys.exit(1)






if __name__ == '__main__':
main()
5 changes: 2 additions & 3 deletions tools/migration-unified-annotation-versioning/migration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -443,7 +443,6 @@ def migrate_segment_index(self, tracing_id, layer_version_mapping):
logger.info(f"Migrating segment index with large batch sizes for {tracing_id} failed with {e}, retrying with small batch sizes...")
self.migrate_all_versions_and_keys_with_prefix("volumeSegmentIndex", tracing_id, layer_version_mapping, transform_key=None, put_buffer_size=10, get_keys_page_size=1, get_keys_version_batch_size=1000)


def migrate_editable_mapping(self, tracing_id: str, layer_version_mapping: LayerVersionMapping, mapping_id_map: MappingIdMap) -> List[int]:
if tracing_id not in mapping_id_map:
return []
Expand All @@ -458,9 +457,9 @@ def migrate_editable_mapping_info(self, tracing_id: str, mapping_id: str, layer_
materialized_versions = self.list_versions(collection, mapping_id)
materialized_versions_unified = []
for materialized_version in materialized_versions:
value_bytes = self.get_bytes(collection, mapping_id, materialized_version)
if materialized_version not in layer_version_mapping[tracing_id]:
if materialized_version not in layer_version_mapping[mapping_id]:
continue
value_bytes = self.get_bytes(collection, mapping_id, materialized_version)
new_version = layer_version_mapping[mapping_id][materialized_version]
materialized_versions_unified.append(new_version)
self.save_bytes(collection, tracing_id, new_version, value_bytes)
Expand Down