From 9b16660508c3b1e9c70af46e38afbe51f8fcc6d1 Mon Sep 17 00:00:00 2001 From: Shaanjot Gill Date: Sat, 9 Sep 2023 01:50:18 +0530 Subject: [PATCH] parent 49e71c8aae319c7d805160696fecb1bcb85f139d author Shaanjot Gill 1694204418 +0530 committer Shaanjot Gill 1695747717 +0530 parent 49e71c8aae319c7d805160696fecb1bcb85f139d author Shaanjot Gill 1694204418 +0530 committer Shaanjot Gill 1695747613 +0530 parent 49e71c8aae319c7d805160696fecb1bcb85f139d author Shaanjot Gill 1694204418 +0530 committer Shaanjot Gill 1695747425 +0530 parent 49e71c8aae319c7d805160696fecb1bcb85f139d author Shaanjot Gill 1694204418 +0530 committer Shaanjot Gill 1695746776 +0530 tagged upgrade impl, fixed fix_ledger_entry function and added tests Signed-off-by: Shaanjot Gill --- UpgradingACA-Py.md | 19 ++ .../default_version_upgrade_config.yml | 4 +- .../commands/tests/test_upgrade.py | 42 ++- aries_cloudagent/commands/upgrade.py | 323 ++++++++++++------ aries_cloudagent/config/argparse.py | 11 + .../models/issuer_rev_reg_record.py | 89 +++-- .../tests/test_issuer_rev_reg_record.py | 191 ++++++++++- aries_cloudagent/revocation/recover.py | 8 +- 8 files changed, 543 insertions(+), 144 deletions(-) diff --git a/UpgradingACA-Py.md b/UpgradingACA-Py.md index 0ad8708564..a829bc4f7f 100644 --- a/UpgradingACA-Py.md +++ b/UpgradingACA-Py.md @@ -59,6 +59,25 @@ is high (such as a "resave connections" upgrade to a deployment with many, many connections), you may want to do a test upgrade offline first, to see if there is likely to be a service disruption during the upgrade. Plan accordingly! +## Tagged upgrades +Upgrades are defined in the [Upgrade Definition YML file], in addition to specifying upgrade actions by version they can also be specified by named tags. Unlike version based upgrades where all applicable version based actions will be performed based upon sorted order of versions, with named tags only actions corresponding to provided tags will be performed. Note: `--force-upgrade` is required when running name tags based upgrade [i.e. provding `--named-tag`] + +Tags are specfied in YML file as below: +``` +fix_issue_rev_reg: + fix_issue_rev_reg_records: true +``` + +Example +``` + ./scripts/run_docker upgrade --force-upgrade --named-tag fix_issue_rev_reg + +In case, running multiple tags [say test1 & test2]: + ./scripts/run_docker upgrade --force-upgrade --named-tag test1 --named-tag test2 +``` + + + ## Exceptions There are a couple of upgrade exception conditions to consider, as outlined diff --git a/aries_cloudagent/commands/default_version_upgrade_config.yml b/aries_cloudagent/commands/default_version_upgrade_config.yml index 8c04b9e48f..25cc32538b 100644 --- a/aries_cloudagent/commands/default_version_upgrade_config.yml +++ b/aries_cloudagent/commands/default_version_upgrade_config.yml @@ -13,4 +13,6 @@ v0.7.1: v0.7.0: update_existing_records: false v0.6.0: - update_existing_records: false \ No newline at end of file + update_existing_records: false +fix_issue_rev_reg: + fix_issue_rev_reg_records: true diff --git a/aries_cloudagent/commands/tests/test_upgrade.py b/aries_cloudagent/commands/tests/test_upgrade.py index 3cbdda9831..d6c5c66e08 100644 --- a/aries_cloudagent/commands/tests/test_upgrade.py +++ b/aries_cloudagent/commands/tests/test_upgrade.py @@ -121,6 +121,44 @@ async def test_upgrade_callable(self): } ) + async def test_upgrade_callable_named_tag(self): + version_storage_record = await self.storage.find_record( + type_filter="acapy_version", tag_query={} + ) + await self.storage.delete_record(version_storage_record) + with async_mock.patch.object( + test_module, + "wallet_config", + async_mock.CoroutineMock( + return_value=( + self.profile, + async_mock.CoroutineMock(did="public DID", verkey="verkey"), + ) + ), + ), async_mock.patch.object( + test_module.yaml, + "safe_load", + async_mock.MagicMock( + return_value={ + "v0.7.2": { + "resave_records": { + "base_record_path": [ + "aries_cloudagent.connections.models.conn_record.ConnRecord" + ] + }, + "update_existing_records": True, + }, + "fix_issue_rev_reg": {"fix_issue_rev_reg_records": True}, + } + ), + ): + await test_module.upgrade( + settings={ + "upgrade.named_tags": ["fix_issue_rev_reg"], + "upgrade.force_upgrade": True, + } + ) + async def test_upgrade_x_same_version(self): version_storage_record = await self.storage.find_record( type_filter="acapy_version", tag_query={} @@ -169,7 +207,9 @@ async def test_upgrade_missing_from_version(self): "upgrade.config_path": "./aries_cloudagent/commands/default_version_upgrade_config.yml", } ) - assert "No upgrade from version found in wallet or" in str(ctx.exception) + assert "Error during upgrade: No upgrade from version or tags found" in str( + ctx.exception + ) async def test_upgrade_x_callable_not_set(self): version_storage_record = await self.storage.find_record( diff --git a/aries_cloudagent/commands/upgrade.py b/aries_cloudagent/commands/upgrade.py index 8706c61bcf..bf2d0dc95a 100644 --- a/aries_cloudagent/commands/upgrade.py +++ b/aries_cloudagent/commands/upgrade.py @@ -3,6 +3,7 @@ import asyncio import logging import os +import json import yaml from configargparse import ArgumentParser @@ -19,16 +20,18 @@ Tuple, ) -from ..core.profile import Profile +from ..core.profile import Profile, ProfileSession from ..config import argparse as arg from ..config.default_context import DefaultContextBuilder from ..config.base import BaseError, BaseSettings from ..config.util import common_config from ..config.wallet import wallet_config -from ..messaging.models.base_record import BaseRecord +from ..messaging.models.base import BaseModelError +from ..messaging.models.base_record import BaseRecord, RecordType from ..storage.base import BaseStorage from ..storage.error import StorageNotFoundError from ..storage.record import StorageRecord +from ..revocation.models.issuer_rev_reg_record import IssuerRevRegRecord from ..utils.classloader import ClassLoader, ClassNotFoundError from ..version import __version__, RECORD_TYPE_ACAPY_VERSION @@ -78,10 +81,10 @@ def setup_version_upgrade_config(self, path: str): """Set ups config dict from the provided YML file.""" with open(path, "r") as stream: config_dict = yaml.safe_load(stream) - version_config_dict = {} - for version, provided_config in config_dict.items(): + tagged_config_dict = {} + for config_id, provided_config in config_dict.items(): recs_list = [] - version_config_dict[version] = {} + tagged_config_dict[config_id] = {} if "resave_records" in provided_config: if provided_config.get("resave_records").get("base_record_path"): recs_list = recs_list + provided_config.get( @@ -93,14 +96,14 @@ def setup_version_upgrade_config(self, path: str): recs_list = recs_list + provided_config.get( "resave_records" ).get("base_exch_record_path") - version_config_dict[version]["resave_records"] = recs_list + tagged_config_dict[config_id]["resave_records"] = recs_list config_key_set = set(provided_config.keys()) try: config_key_set.remove("resave_records") except KeyError: pass if "explicit_upgrade" in provided_config: - version_config_dict[version][ + tagged_config_dict[config_id][ "explicit_upgrade" ] = provided_config.get("explicit_upgrade") try: @@ -108,12 +111,12 @@ def setup_version_upgrade_config(self, path: str): except KeyError: pass for executable in config_key_set: - version_config_dict[version][executable] = ( + tagged_config_dict[config_id][executable] = ( provided_config.get(executable) or False ) - if version_config_dict == {}: + if tagged_config_dict == {}: raise UpgradeError(f"No version configs found in {path}") - self.upgrade_configs = version_config_dict + self.upgrade_configs = tagged_config_dict def get_callable(self, executable: str) -> Optional[Callable]: """Return callable function for executable name.""" @@ -159,9 +162,12 @@ def get_upgrade_version_list( if not sorted_version_list: version_upgrade_config_inst = VersionUpgradeConfig(config_path) upgrade_configs = version_upgrade_config_inst.upgrade_configs - versions_found_in_config = upgrade_configs.keys() + tags_found_in_config = upgrade_configs.keys() + version_found_in_config, _ = _get_version_and_name_tags( + list(tags_found_in_config) + ) sorted_version_list = sorted( - versions_found_in_config, key=lambda x: package_version.parse(x) + version_found_in_config, key=lambda x: package_version.parse(x) ) version_list = [] @@ -193,6 +199,46 @@ async def add_version_record(profile: Profile, version: str): LOGGER.info(f"{RECORD_TYPE_ACAPY_VERSION} storage record set to {version}") +def _get_version_and_name_tags(tags_found_in_config: List) -> Tuple[List, List]: + """Get version and named tag key lists from config.""" + version_found_in_config = [] + named_tag_found_in_config = [] + for tag in tags_found_in_config: + try: + package_version.parse(tag) + version_found_in_config.append(tag) + except package_version.InvalidVersion: + named_tag_found_in_config.append(tag) + return version_found_in_config, named_tag_found_in_config + + +def _perform_upgrade( + upgrade_config: dict, + resave_record_path_sets: set, + executables_call_set: set, + tag: str, +) -> Tuple[set, set]: + """Update and return resave record path and executables call sets.""" + LOGGER.info(f"Running upgrade process for {tag}") + # Step 1 re-saving all BaseRecord and BaseExchangeRecord + if "resave_records" in upgrade_config: + resave_record_paths = upgrade_config.get("resave_records") + for record_path in resave_record_paths: + resave_record_path_sets.add(record_path) + + # Step 2 Update existing records, if required + config_key_set = set(upgrade_config.keys()) + try: + config_key_set.remove("resave_records") + except KeyError: + pass + for callable_name in list(config_key_set): + if upgrade_config.get(callable_name) is False: + continue + executables_call_set.add(callable_name) + return resave_record_path_sets, executables_call_set + + async def upgrade( settings: Optional[Union[Mapping[str, Any], BaseSettings]] = None, profile: Optional[Profile] = None, @@ -211,11 +257,18 @@ async def upgrade( version_upgrade_config_inst = VersionUpgradeConfig( settings.get("upgrade.config_path") ) + upgrade_from_tags = None + force_upgrade_flag = settings.get("upgrade.force_upgrade") or False + if force_upgrade_flag: + upgrade_from_tags = settings.get("upgrade.named_tags") upgrade_configs = version_upgrade_config_inst.upgrade_configs upgrade_to_version = f"v{__version__}" - versions_found_in_config = upgrade_configs.keys() + tags_found_in_config = upgrade_configs.keys() + version_found_in_config, named_tag_found_in_config = _get_version_and_name_tags( + list(tags_found_in_config) + ) sorted_versions_found_in_config = sorted( - versions_found_in_config, key=lambda x: package_version.parse(x) + version_found_in_config, key=lambda x: package_version.parse(x) ) upgrade_from_version_storage = None upgrade_from_version_config = None @@ -240,7 +293,6 @@ async def upgrade( ) ) - force_upgrade_flag = settings.get("upgrade.force_upgrade") or False if upgrade_from_version_storage and upgrade_from_version_config: if ( package_version.parse(upgrade_from_version_storage) @@ -261,103 +313,100 @@ async def upgrade( and not upgrade_from_version_config ): upgrade_from_version = upgrade_from_version_storage - if not upgrade_from_version: + if not upgrade_from_version and not upgrade_from_tags: raise UpgradeError( - "No upgrade from version found in wallet or settings [--from-version]" + "No upgrade from version or tags found in wallet" + " or settings [--from-version or --named-tag]" ) - upgrade_version_in_config = get_upgrade_version_list( - sorted_version_list=sorted_versions_found_in_config, - from_version=upgrade_from_version, - ) - # Perform explicit upgrade check if the function was called during startup - if profile: - ( - explicit_flag, - to_skip_explicit_versions, - explicit_upg_ver, - ) = explicit_upgrade_required_check( - to_apply_version_list=upgrade_version_in_config, - upgrade_config=upgrade_configs, - ) - if explicit_flag: - raise UpgradeError( - "Explicit upgrade flag with critical value found " - f"for {explicit_upg_ver} config. Please use ACA-Py " - "upgrade command to complete the process and proceed." - ) - if len(to_skip_explicit_versions) >= 1: - LOGGER.warning( - "Explicit upgrade flag with warning value found " - f"for {str(to_skip_explicit_versions)} versions. " - "Proceeding with ACA-Py startup. You can apply " - "the explicit upgrades using the ACA-Py upgrade " - "command later." - ) - return + resave_record_path_sets = set() + executables_call_set = set() to_update_flag = False - if upgrade_from_version == upgrade_to_version: - LOGGER.info( + if upgrade_from_version: + upgrade_version_in_config = get_upgrade_version_list( + sorted_version_list=sorted_versions_found_in_config, + from_version=upgrade_from_version, + ) + # Perform explicit upgrade check if the function was called during startup + if profile: ( - f"Version {upgrade_from_version} to upgrade from and " - f"current version to upgrade to {upgrade_to_version} " - "are same. You can apply upgrade from a lower " - "version by running the upgrade command with " - f"--from-version [< {upgrade_to_version}] and " - "--force-upgrade" + explicit_flag, + to_skip_explicit_versions, + explicit_upg_ver, + ) = explicit_upgrade_required_check( + to_apply_version_list=upgrade_version_in_config, + upgrade_config=upgrade_configs, ) - ) - else: - resave_record_path_sets = set() - executables_call_set = set() - for config_from_version in upgrade_version_in_config: - LOGGER.info(f"Running upgrade process for {config_from_version}") - upgrade_config = upgrade_configs.get(config_from_version) - # Step 1 re-saving all BaseRecord and BaseExchangeRecord - if "resave_records" in upgrade_config: - resave_record_paths = upgrade_config.get("resave_records") - for record_path in resave_record_paths: - resave_record_path_sets.add(record_path) - - # Step 2 Update existing records, if required - config_key_set = set(upgrade_config.keys()) - try: - config_key_set.remove("resave_records") - except KeyError: - pass - for callable_name in list(config_key_set): - if upgrade_config.get(callable_name) is False: - continue - executables_call_set.add(callable_name) - - if len(resave_record_path_sets) >= 1 or len(executables_call_set) >= 1: - to_update_flag = True - for record_path in resave_record_path_sets: - try: - rec_type = ClassLoader.load_class(record_path) - except ClassNotFoundError as err: - raise UpgradeError(f"Unknown Record type {record_path}") from err - if not issubclass(rec_type, BaseRecord): + if explicit_flag: raise UpgradeError( - f"Only BaseRecord can be resaved, found: {str(rec_type)}" + "Explicit upgrade flag with critical value found " + f"for {explicit_upg_ver} config. Please use ACA-Py " + "upgrade command to complete the process and proceed." ) - async with root_profile.session() as session: - all_records = await rec_type.query(session) - for record in all_records: - await record.save( - session, - reason="re-saving record during the upgrade process", - ) - if len(all_records) == 0: - LOGGER.info(f"No records of {str(rec_type)} found") - else: - LOGGER.info( - f"All recs of {str(rec_type)} successfully re-saved" - ) - for callable_name in executables_call_set: - _callable = version_upgrade_config_inst.get_callable(callable_name) - if not _callable: - raise UpgradeError(f"No function specified for {callable_name}") - await _callable(root_profile) + if len(to_skip_explicit_versions) >= 1: + LOGGER.warning( + "Explicit upgrade flag with warning value found " + f"for {str(to_skip_explicit_versions)} versions. " + "Proceeding with ACA-Py startup. You can apply " + "the explicit upgrades using the ACA-Py upgrade " + "command later." + ) + return + if upgrade_from_version == upgrade_to_version: + LOGGER.info( + ( + f"Version {upgrade_from_version} to upgrade from and " + f"current version to upgrade to {upgrade_to_version} " + "are same. You can apply upgrade from a lower " + "version by running the upgrade command with " + f"--from-version [< {upgrade_to_version}] and " + "--force-upgrade" + ) + ) + else: + for config_from_version in upgrade_version_in_config: + resave_record_path_sets, executables_call_set = _perform_upgrade( + upgrade_config=upgrade_configs.get(config_from_version), + resave_record_path_sets=resave_record_path_sets, + executables_call_set=executables_call_set, + tag=config_from_version, + ) + if upgrade_from_tags and len(upgrade_from_tags) >= 1: + for named_tag in upgrade_from_tags: + if named_tag not in named_tag_found_in_config: + continue + resave_record_path_sets, executables_call_set = _perform_upgrade( + upgrade_config=upgrade_configs.get(named_tag), + resave_record_path_sets=resave_record_path_sets, + executables_call_set=executables_call_set, + tag=named_tag, + ) + if len(resave_record_path_sets) >= 1 or len(executables_call_set) >= 1: + to_update_flag = True + for record_path in resave_record_path_sets: + try: + rec_type = ClassLoader.load_class(record_path) + except ClassNotFoundError as err: + raise UpgradeError(f"Unknown Record type {record_path}") from err + if not issubclass(rec_type, BaseRecord): + raise UpgradeError( + f"Only BaseRecord can be resaved, found: {str(rec_type)}" + ) + async with root_profile.session() as session: + all_records = await rec_type.query(session) + for record in all_records: + await record.save( + session, + reason="re-saving record during the upgrade process", + ) + if len(all_records) == 0: + LOGGER.info(f"No records of {str(rec_type)} found") + else: + LOGGER.info(f"All recs of {str(rec_type)} successfully re-saved") + for callable_name in executables_call_set: + _callable = version_upgrade_config_inst.get_callable(callable_name) + if not _callable: + raise UpgradeError(f"No function specified for {callable_name}") + await _callable(root_profile) # Update storage version if to_update_flag: @@ -394,6 +443,67 @@ async def update_existing_records(profile: Profile): pass +########################################################## +# Fix for ACA-Py Issue #2485 +# issuance_type attribue in IssuerRevRegRecord was removed +# in 0.5.3 version. IssuerRevRegRecord created previously +# will need +########################################################## + + +async def find_affected_issue_rev_reg_records( + session: ProfileSession, +) -> Sequence[RecordType]: + """Get IssuerRevRegRecord records with issuance_type for re-saving. + + Args: + session: The profile session to use + """ + storage = session.inject(BaseStorage) + rows = await storage.find_all_records( + IssuerRevRegRecord.RECORD_TYPE, + ) + issue_rev_reg_records_to_update = [] + for record in rows: + vals = json.loads(record.value) + to_update = False + try: + record_id = record.id + record_id_name = IssuerRevRegRecord.RECORD_ID_NAME + if record_id_name in vals: + raise ValueError(f"Duplicate {record_id_name} inputs; {vals}") + params = dict(**vals) + # Check for issuance_type and add record_id for later tracking + if "issuance_type" in params: + LOGGER.info( + f"IssuerRevRegRecord {record_id} tagged for fixing issuance_type." + ) + del params["issuance_type"] + to_update = True + params[record_id_name] = record_id + if to_update: + issue_rev_reg_records_to_update.append(IssuerRevRegRecord(**params)) + except BaseModelError as err: + raise BaseModelError(f"{err}, for record id {record.id}") + return issue_rev_reg_records_to_update + + +async def fix_issue_rev_reg_records(profile: Profile): + """Update IssuerRevRegRecord records. + + Args: + profile: Root profile + + """ + async with profile.session() as session: + issue_rev_reg_records = await find_affected_issue_rev_reg_records(session) + for record in issue_rev_reg_records: + await record.save( + session, + reason="re-saving issue_rev_reg record without issuance type", + ) + + def execute(argv: Sequence[str] = None): """Entrypoint.""" parser = arg.create_argument_parser(prog=PROG) @@ -413,7 +523,8 @@ def main(): UPGRADE_EXISTING_RECORDS_FUNCTION_MAPPING = { - "update_existing_records": update_existing_records + "update_existing_records": update_existing_records, + "fix_issue_rev_reg_records": fix_issue_rev_reg_records, } main() diff --git a/aries_cloudagent/config/argparse.py b/aries_cloudagent/config/argparse.py index ac38895254..5167831c32 100644 --- a/aries_cloudagent/config/argparse.py +++ b/aries_cloudagent/config/argparse.py @@ -2139,6 +2139,13 @@ def add_arguments(self, parser: ArgumentParser): ), ) + parser.add_argument( + "--named-tag", + action="append", + env_var="ACAPY_UPGRADE_NAMED_TAGS", + help=("Runs upgrade steps associated with tags provided in the config"), + ) + def get_settings(self, args: Namespace) -> dict: """Extract ACA-Py upgrade process settings.""" settings = {} @@ -2148,4 +2155,8 @@ def get_settings(self, args: Namespace) -> dict: settings["upgrade.from_version"] = args.from_version if args.force_upgrade: settings["upgrade.force_upgrade"] = args.force_upgrade + if args.named_tag: + settings["upgrade.named_tags"] = ( + list(args.named_tag) if args.named_tag else [] + ) return settings diff --git a/aries_cloudagent/revocation/models/issuer_rev_reg_record.py b/aries_cloudagent/revocation/models/issuer_rev_reg_record.py index ccb47e63ed..a027010937 100644 --- a/aries_cloudagent/revocation/models/issuer_rev_reg_record.py +++ b/aries_cloudagent/revocation/models/issuer_rev_reg_record.py @@ -1,6 +1,7 @@ """Issuer revocation registry storage handling.""" import json +import importlib import logging import uuid from functools import total_ordering @@ -13,6 +14,10 @@ from marshmallow import fields, validate from ...core.profile import Profile, ProfileSession +from ...indy.credx.issuer import ( + CATEGORY_CRED_DEF, + CATEGORY_REV_REG_DEF_PRIVATE, +) from ...indy.issuer import IndyIssuer, IndyIssuerError from ...indy.models.revocation import ( IndyRevRegDef, @@ -379,41 +384,59 @@ async def fix_ledger_entry( session, rev_reg_id=self.revoc_reg_id ) - revoked_ids = [] - for rec in recs: - if rec.state == IssuerCredRevRecord.STATE_REVOKED: - revoked_ids.append(int(rec.cred_rev_id)) - if int(rec.cred_rev_id) not in rev_reg_delta["value"]["revoked"]: - # await rec.set_state(session, IssuerCredRevRecord.STATE_ISSUED) - rec_count += 1 - - LOGGER.debug(">>> fixed entry recs count = %s", rec_count) - LOGGER.debug( - ">>> rev_reg_record.revoc_reg_entry.value: %s", - self.revoc_reg_entry.value, + revoked_ids = [] + for rec in recs: + if rec.state == IssuerCredRevRecord.STATE_REVOKED: + revoked_ids.append(int(rec.cred_rev_id)) + if int(rec.cred_rev_id) not in rev_reg_delta["value"]["revoked"]: + # await rec.set_state(session, IssuerCredRevRecord.STATE_ISSUED) + rec_count += 1 + + LOGGER.debug(">>> fixed entry recs count = %s", rec_count) + LOGGER.debug( + ">>> rev_reg_record.revoc_reg_entry.value: %s", + self.revoc_reg_entry.value, + ) + LOGGER.debug('>>> rev_reg_delta.get("value"): %s', rev_reg_delta.get("value")) + + # if we had any revocation discrepencies, check the accumulator value + if rec_count > 0: + if (self.revoc_reg_entry.value and rev_reg_delta.get("value")) and not ( + self.revoc_reg_entry.value.accum == rev_reg_delta["value"]["accum"] + ): + # self.revoc_reg_entry = rev_reg_delta["value"] + # await self.save(session) + accum_count += 1 + async with profile.session() as session: + issuer_rev_reg_record = ( + await IssuerRevRegRecord.retrieve_by_revoc_reg_id( + session, self.revoc_reg_id + ) + ) + cred_def_id = issuer_rev_reg_record.cred_def_id + _cred_def = await session.handle.fetch(CATEGORY_CRED_DEF, cred_def_id) + _rev_reg_def_private = await session.handle.fetch( + CATEGORY_REV_REG_DEF_PRIVATE, self.revoc_reg_id + ) + credx_module = importlib.import_module("indy_credx") + cred_defn = credx_module.CredentialDefinition.load(_cred_def.value_json) + rev_reg_defn_private = ( + credx_module.RevocationRegistryDefinitionPrivate.load( + _rev_reg_def_private.value_json + ) ) - LOGGER.debug( - '>>> rev_reg_delta.get("value"): %s', rev_reg_delta.get("value") + calculated_txn = await generate_ledger_rrrecovery_txn( + genesis_transactions, + self.revoc_reg_id, + revoked_ids, + cred_defn, + rev_reg_defn_private, ) + recovery_txn = json.loads(calculated_txn.to_json()) - # if we had any revocation discrepencies, check the accumulator value - if rec_count > 0: - if (self.revoc_reg_entry.value and rev_reg_delta.get("value")) and not ( - self.revoc_reg_entry.value.accum == rev_reg_delta["value"]["accum"] - ): - # self.revoc_reg_entry = rev_reg_delta["value"] - # await self.save(session) - accum_count += 1 - - calculated_txn = await generate_ledger_rrrecovery_txn( - genesis_transactions, - self.revoc_reg_id, - revoked_ids, - ) - recovery_txn = json.loads(calculated_txn.to_json()) - - LOGGER.debug(">>> apply_ledger_update = %s", apply_ledger_update) - if apply_ledger_update: + LOGGER.debug(">>> apply_ledger_update = %s", apply_ledger_update) + if apply_ledger_update: + async with profile.session() as session: ledger = session.inject_or(BaseLedger) if not ledger: reason = "No ledger available" @@ -426,7 +449,7 @@ async def fix_ledger_entry( self.revoc_reg_id, "CL_ACCUM", recovery_txn ) - applied_txn = ledger_response["result"] + applied_txn = ledger_response["result"] return (rev_reg_delta, recovery_txn, applied_txn) diff --git a/aries_cloudagent/revocation/models/tests/test_issuer_rev_reg_record.py b/aries_cloudagent/revocation/models/tests/test_issuer_rev_reg_record.py index 80d15592ea..bc292daeb4 100644 --- a/aries_cloudagent/revocation/models/tests/test_issuer_rev_reg_record.py +++ b/aries_cloudagent/revocation/models/tests/test_issuer_rev_reg_record.py @@ -1,10 +1,14 @@ +import importlib import json from os.path import join +from typing import Any, Mapping, Type from asynctest import TestCase as AsyncTestCase, mock as async_mock -from ....core.in_memory import InMemoryProfile +from ....core.in_memory import InMemoryProfile, InMemoryProfileSession +from ....core.profile import Profile, ProfileSession +from ....config.injection_context import InjectionContext from ....indy.issuer import IndyIssuer, IndyIssuerError from ....indy.util import indy_client_dir from ....ledger.base import BaseLedger @@ -78,6 +82,191 @@ async def test_order(self): await rec1.save(session, reason="another record") assert rec0 < rec1 + async def test_fix_ledger_entry(self): + mock_cred_def = { + "ver": "1.0", + "id": "55GkHamhTU1ZbTbV2ab9DE:3:CL:15:tag", + "schemaId": "15", + "type": "CL", + "tag": "tag", + "value": { + "primary": { + "n": "97755077532337293488745809101840506138839796255951676757957095601046950252131007247574478925999370347259264200571203215922043340992124710162716447600735862203372184211636922189157817784130889637892277882125570534881418101997106854432924131454003284028861730869486619833238782461572414320131411215654018054873708704239239964147885524705957168226001184349267512368273047027640698617139590254809517853853130886625330234652284599369944951191468504513070325577275942459149020744004320427886430812173881305323670033871442431456536710164496981741573947141839358296893048748871620778358547174947418982842584086910766377426381", + "s": "93857461424756524831309204327541325286955427618197438652468158979904850495412765053870776608842922458588066684799662534588058064651312983375745531372709501399468055860133002734655992193122962210761579825894255323571233899698407599997585154150003119067126768230932260880730480485869205348502571475803409653237711195597513501909794351191631697504002513031303493339100149587811674358458553310268294288630892274988762794340699542957746255964836213015732087927295229891638172796974267746722775815971385993320758281434138548295910631509522670548769524207136946079991721576359187532370389806940124963337347442220291195381379", + "r": { + "attr": "47851812221002419738510348434589719299005636707459946297826216114810122000929029718256700990932773158900801032759636000731156275563407397654401429007893443548977293352519606768993036568324588287173469229957113813985349265315838049185336810444812307980611094706266979152047930521019494220669875858786312470775883694491178666122652050631237335108908878703921025524371908258656719798575775820756662770857250313841640739199537587617643616110506607528089021428997787577846819067855712454919148492951257992078447468260961148762693304228013316282905649275245604958548728173709772269908692834231109401462806738322298320014588", + "master_secret": "1456034034452917866826058280183727425788793902981406386175016190082488912409448091384258390876182506491481376203626852147024966574074505159095019883608712317385148607657338114099655296899988391276120507283428334300992065352980582993987058928311851237491145495605817715962285966152190021168736714801270287839185496030660074232737687865821410617700173791873244132463428232777257496952831001540198725563550215668732380420651897876237788070219793125826917973396212392661183288071986411608233468079014270033759863563636650489933951357555474357165922955337708555697826065745245669404874498188819137127409608428703992429087", + }, + "rctxt": "3181166664902801317779556377385630436151550717862204865421515198259158965590304479081007449054293128232193810709014084698474265818919401580417293157753663769438333622403413264724381527519123803324371803790394771682351074853790156764071298806108016312946683322202825645967662223488370365263607749486727675784672879635222504232881959377264213229748333060407839919218390751977625946072140500297691925789411870206929445018192788803161174534714033652405735420578422669164795825360682590769380466620583112381320768898271838621002245390378640966896034356391997262998321678222800285339726066703530463747958328257763842981029", + "z": "88882571993858813307170085778645601203164536169839951148860666045278872319305671063809074514017614163768224290285339562133145800179699201796197881847784025241295744277627025401945924707873790519590490294771194748427422972923806879605569348426345486261878031328212177481424474160598821738627280954513834712377303213119700398636139443890766009925681362795514758039170870017744796137650484010207378636277505131056606153648732287440383301414486959605962454376715897921966027018433464774363787319454306102402364604589381426633687528444586300536019182461458586285601075166524298872635741326726895166696312685388693673327358", + }, + "revocation": { + "g": "1 1ACDF1F880164FAE240C033DD2CC7E80130264E1A3BC81A3B2BD28E65827E1DA 1 0A4DD8DBB73B2A26CD2A576ECC0C5AB609346EDC923FFF20685EBE3B618A841A 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8", + "g_dash": "1 206CF4DECDE37A09315EE453D08E64CE6EFF4B89D6FCADFB456396AD1C57C442 1 14E36B42881BD9CA04608002810B4C9923BCC52D379A8F216FA57C6392A3BE83 1 138F620C8EEBFEAF9D1EF90152A11B9C0BDF3AED384324767E2293A4FC12F784 1 073227CD0CC0A6D5101A3CE03BDF6D29E6BACB33A090A4AE95F688F45BD54316 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8 1 0000000000000000000000000000000000000000000000000000000000000000", + "h": "1 0B608579F0A9A830862E45AC4E97F5CB30FD118379A1C408363D92970DBE989C 1 1671922BCECA4E69B693851EA3C78CB3B2714CACAAB64DD5B5CF3C5A187A572A 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8", + "h0": "1 1BCE3325E76EF9FA7566732BD17B5B82B1D0E9559B393D368A0BD09761AE2A39 1 10A1DA70F60369FBD69742B1564E98C85188312CFCCDD8AD5D11978F3FA1376D 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8", + "h1": "1 145B2DB05BFAAB9AF90401897E318B73BCEE7178D2857A551561DE0893098137 1 17A23E4161EF4392A56F13B2B0F40E0FA1DD28CDBD3FF33FDD67C5D998A10B6D 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8", + "h2": "1 14E18A1496B043246A4A860159EFCF49621606AF3E46A9854B8949E9F8C0FD97 1 1EFE5D486448A8F7A44BE358B674C0CF6FF09AAFDDDA416462943311CCA1A291 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8", + "htilde": "1 152FBD85CD4C81796599582D2CFE37407760B348A7816ADE2C9B76077C62440D 1 23DDDD15C99111413585FD282E05095FC27803EA8DCF94570B3FD2A53658AE00 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8", + "h_cap": "1 06333FED8C082FE894D02CC04A53B78D801ABC9CE09D0C5B731FEDF82A1BBB5F 1 06FD92996796D6B81F7803C3CEA3891741552B452E3F6C735F5BFE096234442A 1 23C5C9462697381BDF16658D133E5B0888C909E0EA01F041FEDC8E3A4F84BBF6 1 0511A0E8510AAF8454EB78446C563A8719622DADFF4AE86DF07E2969D2E4E48F 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8 1 0000000000000000000000000000000000000000000000000000000000000000", + "u": "1 1566C7AC1F99896C4E0CD7BD1B77C241614CE03337201FC6BB886650D78AC20D 1 1C73D8607361E539355F39297C3D8073B6CCA2DBF8A7AC50A50BF2FF20CAD7E9 1 1BF4F8AAE80EFAE308FF9ED399F24546306623EE8BECF6A2B39D939B1FCE94DA 1 0646620C014AD0A78BF83748F165211EFF811B526859F71AD184DA462A0C9303 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8 1 0000000000000000000000000000000000000000000000000000000000000000", + "pk": "1 16373692E1B40E44D2010B1CC7023E89E96DD9F45DB3F837D532A1299E80DB23 1 0A0DA35D2830848D4C04FADBB3E726525ADE9DC5F3364AB6286F642E904BB913 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8", + "y": "1 1C132346B6D6999F872132568ACB6F722FF4F93E08F0F6A1433E25F0937FBCB8 1 093034020F4F2087FDAA63D2EBD9BC02A7FF88F914F5971F285CD21BC200F9BD 1 04FB2599E6FA39E12F7F4A9B1C152E753E6584755843D7B6F746904A78396650 1 2291AC4F56DC5E0FEA5C580ACC2521714A3A8F3C79D78D475AEC473FB631E131 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8 1 0000000000000000000000000000000000000000000000000000000000000000", + }, + }, + } + mock_reg_rev_def_private = { + "value": { + "gamma": "11D1EF7D29B67A898E8BFDECEF24AD3B23A4984636F31AA84E1958A724A3579A" + } + } + + class TestProfile(InMemoryProfile): + def session(self, context: InjectionContext = None) -> "ProfileSession": + return TestProfileSession(self, context=context) + + @classmethod + def test_profile( + cls, settings: Mapping[str, Any] = None, bind: Mapping[Type, Any] = None + ) -> "TestProfile": + profile = TestProfile( + context=InjectionContext(enforce_typing=False, settings=settings), + name=InMemoryProfile.TEST_PROFILE_NAME, + ) + if bind: + for k, v in bind.items(): + if v: + profile.context.injector.bind_instance(k, v) + else: + profile.context.injector.clear_binding(k) + return profile + + @classmethod + def test_session( + cls, settings: Mapping[str, Any] = None, bind: Mapping[Type, Any] = None + ) -> "TestProfileSession": + session = TestProfileSession(cls.test_profile(), settings=settings) + session._active = True + session._init_context() + if bind: + for k, v in bind.items(): + if v: + session.context.injector.bind_instance(k, v) + else: + session.context.injector.clear_binding(k) + return session + + class TestProfileSession(InMemoryProfileSession): + def __init__( + self, + profile: Profile, + *, + context: InjectionContext = None, + settings: Mapping[str, Any] = None, + ): + super().__init__(profile=profile, context=context, settings=settings) + self.handle_counter = 0 + + @property + def handle(self): + if self.handle_counter == 0: + self.handle_counter = self.handle_counter + 1 + return async_mock.MagicMock( + fetch=async_mock.CoroutineMock( + return_value=async_mock.MagicMock( + value_json=json.dumps(mock_cred_def) + ) + ) + ) + else: + return async_mock.MagicMock( + fetch=async_mock.CoroutineMock( + return_value=async_mock.MagicMock( + value_json=json.dumps(mock_reg_rev_def_private), + ), + ) + ) + + credx_module = importlib.import_module("indy_credx") + rev_reg_delta = credx_module.RevocationRegistryDelta.load( + json.dumps( + { + "ver": "1.0", + "value": { + "accum": "1 0792BD1C8C1A529173FDF54A5B30AC90C2472956622E9F04971D36A9BF77C2C5 1 13B18B6B68AD62605C74FD61088814338EDEEB41C2195F96EC0E83B2B3D0258F 1 102ED0DDE96F6367199CE1C0B138F172BC913B65E37250581606974034F4CA20 1 1C53786D2C15190B57167CDDD2A046CAD63970B5DE43F4D492D4F46B8EEE6FF1 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8 1 0000000000000000000000000000000000000000000000000000000000000000" + }, + } + ) + ) + + TEST_GENESIS_TXN = "test_genesis_txn" + rec = IssuerRevRegRecord( + issuer_did=TEST_DID, + revoc_reg_id=REV_REG_ID, + revoc_reg_def=REV_REG_DEF, + revoc_def_type="CL_ACCUM", + revoc_reg_entry=REV_REG_ENTRY, + cred_def_id=CRED_DEF_ID, + tails_public_uri=TAILS_URI, + ) + _test_rev_reg_delta = { + "ver": "1.0", + "value": {"accum": "ACCUM", "issued": [1, 2], "revoked": [3, 4]}, + } + self.ledger.get_revoc_reg_delta = async_mock.CoroutineMock( + return_value=( + _test_rev_reg_delta, + 1234567890, + ) + ) + self.ledger.send_revoc_reg_entry = async_mock.CoroutineMock( + return_value={ + "result": {"...": "..."}, + }, + ) + _test_session = TestProfile.test_session( + settings={"tails_server_base_url": "http://1.2.3.4:8088"}, + ) + _test_profile = _test_session.profile + _test_profile.context.injector.bind_instance(BaseLedger, self.ledger) + with async_mock.patch.object( + test_module.IssuerCredRevRecord, + "query_by_ids", + async_mock.CoroutineMock( + return_value=[ + test_module.IssuerCredRevRecord( + record_id=test_module.UUID4_EXAMPLE, + state=test_module.IssuerCredRevRecord.STATE_REVOKED, + cred_ex_id=test_module.UUID4_EXAMPLE, + rev_reg_id=REV_REG_ID, + cred_rev_id="1", + ) + ] + ), + ), async_mock.patch.object( + test_module.IssuerRevRegRecord, + "retrieve_by_revoc_reg_id", + async_mock.CoroutineMock(return_value=rec), + ), async_mock.patch.object( + test_module, + "generate_ledger_rrrecovery_txn", + async_mock.CoroutineMock(return_value=rev_reg_delta), + ): + assert ( + _test_rev_reg_delta, + { + "ver": "1.0", + "value": { + "accum": "1 0792BD1C8C1A529173FDF54A5B30AC90C2472956622E9F04971D36A9BF77C2C5 1 13B18B6B68AD62605C74FD61088814338EDEEB41C2195F96EC0E83B2B3D0258F 1 102ED0DDE96F6367199CE1C0B138F172BC913B65E37250581606974034F4CA20 1 1C53786D2C15190B57167CDDD2A046CAD63970B5DE43F4D492D4F46B8EEE6FF1 2 095E45DDF417D05FB10933FFC63D474548B7FFFF7888802F07FFFFFF7D07A8A8 1 0000000000000000000000000000000000000000000000000000000000000000" + }, + }, + {"...": "..."}, + ) == await rec.fix_ledger_entry( + profile=_test_profile, + apply_ledger_update=True, + genesis_transactions=json.dumps(TEST_GENESIS_TXN), + ) + async def test_generate_registry_etc(self): rec = IssuerRevRegRecord( issuer_did=TEST_DID, diff --git a/aries_cloudagent/revocation/recover.py b/aries_cloudagent/revocation/recover.py index 49adcbc5ed..f2bf38267c 100644 --- a/aries_cloudagent/revocation/recover.py +++ b/aries_cloudagent/revocation/recover.py @@ -85,7 +85,9 @@ async def fetch_txns(genesis_txns, registry_id): return defn, registry, delta, revoked, tails_temp -async def generate_ledger_rrrecovery_txn(genesis_txns, registry_id, set_revoked): +async def generate_ledger_rrrecovery_txn( + genesis_txns, registry_id, set_revoked, cred_def, rev_reg_def_private +): """Generate a new ledger accum entry, based on wallet vs ledger revocation state.""" new_delta = None @@ -111,7 +113,9 @@ async def generate_ledger_rrrecovery_txn(genesis_txns, registry_id, set_revoked) LOGGER.debug("tails_temp: %s", tails_temp.name) update_registry = registry.copy() - new_delta = update_registry.update(defn, [], updates, tails_temp.name) + new_delta = update_registry.update( + cred_def, defn, rev_reg_def_private, [], updates + ) LOGGER.debug("New delta:") LOGGER.debug(new_delta.to_json())