From 32ce0662057fef43961f2d9860290300451d6b26 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 7 Jan 2025 17:32:55 +0800 Subject: [PATCH] fix --- .../gravitino/filesystem/gvfs.py | 17 +++++++------ .../test_gvfs_with_abs_credential.py | 24 +++++++++---------- .../test_gvfs_with_gcs_credential.py | 8 +++---- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index a8e8f7c9cf2..177eee2d751 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -50,7 +50,6 @@ from gravitino.exceptions.base import ( GravitinoRuntimeException, NoSuchCredentialException, - CatalogNotInUseException, ) from gravitino.filesystem.gvfs_config import GVFSConfig from gravitino.name_identifier import NameIdentifier @@ -896,8 +895,8 @@ def _get_fileset_catalog(self, catalog_ident: NameIdentifier): finally: write_lock.release() - def _file_system_is_not_expired(self, expire_time: int): - return expire_time > time.time() * 1000 + def _file_system_expired(self, expire_time: int): + return expire_time <= time.time() * 1000 # Disable Too many branches (13/12) (too-many-branches) # pylint: disable=R0912 @@ -915,7 +914,7 @@ def _get_filesystem( name_identifier ) if cache_value is not None: - if self._file_system_is_not_expired(cache_value[0]): + if not self._file_system_expired(cache_value[0]): return cache_value[1] finally: read_lock.release() @@ -928,7 +927,7 @@ def _get_filesystem( ) if cache_value is not None: - if self._file_system_is_not_expired(cache_value[0]): + if not self._file_system_expired(cache_value[0]): return cache_value[1] new_cache_value: Tuple[int, AbstractFileSystem] @@ -971,7 +970,7 @@ def _get_gcs_filesystem(self, fileset_catalog: Catalog, identifier: NameIdentifi NameIdentifier.of(identifier.namespace().level(2), identifier.name()) ) credentials = fileset.support_credentials().get_credentials() - except (NoSuchCredentialException, CatalogNotInUseException) as e: + except NoSuchCredentialException as e: logger.warning("Failed to get credentials from fileset: %s", e) credentials = [] @@ -1005,7 +1004,7 @@ def _get_s3_filesystem(self, fileset_catalog: Catalog, identifier: NameIdentifie NameIdentifier.of(identifier.namespace().level(2), identifier.name()) ) credentials = fileset.support_credentials().get_credentials() - except (NoSuchCredentialException, CatalogNotInUseException) as e: + except NoSuchCredentialException as e: logger.warning("Failed to get credentials from fileset: %s", e) credentials = [] @@ -1066,7 +1065,7 @@ def _get_oss_filesystem(self, fileset_catalog: Catalog, identifier: NameIdentifi NameIdentifier.of(identifier.namespace().level(2), identifier.name()) ) credentials = fileset.support_credentials().get_credentials() - except (NoSuchCredentialException, CatalogNotInUseException) as e: + except NoSuchCredentialException as e: logger.warning("Failed to get credentials from fileset: %s", e) credentials = [] @@ -1129,7 +1128,7 @@ def _get_abs_filesystem(self, fileset_catalog: Catalog, identifier: NameIdentifi NameIdentifier.of(identifier.namespace().level(2), identifier.name()) ) credentials = fileset.support_credentials().get_credentials() - except (NoSuchCredentialException, CatalogNotInUseException) as e: + except NoSuchCredentialException as e: logger.warning("Failed to get credentials from fileset: %s", e) credentials = [] diff --git a/clients/client-python/tests/integration/test_gvfs_with_abs_credential.py b/clients/client-python/tests/integration/test_gvfs_with_abs_credential.py index 6ad538d0dd5..819befe6fe3 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_abs_credential.py +++ b/clients/client-python/tests/integration/test_gvfs_with_abs_credential.py @@ -38,12 +38,12 @@ def azure_abs_with_credential_is_prepared(): return ( - os.environ.get("ABS_STS_ACCOUNT_NAME") - and os.environ.get("ABS_STS_ACCOUNT_KEY") - and os.environ.get("ABS_STS_CONTAINER_NAME") - and os.environ.get("ABS_STS_TENANT_ID") - and os.environ.get("ABS_STS_CLIENT_ID") - and os.environ.get("ABS_STS_CLIENT_SECRET") + os.environ.get("ABS_ACCOUNT_NAME_FOR_CREDENTIAL") + and os.environ.get("ABS_ACCOUNT_KEY_FOR_CREDENTIAL") + and os.environ.get("ABS_CONTAINER_NAME_FOR_CREDENTIAL") + and os.environ.get("ABS_TENANT_ID_FOR_CREDENTIAL") + and os.environ.get("ABS_CLIENT_ID_FOR_CREDENTIAL") + and os.environ.get("ABS_CLIENT_SECRET_FOR_CREDENTIAL") ) @@ -54,12 +54,12 @@ def azure_abs_with_credential_is_prepared(): class TestGvfsWithCredentialABS(TestGvfsWithABS): # Before running this test, please set the make sure azure-bundle-xxx.jar has been # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory - azure_abs_account_key = os.environ.get("ABS_STS_ACCOUNT_KEY") - azure_abs_account_name = os.environ.get("ABS_STS_ACCOUNT_NAME") - azure_abs_container_name = os.environ.get("ABS_STS_CONTAINER_NAME") - azure_abs_tenant_id = os.environ.get("ABS_STS_TENANT_ID") - azure_abs_client_id = os.environ.get("ABS_STS_CLIENT_ID") - azure_abs_client_secret = os.environ.get("ABS_STS_CLIENT_SECRET") + azure_abs_account_key = os.environ.get("ABS_ACCOUNT_NAME_FOR_CREDENTIAL") + azure_abs_account_name = os.environ.get("ABS_ACCOUNT_KEY_FOR_CREDENTIAL") + azure_abs_container_name = os.environ.get("ABS_CONTAINER_NAME_FOR_CREDENTIAL") + azure_abs_tenant_id = os.environ.get("ABS_TENANT_ID_FOR_CREDENTIAL") + azure_abs_client_id = os.environ.get("ABS_CLIENT_ID_FOR_CREDENTIAL") + azure_abs_client_secret = os.environ.get("ABS_CLIENT_SECRET_FOR_CREDENTIAL") metalake_name: str = "TestGvfsWithCredentialABS_metalake" + str(randint(1, 10000)) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs_credential.py b/clients/client-python/tests/integration/test_gvfs_with_gcs_credential.py index c8f1b78a339..eec502a13bb 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_gcs_credential.py +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs_credential.py @@ -32,8 +32,8 @@ def gcs_with_credential_is_configured(): return all( [ - os.environ.get("GCS_STS_SERVICE_ACCOUNT_JSON_PATH") is not None, - os.environ.get("GCS_STS_BUCKET_NAME") is not None, + os.environ.get("GCS_SERVICE_ACCOUNT_JSON_PATH_FOR_CREDENTIAL") is not None, + os.environ.get("GCS_BUCKET_NAME_FOR_CREDENTIAL") is not None, ] ) @@ -42,8 +42,8 @@ def gcs_with_credential_is_configured(): class TestGvfsWithGCSCredential(TestGvfsWithGCS): # Before running this test, please set the make sure gcp-bundle-x.jar has been # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory - key_file = os.environ.get("GCS_STS_SERVICE_ACCOUNT_JSON_PATH") - bucket_name = os.environ.get("GCS_STS_BUCKET_NAME") + key_file = os.environ.get("GCS_SERVICE_ACCOUNT_JSON_PATH_FOR_CREDENTIAL") + bucket_name = os.environ.get("GCS_BUCKET_NAME_FOR_CREDENTIAL") metalake_name: str = "TestGvfsWithGCSCredential_metalake" + str(randint(1, 10000)) @classmethod