Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
yuqi1129 committed Jan 7, 2025
1 parent d0b7486 commit 32ce066
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 25 deletions.
17 changes: 8 additions & 9 deletions clients/client-python/gravitino/filesystem/gvfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
from gravitino.exceptions.base import (
GravitinoRuntimeException,
NoSuchCredentialException,
CatalogNotInUseException,
)
from gravitino.filesystem.gvfs_config import GVFSConfig
from gravitino.name_identifier import NameIdentifier
Expand Down Expand Up @@ -896,8 +895,8 @@ def _get_fileset_catalog(self, catalog_ident: NameIdentifier):
finally:
write_lock.release()

def _file_system_is_not_expired(self, expire_time: int):
return expire_time > time.time() * 1000
def _file_system_expired(self, expire_time: int):
return expire_time <= time.time() * 1000

# Disable Too many branches (13/12) (too-many-branches)
# pylint: disable=R0912
Expand All @@ -915,7 +914,7 @@ def _get_filesystem(
name_identifier
)
if cache_value is not None:
if self._file_system_is_not_expired(cache_value[0]):
if not self._file_system_expired(cache_value[0]):
return cache_value[1]
finally:
read_lock.release()
Expand All @@ -928,7 +927,7 @@ def _get_filesystem(
)

if cache_value is not None:
if self._file_system_is_not_expired(cache_value[0]):
if not self._file_system_expired(cache_value[0]):
return cache_value[1]

new_cache_value: Tuple[int, AbstractFileSystem]
Expand Down Expand Up @@ -971,7 +970,7 @@ def _get_gcs_filesystem(self, fileset_catalog: Catalog, identifier: NameIdentifi
NameIdentifier.of(identifier.namespace().level(2), identifier.name())
)
credentials = fileset.support_credentials().get_credentials()
except (NoSuchCredentialException, CatalogNotInUseException) as e:
except NoSuchCredentialException as e:
logger.warning("Failed to get credentials from fileset: %s", e)
credentials = []

Expand Down Expand Up @@ -1005,7 +1004,7 @@ def _get_s3_filesystem(self, fileset_catalog: Catalog, identifier: NameIdentifie
NameIdentifier.of(identifier.namespace().level(2), identifier.name())
)
credentials = fileset.support_credentials().get_credentials()
except (NoSuchCredentialException, CatalogNotInUseException) as e:
except NoSuchCredentialException as e:
logger.warning("Failed to get credentials from fileset: %s", e)
credentials = []

Expand Down Expand Up @@ -1066,7 +1065,7 @@ def _get_oss_filesystem(self, fileset_catalog: Catalog, identifier: NameIdentifi
NameIdentifier.of(identifier.namespace().level(2), identifier.name())
)
credentials = fileset.support_credentials().get_credentials()
except (NoSuchCredentialException, CatalogNotInUseException) as e:
except NoSuchCredentialException as e:
logger.warning("Failed to get credentials from fileset: %s", e)
credentials = []

Expand Down Expand Up @@ -1129,7 +1128,7 @@ def _get_abs_filesystem(self, fileset_catalog: Catalog, identifier: NameIdentifi
NameIdentifier.of(identifier.namespace().level(2), identifier.name())
)
credentials = fileset.support_credentials().get_credentials()
except (NoSuchCredentialException, CatalogNotInUseException) as e:
except NoSuchCredentialException as e:
logger.warning("Failed to get credentials from fileset: %s", e)
credentials = []

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@

def azure_abs_with_credential_is_prepared():
return (
os.environ.get("ABS_STS_ACCOUNT_NAME")
and os.environ.get("ABS_STS_ACCOUNT_KEY")
and os.environ.get("ABS_STS_CONTAINER_NAME")
and os.environ.get("ABS_STS_TENANT_ID")
and os.environ.get("ABS_STS_CLIENT_ID")
and os.environ.get("ABS_STS_CLIENT_SECRET")
os.environ.get("ABS_ACCOUNT_NAME_FOR_CREDENTIAL")
and os.environ.get("ABS_ACCOUNT_KEY_FOR_CREDENTIAL")
and os.environ.get("ABS_CONTAINER_NAME_FOR_CREDENTIAL")
and os.environ.get("ABS_TENANT_ID_FOR_CREDENTIAL")
and os.environ.get("ABS_CLIENT_ID_FOR_CREDENTIAL")
and os.environ.get("ABS_CLIENT_SECRET_FOR_CREDENTIAL")
)


Expand All @@ -54,12 +54,12 @@ def azure_abs_with_credential_is_prepared():
class TestGvfsWithCredentialABS(TestGvfsWithABS):
# Before running this test, please set the make sure azure-bundle-xxx.jar has been
# copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory
azure_abs_account_key = os.environ.get("ABS_STS_ACCOUNT_KEY")
azure_abs_account_name = os.environ.get("ABS_STS_ACCOUNT_NAME")
azure_abs_container_name = os.environ.get("ABS_STS_CONTAINER_NAME")
azure_abs_tenant_id = os.environ.get("ABS_STS_TENANT_ID")
azure_abs_client_id = os.environ.get("ABS_STS_CLIENT_ID")
azure_abs_client_secret = os.environ.get("ABS_STS_CLIENT_SECRET")
azure_abs_account_key = os.environ.get("ABS_ACCOUNT_NAME_FOR_CREDENTIAL")
azure_abs_account_name = os.environ.get("ABS_ACCOUNT_KEY_FOR_CREDENTIAL")
azure_abs_container_name = os.environ.get("ABS_CONTAINER_NAME_FOR_CREDENTIAL")
azure_abs_tenant_id = os.environ.get("ABS_TENANT_ID_FOR_CREDENTIAL")
azure_abs_client_id = os.environ.get("ABS_CLIENT_ID_FOR_CREDENTIAL")
azure_abs_client_secret = os.environ.get("ABS_CLIENT_SECRET_FOR_CREDENTIAL")

metalake_name: str = "TestGvfsWithCredentialABS_metalake" + str(randint(1, 10000))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@
def gcs_with_credential_is_configured():
return all(
[
os.environ.get("GCS_STS_SERVICE_ACCOUNT_JSON_PATH") is not None,
os.environ.get("GCS_STS_BUCKET_NAME") is not None,
os.environ.get("GCS_SERVICE_ACCOUNT_JSON_PATH_FOR_CREDENTIAL") is not None,
os.environ.get("GCS_BUCKET_NAME_FOR_CREDENTIAL") is not None,
]
)

Expand All @@ -42,8 +42,8 @@ def gcs_with_credential_is_configured():
class TestGvfsWithGCSCredential(TestGvfsWithGCS):
# Before running this test, please set the make sure gcp-bundle-x.jar has been
# copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory
key_file = os.environ.get("GCS_STS_SERVICE_ACCOUNT_JSON_PATH")
bucket_name = os.environ.get("GCS_STS_BUCKET_NAME")
key_file = os.environ.get("GCS_SERVICE_ACCOUNT_JSON_PATH_FOR_CREDENTIAL")
bucket_name = os.environ.get("GCS_BUCKET_NAME_FOR_CREDENTIAL")
metalake_name: str = "TestGvfsWithGCSCredential_metalake" + str(randint(1, 10000))

@classmethod
Expand Down

0 comments on commit 32ce066

Please sign in to comment.