Skip to content

Commit

Permalink
/vsigs/: make sure access token with GOOGLE_APPLICATION_FILE in autho…
Browse files Browse the repository at this point in the history 
…rized_user mode is cached

and rework all access token caching logic to support simultaneous
different configurations (if accessing files using different
credentials)
  • Loading branch information
@rouault
rouault committed Aug 30, 2024
1 parent a220d4a commit 779e94a
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 74 deletions.
9 changes: 9 additions & 0 deletions autotest/gcore/vsigs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1188,6 +1188,15 @@ def method(request):

assert data == "foo"

handler = webserver.SequentialHandler()
handler.add("GET", "/gs_fake_bucket/resource2", custom_method=method)
with webserver.install_http_handler(handler):
f = open_for_read("/vsigs/gs_fake_bucket/resource2")
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode("ascii")
gdal.VSIFCloseL(f)
assert data == "foo"

gdal.Unlink("/vsimem/service_account.json")


Expand Down
145 changes: 73 additions & 72 deletions port/cpl_google_cloud.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,65 @@
#include "cpl_sha256.h"
#include "cpl_time.h"
#include "cpl_http.h"
#include "cpl_multiproc.h"
#include "cpl_mem_cache.h"
#include "cpl_aws.h"
#include "cpl_json.h"

#include <mutex>
#include <utility>

#ifdef HAVE_CURL

static CPLMutex *hMutex = nullptr;
static bool bFirstTimeForDebugMessage = true;
static GOA2Manager oStaticManager;

struct GOA2ManagerCache
{
struct ManagerWithMutex
{
std::mutex oMutex{};
GOA2Manager oManager{};

explicit ManagerWithMutex(const GOA2Manager &oManagerIn)
: oManager(oManagerIn)
{
}
};

std::mutex oMutexGOA2ManagerCache{};
lru11::Cache<std::string, std::shared_ptr<ManagerWithMutex>>
oGOA2ManagerCache{};

std::string GetBearer(const GOA2Manager &oManager)
{
const std::string osKey(oManager.GetKey());
std::shared_ptr<ManagerWithMutex> poSharedManager;
{
std::lock_guard oLock(oMutexGOA2ManagerCache);
if (!oGOA2ManagerCache.tryGet(osKey, poSharedManager))
{
poSharedManager = std::make_shared<ManagerWithMutex>(oManager);
oGOA2ManagerCache.insert(osKey, poSharedManager);
}
}
{
std::lock_guard oLock(poSharedManager->oMutex);
const char *pszBearer = poSharedManager->oManager.GetBearer();
return std::string(pszBearer ? pszBearer : "");
}
}

void clear()
{
std::lock_guard oLock(oMutexGOA2ManagerCache);
oGOA2ManagerCache.clear();
}

static GOA2ManagerCache &GetSingleton()
{
static GOA2ManagerCache goGOA2ManagerCache;
return goGOA2ManagerCache;
}
};

/************************************************************************/
/* CPLIsMachineForSureGCEInstance() */
Expand All @@ -66,26 +116,19 @@ bool CPLIsMachineForSureGCEInstance()
bool bIsGCEInstance = false;
if (CPLTestBool(CPLGetConfigOption("CPL_GCE_CHECK_LOCAL_FILES", "YES")))
{
static bool bIsGCEInstanceStatic = false;
static bool bDone = false;
static bool bIsGCEInstanceStatic = []()
{
CPLMutexHolder oHolder(&hMutex);
if (!bDone)
bool bIsGCE = false;
VSILFILE *fp = VSIFOpenL("/sys/class/dmi/id/product_name", "rb");
if (fp)
{
bDone = true;

VSILFILE *fp =
VSIFOpenL("/sys/class/dmi/id/product_name", "rb");
if (fp)
{
const char *pszLine = CPLReadLineL(fp);
bIsGCEInstanceStatic =
pszLine &&
STARTS_WITH_CI(pszLine, "Google Compute Engine");
VSIFCloseL(fp);
}
const char *pszLine = CPLReadLineL(fp);
bIsGCE =
pszLine && STARTS_WITH_CI(pszLine, "Google Compute Engine");
VSIFCloseL(fp);
}
}
return bIsGCE;
}();
bIsGCEInstance = bIsGCEInstanceStatic;
}
return bIsGCEInstance;
Expand Down Expand Up @@ -423,14 +466,6 @@ bool VSIGSHandleHelper::GetConfiguration(const std::string &osPathForOption,
osPathForOption.c_str(), "GS_OAUTH2_REFRESH_TOKEN", ""));
if (!osRefreshToken.empty())
{
if (oStaticManager.GetAuthMethod() ==
GOA2Manager::ACCESS_TOKEN_FROM_REFRESH)
{
CPLMutexHolder oHolder(&hMutex);
oManager = oStaticManager;
return true;
}

std::string osClientId = VSIGetPathSpecificOption(
osPathForOption.c_str(), "GS_OAUTH2_CLIENT_ID", "");
std::string osClientSecret = VSIGetPathSpecificOption(
Expand Down Expand Up @@ -587,14 +622,6 @@ bool VSIGSHandleHelper::GetConfiguration(const std::string &osPathForOption,
{
if (!osOAuth2RefreshToken.empty())
{
if (oStaticManager.GetAuthMethod() ==
GOA2Manager::ACCESS_TOKEN_FROM_REFRESH)
{
CPLMutexHolder oHolder(&hMutex);
oManager = oStaticManager;
return true;
}

std::string osClientId =
CPLGetConfigOption("GS_OAUTH2_CLIENT_ID", "");
std::string osClientSecret =
Expand Down Expand Up @@ -652,6 +679,7 @@ bool VSIGSHandleHelper::GetConfiguration(const std::string &osPathForOption,
CPLDebug("GS", "%s", osMsg.c_str());
}
bFirstTimeForDebugMessage = false;

return oManager.SetAuthFromRefreshToken(
osOAuth2RefreshToken.c_str(), osClientId.c_str(),
osClientSecret.c_str(), nullptr);
Expand All @@ -670,28 +698,18 @@ bool VSIGSHandleHelper::GetConfiguration(const std::string &osPathForOption,
}
}

if (oStaticManager.GetAuthMethod() == GOA2Manager::GCE)
{
CPLMutexHolder oHolder(&hMutex);
oManager = oStaticManager;
return true;
}
// Some Travis-CI workers are GCE machines, and for some tests, we don't
// want this code path to be taken. And on AppVeyor/Window, we would also
// attempt a network access
else if (!CPLTestBool(CPLGetConfigOption("CPL_GCE_SKIP", "NO")) &&
CPLIsMachinePotentiallyGCEInstance())
if (!CPLTestBool(CPLGetConfigOption("CPL_GCE_SKIP", "NO")) &&
CPLIsMachinePotentiallyGCEInstance())
{
oManager.SetAuthFromGCE(nullptr);
if (oManager.GetBearer() != nullptr)

if (!GOA2ManagerCache::GetSingleton().GetBearer(oManager).empty())
{
CPLDebug("GS", "Using GCE inherited permissions");

{
CPLMutexHolder oHolder(&hMutex);
oStaticManager = oManager;
}

bFirstTimeForDebugMessage = false;
return true;
}
Expand Down Expand Up @@ -793,18 +811,14 @@ VSIGSHandleHelper::GetCurlHeaders(const std::string &osVerb,

if (m_oManager.GetAuthMethod() != GOA2Manager::NONE)
{
const char *pszBearer = m_oManager.GetBearer();
if (pszBearer == nullptr)
const std::string osBearer =
GOA2ManagerCache::GetSingleton().GetBearer(m_oManager);
if (osBearer.empty())
return nullptr;

{
CPLMutexHolder oHolder(&hMutex);
oStaticManager = m_oManager;
}

struct curl_slist *headers = nullptr;
headers = curl_slist_append(
headers, CPLSPrintf("Authorization: Bearer %s", pszBearer));
headers, CPLSPrintf("Authorization: Bearer %s", osBearer.c_str()));

if (!m_osUserProject.empty())
{
Expand Down Expand Up @@ -832,26 +846,13 @@ VSIGSHandleHelper::GetCurlHeaders(const std::string &osVerb,
m_osSecretAccessKey, m_osAccessKeyId, m_osUserProject);
}

/************************************************************************/
/* CleanMutex() */
/************************************************************************/

void VSIGSHandleHelper::CleanMutex()
{
if (hMutex != nullptr)
CPLDestroyMutex(hMutex);
hMutex = nullptr;
}

/************************************************************************/
/* ClearCache() */
/************************************************************************/

void VSIGSHandleHelper::ClearCache()
{
CPLMutexHolder oHolder(&hMutex);

oStaticManager = GOA2Manager();
GOA2ManagerCache::GetSingleton().clear();
bFirstTimeForDebugMessage = true;
}

Expand Down
1 change: 0 additions & 1 deletion port/cpl_google_cloud.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ class VSIGSHandleHelper final : public IVSIS3LikeHandleHelper

std::string GetSignedURL(CSLConstList papszOptions);

static void CleanMutex();
static void ClearCache();
};

Expand Down
34 changes: 34 additions & 0 deletions port/cpl_http.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,9 +277,43 @@ class GOA2Manager
return m_osClientEmail;
}

/** Returns a key that can be used to uniquely identify the instance
* parameters (excluding bearer)
*/
std::string GetKey() const
{
std::string osKey(std::to_string(static_cast<int>(m_eMethod))
.append(",client-id=")
.append(m_osClientId)
.append(",client-secret=")
.append(m_osClientSecret)
.append(",refresh-token=")
.append(m_osRefreshToken)
.append(",private-key=")
.append(m_osPrivateKey)
.append(",client-email=")
.append(m_osClientEmail)
.append(",scope=")
.append(m_osScope));
osKey.append(",additional-claims=");
for (const auto *pszOption : m_aosAdditionalClaims)
{
osKey.append(pszOption);
osKey.append("+");
}
osKey.append(",options=");
for (const auto *pszOption : m_aosOptions)
{
osKey.append(pszOption);
osKey.append("+");
}
return osKey;
}

private:
mutable CPLString m_osCurrentBearer{};
mutable time_t m_nExpirationTime = 0;

AuthMethod m_eMethod = NONE;

// for ACCESS_TOKEN_FROM_REFRESH
Expand Down
1 change: 0 additions & 1 deletion port/cpl_vsil_gs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,6 @@ class VSIGSHandle final : public IVSIS3LikeHandle
VSIGSFSHandler::~VSIGSFSHandler()
{
VSICurlFilesystemHandlerBase::ClearCache();
VSIGSHandleHelper::CleanMutex();
}

/************************************************************************/
Expand Down

0 comments on commit 779e94a

Please sign in to comment.