Skip to content

Commit

Permalink
Send a hash filelist up to the room to ensure we don't duplicate work
Browse files Browse the repository at this point in the history
  • Loading branch information
@redstrate
redstrate committed Jul 31, 2024
1 parent a382c07 commit 0658f09
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 44 deletions.
2 changes: 1 addition & 1 deletion launcher/include/launchercore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ class LauncherCore : public QObject

QCoro::Task<> fetchNews();

QCoro::Task<> handleGameExit(Profile *profile);
QCoro::Task<> handleGameExit(const Profile *profile);

SteamAPI *m_steamApi = nullptr;

Expand Down
7 changes: 4 additions & 3 deletions launcher/include/syncmanager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

#include <Quotient/accountregistry.h>
#include <Quotient/connection.h>
#include <Task>
#include <qcorotask.h>

/**
* @brief Handles setting up the connection to Matrix and all of the fun things needed to do for that.
Expand Down Expand Up @@ -63,6 +63,7 @@ class SyncManager : public QObject
struct PreviousCharacterData {
QString mxcUri;
QString hostname;
QMap<QString, QString> fileHashes;
};

/**
Expand All @@ -74,12 +75,12 @@ class SyncManager : public QObject
* @brief Uploads character data for @p id from @p path.
* @return True if uploaded successfuly, false otherwise.
*/
QCoro::Task<bool> uploadedCharacterData(const QString &id, const QString &path);
QCoro::Task<bool> uploadCharacterArchive(const QString &id, const QString &path, const QMap<QString, QString> &fileHashes);

/**
* @brief Downloads the character data archive from @p mxcUri and extracts it in @p destPath.
*/
QCoro::Task<bool> downloadCharacterData(const QString &mxcUri, const QString &destPath);
QCoro::Task<bool> downloadCharacterArchive(const QString &mxcUri, const QString &destPath);

/**
* @brief Checks if there's a lock.
Expand Down
79 changes: 52 additions & 27 deletions launcher/src/charactersync.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,17 @@

#include <KLocalizedString>
#include <KZip>
#include <QCoro>
#include <qcorosignal.h>
#include <qcorotask.h>

#include "astra_log.h"
#include "syncmanager.h"

const auto gearsetFilename = QStringLiteral("GEARSET.DAT");

CharacterSync::CharacterSync(Account &account, LauncherCore &launcher, QObject *parent)
: launcher(launcher)
: QObject(parent)
, launcher(launcher)
, m_account(account)
{
}
Expand All @@ -22,9 +26,8 @@ QCoro::Task<bool> CharacterSync::sync(const bool initialSync)
co_return true;
}

auto syncManager = launcher.syncManager();
const auto syncManager = launcher.syncManager();
if (!syncManager->connected()) {
qInfo() << "B";
// TODO: provide an option to continue in the UI
Q_EMIT launcher.loginError(i18n("Failed to connect to sync server! Please check your sync settings."));
co_return false;
Expand Down Expand Up @@ -78,21 +81,38 @@ QCoro::Task<bool> CharacterSync::sync(const bool initialSync)
const QString id = dir.fileName(); // FFXIV_CHR0040000001000001 for example
const auto previousData = co_await syncManager->getUploadedCharacterData(id);

// TODO: make this a little bit smarter. We shouldn't waste time re-uploading data that's exactly the same.
if (!initialSync || !previousData.has_value()) {
// if we didn't upload character data yet, upload it now
co_await uploadCharacterData(dir.absoluteFilePath(), id);
} else {
// otherwise, download it

const bool exists = QFile::exists(dir.absoluteFilePath() + QStringLiteral("/GEARSET.DAT"));
// The files are packed into an archive. So if only one of the files doesn't exist or fails the hash check, download the whole thing and overwrite.
bool areFilesDifferent = false;
for (const auto &[file, hash] : previousData->fileHashes.asKeyValueRange()) {
QFile existingFile(QDir(dir.absoluteFilePath()).absoluteFilePath(file));
if (!existingFile.exists()) {
areFilesDifferent = true;
qCDebug(ASTRA_LOG) << id << "does not match locally, reason:" << existingFile.fileName() << "does not exist";
break;
}

// but check first if it's our hostname. only skip if it exists
if (exists && QSysInfo::machineHostName() == previousData->hostname) {
qCDebug(ASTRA_LOG) << "Skipping" << id << "We uploaded this data.";
continue;
existingFile.open(QIODevice::ReadOnly);
const auto existingHash = QString::fromUtf8(QCryptographicHash::hash(existingFile.readAll(), QCryptographicHash::Algorithm::Sha256).toHex());
if (existingHash != hash) {
areFilesDifferent = true;
qCDebug(ASTRA_LOG) << id << "does not match locally, reason: hashes do not match for" << file;
break;
}
}

const bool hasPreviousUpload = !previousData.has_value();
const bool isGameClosing = !initialSync;

// We want to upload if the files are truly different, or there is no existing data on the server.
const bool needsUpload = (areFilesDifferent && isGameClosing) || hasPreviousUpload;

// We want to download if the files are different.
const bool needsDownload = areFilesDifferent;

if (needsUpload) {
// if we didn't upload character data yet, upload it now
co_await uploadCharacterData(dir.absoluteFilePath(), id);
} else if (needsDownload) {
co_await downloadCharacterData(dir.absoluteFilePath(), id, previousData->mxcUri);
}
}
Expand All @@ -103,41 +123,46 @@ QCoro::Task<bool> CharacterSync::sync(const bool initialSync)
QCoro::Task<void> CharacterSync::uploadCharacterData(const QDir &dir, const QString &id)
{
qCDebug(ASTRA_LOG) << "Uploading" << dir << id;
QTemporaryDir tempDir;
const QTemporaryDir tempDir;

auto tempZipPath = tempDir.filePath(QStringLiteral("%1.zip").arg(id));
const auto tempZipPath = tempDir.filePath(QStringLiteral("%1.zip").arg(id));

KZip *zip = new KZip(tempZipPath);
const auto zip = new KZip(tempZipPath);
zip->setCompression(KZip::DeflateCompression);
zip->open(QIODevice::WriteOnly);

QFile gearsetFile(dir.filePath(QStringLiteral("GEARSET.DAT")));
QFile gearsetFile(dir.filePath(gearsetFilename));
gearsetFile.open(QFile::ReadOnly);

zip->writeFile(QStringLiteral("GEARSET.DAT"), gearsetFile.readAll());
const auto data = gearsetFile.readAll();

zip->writeFile(gearsetFilename, data);
zip->close();

co_await launcher.syncManager()->uploadedCharacterData(id, tempZipPath);
QMap<QString, QString> fileHashes;
fileHashes[gearsetFilename] = QString::fromUtf8(QCryptographicHash::hash(data, QCryptographicHash::Algorithm::Sha256).toHex());

co_await launcher.syncManager()->uploadCharacterArchive(id, tempZipPath, fileHashes);
// TODO: error handling

co_return;
}

QCoro::Task<void> CharacterSync::downloadCharacterData(const QDir &dir, const QString &id, const QString &contentUri)
{
QTemporaryDir tempDir;
const QTemporaryDir tempDir;

auto tempZipPath = tempDir.filePath(QStringLiteral("%1.zip").arg(id));
const auto tempZipPath = tempDir.filePath(QStringLiteral("%1.zip").arg(id));

co_await launcher.syncManager()->downloadCharacterData(contentUri, tempZipPath);
co_await launcher.syncManager()->downloadCharacterArchive(contentUri, tempZipPath);

KZip *zip = new KZip(tempZipPath);
auto zip = new KZip(tempZipPath);
zip->setCompression(KZip::DeflateCompression);
zip->open(QIODevice::ReadOnly);

qCDebug(ASTRA_LOG) << "contents:" << zip->directory()->entries();

zip->directory()->file(QStringLiteral("GEARSET.DAT"))->copyTo(dir.absolutePath());
Q_UNUSED(zip->directory()->file(gearsetFilename)->copyTo(dir.absolutePath()))

qCDebug(ASTRA_LOG) << "Extracted character data!";

Expand Down
3 changes: 2 additions & 1 deletion launcher/src/launchercore.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -515,9 +515,10 @@ QCoro::Task<> LauncherCore::fetchNews()
Q_EMIT newsChanged();
}

QCoro::Task<> LauncherCore::handleGameExit(Profile *profile)
QCoro::Task<> LauncherCore::handleGameExit(const Profile *profile)
{
#ifdef BUILD_SYNC
// TODO: once we have Steam API support we can tell Steam to delay putting the Deck to sleep until our upload is complete
if (m_settings->enableSync()) {
Q_EMIT showWindow();

Expand Down
44 changes: 32 additions & 12 deletions launcher/src/syncmanager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ const auto roomType = QStringLiteral("zone.xiv.astra-sync");
const auto syncEventType = QStringLiteral("zone.xiv.astra.sync");
const auto lockEventType = QStringLiteral("zone.xiv.astra.lock");

const auto hostnameKey = QStringLiteral("hostname");
const auto latestKey = QStringLiteral("latest");
const auto noneKey = QStringLiteral("none");
const auto filesKey = QStringLiteral("files");
const auto contentUriKey = QStringLiteral("content-uri");

using namespace Quotient;

SyncManager::SyncManager(QObject *parent)
Expand Down Expand Up @@ -104,7 +110,10 @@ Quotient::Connection *SyncManager::connection() const

QCoro::Task<> SyncManager::sync()
{
// TODO: de-duplicate sync() calls. otherwise if they happen in quick succession, they wait on each other which is useless for our use case
// We don't need two syncs running at once.
if (connection()->syncJob()) {
co_return;
}

auto connection = m_accountRegistry.accounts().first();
connection->sync();
Expand Down Expand Up @@ -209,12 +218,18 @@ QCoro::Task<std::optional<SyncManager::PreviousCharacterData>> SyncManager::getU
co_return std::nullopt;
} else {
qCDebug(ASTRA_LOG) << "previous sync event:" << syncEvent;
co_return PreviousCharacterData{.mxcUri = syncEvent[QStringLiteral("content-uri")].toString(),
.hostname = syncEvent[QStringLiteral("hostname")].toString()};

auto filesVariantMap = syncEvent[filesKey].toVariant().toMap();
QMap<QString, QString> fileHashes;
for (const auto &[file, hashVariant] : filesVariantMap.asKeyValueRange()) {
fileHashes[file] = hashVariant.toString();
}

co_return PreviousCharacterData{.mxcUri = syncEvent[contentUriKey].toString(), .hostname = syncEvent[hostnameKey].toString(), .fileHashes = fileHashes};
}
}

QCoro::Task<bool> SyncManager::uploadedCharacterData(const QString &id, const QString &path)
QCoro::Task<bool> SyncManager::uploadCharacterArchive(const QString &id, const QString &path, const QMap<QString, QString> &fileHashes)
{
Q_ASSERT(m_currentRoom);

Expand All @@ -225,16 +240,22 @@ QCoro::Task<bool> SyncManager::uploadedCharacterData(const QString &id, const QS

const QUrl contentUri = uploadFileJob->contentUri();

QVariantMap fileHashesVariant;
for (const auto &[file, hash] : fileHashes.asKeyValueRange()) {
fileHashesVariant[file] = QVariant::fromValue(hash);
}

auto syncSetState = m_currentRoom->setState(
syncEventType,
id,
QJsonObject{{{QStringLiteral("content-uri"), contentUri.toString()}, {QStringLiteral("hostname"), QSysInfo::machineHostName()}}});
QJsonObject{
{{contentUriKey, contentUri.toString()}, {hostnameKey, QSysInfo::machineHostName()}, {filesKey, QJsonObject::fromVariantMap(fileHashesVariant)}}});
co_await qCoro(syncSetState, &BaseJob::finished);

co_return true;
}

QCoro::Task<bool> SyncManager::downloadCharacterData(const QString &mxcUri, const QString &destPath)
QCoro::Task<bool> SyncManager::downloadCharacterArchive(const QString &mxcUri, const QString &destPath)
{
auto job = connection()->downloadFile(QUrl::fromUserInput(mxcUri), destPath);
co_await qCoro(job, &BaseJob::finished);
Expand All @@ -246,14 +267,14 @@ QCoro::Task<bool> SyncManager::downloadCharacterData(const QString &mxcUri, cons

QCoro::Task<std::optional<QString>> SyncManager::checkLock()
{
const auto lockEvent = m_currentRoom->currentState().contentJson(syncEventType, QStringLiteral("latest"));
const auto lockEvent = m_currentRoom->currentState().contentJson(syncEventType, latestKey);
if (lockEvent.isEmpty()) {
co_return std::nullopt;
}

qCDebug(ASTRA_LOG) << "previous lock event:" << lockEvent;
const QString hostname = lockEvent[QStringLiteral("hostname")].toString();
if (hostname == QStringLiteral("none")) {
const QString hostname = lockEvent[hostnameKey].toString();
if (hostname == noneKey) {
co_return std::nullopt;
}

Expand All @@ -262,15 +283,14 @@ QCoro::Task<std::optional<QString>> SyncManager::checkLock()

QCoro::Task<> SyncManager::setLock()
{
auto lockSetState =
m_currentRoom->setState(syncEventType, QStringLiteral("latest"), QJsonObject{{QStringLiteral("hostname"), QSysInfo::machineHostName()}});
auto lockSetState = m_currentRoom->setState(syncEventType, latestKey, QJsonObject{{hostnameKey, QSysInfo::machineHostName()}});
co_await qCoro(lockSetState, &BaseJob::finished);
co_return;
}

QCoro::Task<> SyncManager::breakLock()
{
auto lockSetState = m_currentRoom->setState(syncEventType, QStringLiteral("latest"), QJsonObject{{QStringLiteral("hostname"), QStringLiteral("none")}});
auto lockSetState = m_currentRoom->setState(syncEventType, latestKey, QJsonObject{{hostnameKey, noneKey}});
co_await qCoro(lockSetState, &BaseJob::finished);
co_return;
}
Expand Down

0 comments on commit 0658f09

Please sign in to comment.