Skip to content

Commit

Permalink
Merge pull request #43574 from nextcloud/enh/noid/lazyappconfig-migra…
Browse files Browse the repository at this point in the history
…te-metadata

enh(metadata): migrate metadata to lazy appconfig
  • Loading branch information
ArtificialOwl authored Apr 17, 2024
2 parents 3aa9c53 + 4e1814a commit b4004a2
Show file tree
Hide file tree
Showing 7 changed files with 55 additions and 54 deletions.
4 changes: 2 additions & 2 deletions lib/private/Files/Cache/Cache.php
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ public function get($file) {
} elseif (!$data) {
return $data;
} else {
$data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? [];
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
return self::cacheEntryFromData($data, $this->mimetypeLoader);
}
}
Expand Down Expand Up @@ -243,7 +243,7 @@ public function getFolderContentsById($fileId) {
$result->closeCursor();

return array_map(function (array $data) use ($metadataQuery) {
$data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? [];
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
return self::cacheEntryFromData($data, $this->mimetypeLoader);
}, $files);
}
Expand Down
6 changes: 3 additions & 3 deletions lib/private/Files/Cache/CacheQueryBuilder.php
Original file line number Diff line number Diff line change
Expand Up @@ -138,11 +138,11 @@ public function whereParentInParameter(string $parameter) {
/**
* join metadata to current query builder and returns an helper
*
* @return IMetadataQuery|null NULL if no metadata have never been generated
* @return IMetadataQuery
*/
public function selectMetadata(): ?IMetadataQuery {
public function selectMetadata(): IMetadataQuery {
$metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid');
$metadataQuery?->retrieveMetadata();
$metadataQuery->retrieveMetadata();
return $metadataQuery;
}
}
7 changes: 1 addition & 6 deletions lib/private/Files/Cache/QuerySearchHelper.php
Original file line number Diff line number Diff line change
Expand Up @@ -195,12 +195,7 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array
$files = $result->fetchAll();

$rawEntries = array_map(function (array $data) use ($metadataQuery) {
// migrate to null safe ...
if ($metadataQuery === null) {
$data['metadata'] = [];
} else {
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
}
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
return Cache::cacheEntryFromData($data, $this->mimetypeLoader);
}, $files);

Expand Down
44 changes: 8 additions & 36 deletions lib/private/FilesMetadata/FilesMetadataManager.php
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@
use OCP\FilesMetadata\IMetadataQuery;
use OCP\FilesMetadata\Model\IFilesMetadata;
use OCP\FilesMetadata\Model\IMetadataValueWrapper;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\IAppConfig;
use Psr\Log\LoggerInterface;

/**
Expand All @@ -69,7 +68,7 @@ class FilesMetadataManager implements IFilesMetadataManager {
public function __construct(
private IEventDispatcher $eventDispatcher,
private IJobList $jobList,
private IConfig $config,
private IAppConfig $appConfig,
private LoggerInterface $logger,
private MetadataRequestService $metadataRequestService,
private IndexRequestService $indexRequestService,
Expand Down Expand Up @@ -206,7 +205,7 @@ public function saveMetadata(IFilesMetadata $filesMetadata): void {
// update metadata types list
$current = $this->getKnownMetadata();
$current->import($filesMetadata->jsonSerialize(true));
$this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current));
$this->appConfig->setValueArray('core', self::CONFIG_KEY, $current->jsonSerialize(), lazy: true);
}

/**
Expand Down Expand Up @@ -235,20 +234,16 @@ public function deleteMetadata(int $fileId): void {
* @param string $fileIdField alias of the field that contains file ids
*
* @inheritDoc
* @return IMetadataQuery|null
* @return IMetadataQuery
* @see IMetadataQuery
* @since 28.0.0
*/
public function getMetadataQuery(
IQueryBuilder $qb,
string $fileTableAlias,
string $fileIdField
): ?IMetadataQuery {
if (!$this->metadataInitiated()) {
return null;
}

return new MetadataQuery($qb, $this->getKnownMetadata(), $fileTableAlias, $fileIdField);
): IMetadataQuery {
return new MetadataQuery($qb, $this, $fileTableAlias, $fileIdField);
}

/**
Expand All @@ -263,8 +258,7 @@ public function getKnownMetadata(): IFilesMetadata {
$this->all = new FilesMetadata();

try {
$data = json_decode($this->config->getAppValue('core', self::CONFIG_KEY, '[]'), true, 127, JSON_THROW_ON_ERROR);
$this->all->import($data);
$this->all->import($this->appConfig->getValueArray('core', self::CONFIG_KEY, lazy: true));
} catch (JsonException) {
$this->logger->warning('issue while reading stored list of metadata. Advised to run ./occ files:scan --all --generate-metadata');
}
Expand Down Expand Up @@ -310,7 +304,7 @@ public function initMetadata(
}

$current->import([$key => ['type' => $type, 'indexed' => $indexed, 'editPermission' => $editPermission]]);
$this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current));
$this->appConfig->setValueArray('core', self::CONFIG_KEY, $current->jsonSerialize(), lazy: true);
$this->all = $current;
}

Expand All @@ -323,26 +317,4 @@ public static function loadListeners(IEventDispatcher $eventDispatcher): void {
$eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class);
$eventDispatcher->addServiceListener(CacheEntryRemovedEvent::class, MetadataDelete::class);
}

/**
* Will confirm that tables were created and store an app value to cache the result.
* Can be removed in 29 as this is to avoid strange situation when Nextcloud files were
* replaced but the upgrade was not triggered yet.
*
* @return bool
*/
private function metadataInitiated(): bool {
if ($this->config->getAppValue('core', self::MIGRATION_DONE, '0') === '1') {
return true;
}

$dbConnection = \OCP\Server::get(IDBConnection::class);
if ($dbConnection->tableExists(MetadataRequestService::TABLE_METADATA)) {
$this->config->setAppValue('core', self::MIGRATION_DONE, '1');

return true;
}

return false;
}
}
30 changes: 28 additions & 2 deletions lib/private/FilesMetadata/MetadataQuery.php
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException;
use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException;
use OCP\FilesMetadata\IFilesMetadataManager;
use OCP\FilesMetadata\IMetadataQuery;
use OCP\FilesMetadata\Model\IFilesMetadata;
use OCP\FilesMetadata\Model\IMetadataValueWrapper;
use Psr\Log\LoggerInterface;

/**
* @inheritDoc
Expand All @@ -43,12 +45,23 @@ class MetadataQuery implements IMetadataQuery {
private array $knownJoinedIndex = [];
public function __construct(
private IQueryBuilder $queryBuilder,
private IFilesMetadata $knownMetadata,
private IFilesMetadata|IFilesMetadataManager $manager,
private string $fileTableAlias = 'fc',
private string $fileIdField = 'fileid',
private string $alias = 'meta',
private string $aliasIndexPrefix = 'meta_index'
) {
if ($manager instanceof IFilesMetadata) {
/**
* Since 29, because knownMetadata is stored in lazy appconfig, it seems smarter
* to not call getKnownMetadata() at the load of this class as it is only needed
* in {@see getMetadataValueField}.
*
* FIXME: remove support for IFilesMetadata
*/
$logger = \OCP\Server::get(LoggerInterface::class);
$logger->debug('It is deprecated to use IFilesMetadata as second parameter when calling MetadataQuery::__construct()');
}
}

/**
Expand Down Expand Up @@ -158,7 +171,20 @@ public function getMetadataKeyField(string $metadataKey): string {
* @since 28.0.0
*/
public function getMetadataValueField(string $metadataKey): string {
return match ($this->knownMetadata->getType($metadataKey)) {
if ($this->manager instanceof IFilesMetadataManager) {
/**
* Since 29, because knownMetadata is stored in lazy appconfig, it seems smarter
* to not call getKnownMetadata() at the load of this class as it is only needed
* in this method.
*
* FIXME: keep only this line and remove support for previous IFilesMetadata in constructor
*/
$knownMetadata = $this->manager->getKnownMetadata();
} else {
$knownMetadata = $this->manager;
}

return match ($knownMetadata->getType($metadataKey)) {
IMetadataValueWrapper::TYPE_STRING => $this->joinedTableAlias($metadataKey) . '.meta_value_string',
IMetadataValueWrapper::TYPE_INT, IMetadataValueWrapper::TYPE_BOOL => $this->joinedTableAlias($metadataKey) . '.meta_value_int',
default => throw new FilesMetadataTypeException('metadata is not set as indexed'),
Expand Down
16 changes: 11 additions & 5 deletions lib/public/FilesMetadata/IFilesMetadataManager.php
Original file line number Diff line number Diff line change
Expand Up @@ -122,30 +122,35 @@ public function deleteMetadata(int $fileId): void;
* @param string $fileTableAlias alias of the table that contains data about files
* @param string $fileIdField alias of the field that contains file ids
*
* @return IMetadataQuery|null NULL if table are not set yet or never used
* @return IMetadataQuery
* @see IMetadataQuery
* @since 28.0.0
*/
public function getMetadataQuery(
IQueryBuilder $qb,
string $fileTableAlias,
string $fileIdField
): ?IMetadataQuery;
): IMetadataQuery;

/**
* returns all type of metadata currently available.
* The list is stored in a IFilesMetadata with null values but correct type.
*
* Note: this method loads lazy appconfig values.
*
* @return IFilesMetadata
* @since 28.0.0
*/
public function getKnownMetadata(): IFilesMetadata;

/**
* initiate a metadata key with its type.
* Initiate a metadata key with its type.
*
* The call is mandatory before using the metadata property in a webdav request.
* It is not needed to only use this method when the app is enabled: the method can be
* called each time during the app loading as the metadata will only be initiated if not known
* The call should be part of a migration/repair step and not be called on app's boot
* process as it is using lazy-appconfig value
*
* Note: this method loads lazy appconfig values.
*
* @param string $key metadata key
* @param string $type metadata type
Expand All @@ -164,6 +169,7 @@ public function getKnownMetadata(): IFilesMetadata;
* @see IMetadataValueWrapper::EDIT_REQ_WRITE_PERMISSION
* @see IMetadataValueWrapper::EDIT_REQ_READ_PERMISSION
* @since 28.0.0
* @since 29.0.0 uses lazy config value - do not use this method out of repair steps
*/
public function initMetadata(string $key, string $type, bool $indexed, int $editPermission): void;
}
2 changes: 2 additions & 0 deletions lib/public/FilesMetadata/IMetadataQuery.php
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ public function getMetadataKeyField(string $metadataKey): string;
/**
* returns the name of the field for metadata string value to be used in query expressions
*
* Note: this method loads lazy appconfig values.
*
* @param string $metadataKey metadata key
*
* @return string table field
Expand Down

0 comments on commit b4004a2

Please sign in to comment.