Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert remaining DatasetTest method to BTB #4014

Merged
merged 4 commits into from
Sep 11, 2023
Merged

Convert remaining DatasetTest method to BTB #4014

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
216e4a6
moved testResourcePurgeDraft to BTB test
paul-m Sep 11, 2023
aecbd7d
remove omit_container
paul-m Sep 11, 2023
00c3c34
unneeded todo
paul-m Sep 11, 2023
2aed80f
docs for strictConfigSchema, using test container for dataset_info
paul-m Sep 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,6 @@ commands:
ddev get https://github.com/GetDKAN/dkan-ddev-addon/archive/refs/heads/<< parameters.addon_branch >>.tar.gz
# Modify config to use our PHP version.
sed -i 's/^php_version.*$/php_version: "<< parameters.php_version >>"/' .ddev/config.dkan.yaml
# Clear out omit_containers.
sed -i '/omit_containers/d' .ddev/config.dkan.yaml
# Don't load the DBA container.
echo "omit_containers: [dba]" >> .ddev/config.dkan.yaml
ddev restart
ddev status
ddev dkan-init --project-version << parameters.dkan_recommended_branch>> --force
Expand Down
107 changes: 82 additions & 25 deletions tests/src/Functional/DatasetBTBTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,19 @@

namespace Drupal\Tests\dkan\Functional;

use Drupal\common\DataResource;
use Drupal\Core\Queue\QueueFactory;
use Drupal\datastore\Service\ResourceLocalizer;
use Drupal\harvest\Load\Dataset;
use Drupal\harvest\HarvestService;
use Drupal\metastore\MetastoreService;
use Drupal\node\NodeStorage;
use Drupal\search_api\Entity\Index;
use Drupal\Tests\BrowserTestBase;
use Harvest\ETL\Extract\DataJson;
use RootedData\RootedJsonData;

/**
* Class DatasetTest
* Dataset tests.
*
* @group dkan
* @group functional
Expand All @@ -29,23 +29,83 @@ class DatasetBTBTest extends BrowserTestBase {
'field',
'harvest',
'metastore',
'metastore_search',
'node',
'search_api',
];

/**
* {@inheritdoc}
*/
protected $defaultTheme = 'starterkit_theme';
protected $defaultTheme = 'stark';

/**
* {@inheritdoc}
*/
protected $strictConfigSchema = FALSE;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@paul-m and I discussed this. This setting is related to change record https://www.drupal.org/node/3362879 and if it is left true, the test will fail due to poorly defined config schema in contrib modules as well. Suggest adding a comment referencing this!


private const S3_PREFIX = 'https://dkan-default-content-files.s3.amazonaws.com/phpunit';
private const FILENAME_PREFIX = 'dkan_default_content_files_s3_amazonaws_com_phpunit_';

/**
* Test the resource purger when the default moderation state is 'draft'.
*/
public function testResourcePurgeDraft() {
$id_1 = uniqid(__FUNCTION__ . '1');
$id_2 = uniqid(__FUNCTION__ . '2');
$id_3 = uniqid(__FUNCTION__ . '3');

// Set default moderation state to draft.
$this->config('workflows.workflow.dkan_publishing')
->set('type_settings.default_moderation_state', 'draft')
->save();

// Post, update and publish a dataset with multiple, changing resources.
$this->storeDatasetRunQueues($id_1, '1.1', ['1.csv', '2.csv'], 'post');
$this->storeDatasetRunQueues($id_1, '1.2', ['3.csv', '1.csv'], 'put');
$this->getMetastore()->publish('dataset', $id_1);
$this->storeDatasetRunQueues($id_1, '1.3', ['1.csv', '5.csv'], 'put');

/** @var \Drupal\common\DatasetInfo $datasetInfo */
$datasetInfo = \Drupal::service('dkan.common.dataset_info');
$info = $datasetInfo->gather($id_1);
$this->assertStringEndsWith('1.csv', $info['latest_revision']['distributions'][0]['file_path']);
$this->assertStringEndsWith('5.csv', $info['latest_revision']['distributions'][1]['file_path']);
$this->assertStringEndsWith('3.csv', $info['published_revision']['distributions'][0]['file_path']);
$this->assertStringEndsWith('1.csv', $info['published_revision']['distributions'][1]['file_path']);

// Verify that only the resources associated with the published and the
// latest revision.
$this->assertEquals(['1.csv', '3.csv', '5.csv'], $this->checkFiles());
$this->assertEquals(3, $this->countTables());

// Add more datasets, only publishing some.
$this->storeDatasetRunQueues($id_2, '2.1', ['2.csv'], 'post');
$this->storeDatasetRunQueues($id_3, '3.1', ['3.csv'], 'post');
$this->getMetastore()->publish('dataset', $id_2);
// Reindex.
$index = Index::load('dkan');
$index->clear();
$index->indexItems();

// Verify search results contain the '1.2' version of $id_1, $id_2 but not $id_3.
$searchResults = $this->container->get('dkan.metastore_search.service')
->search();
$this->assertEquals(2, $searchResults->total);
$this->assertArrayHasKey('dkan_dataset/' . $id_1, $searchResults->results);
$this->assertEquals('1.2', $searchResults->results['dkan_dataset/' . $id_1]->title);
$this->assertArrayHasKey('dkan_dataset/' . $id_2, $searchResults->results);
$this->assertArrayNotHasKey('dkan_dataset/' . $id_3, $searchResults->results);
}

public function testChangingDatasetResourcePerspectiveOnOutput() {
$this->datastoreImportAndQuery();

drupal_flush_all_caches();

$this->changeDatasetsResourceOutputPerspective(ResourceLocalizer::LOCAL_URL_PERSPECTIVE);
$this->config('metastore.settings')
->set('resource_perspective_display', ResourceLocalizer::LOCAL_URL_PERSPECTIVE)
->save();

$metadata = $this->getMetastore()->get('dataset', 123);
$dataset = json_decode($metadata);
Expand All @@ -72,7 +132,7 @@ public function testResourcePurgePublished() {
}

/**
* Test archiving of datasets after a harvest
* Test archiving of datasets after a harvest.
*/
public function testHarvestArchive() {
$plan = $this->getPlan('testHarvestArchive', 'catalog-step-1.json');
Expand Down Expand Up @@ -151,7 +211,7 @@ public function testOrphanDraftDistributionCleanup() {
$this->storeDatasetRunQueues($id_1, '1', ['1.csv']);

// Get the dataset info.
$metadata = \Drupal::service('dkan.common.dataset_info')->gather($id_1);
$metadata = $this->container->get('dkan.common.dataset_info')->gather($id_1);
$distributionTable = $metadata['latest_revision']['distributions'][0]['table_name'];

// Confirm distribution table exists.
Expand All @@ -168,14 +228,20 @@ public function testOrphanDraftDistributionCleanup() {
$this->assertDirectoryExists('public://resources/' . $resourceDirectory);

// Update the modified date for the dataset.
$this->getMetastore()->patch('dataset', $id_1, json_encode(['modified' => '06-05-2020']));
$this->getMetastore()->patch('dataset', $id_1, json_encode(['modified' => '06-05-2222']));

// Simulate datastore_import and cleanup queues post update.
$this->runQueues(['datastore_import', 'orphan_reference_processor']);
$this->runQueues([
'datastore_import',
'orphan_reference_processor',
'orphan_resource_remover',
]);

// Confirm original distribution table removed.
$distributionTableExists = $databaseSchema->tableExists($distributionTable);
$this->assertFalse($distributionTableExists, $distributionTable . ' removed.');
$this->assertFalse(
$databaseSchema->tableExists($distributionTable),
'Distribution table exists: ' . $distributionTable
);

// Confirm original distribution local directory removed.
$this->assertDirectoryDoesNotExist('public://resources/' . $resourceDirectory);
Expand Down Expand Up @@ -262,9 +328,9 @@ private function datasetPostAndRetrieve(): object {
);

$datasetRootedJsonData = $this->getMetastore()->get('dataset', $uuid);
$this->assertIsString("$datasetRootedJsonData");

$retrievedDataset = json_decode($datasetRootedJsonData);
$this->assertInstanceOf(RootedJsonData::class, $datasetRootedJsonData);
// Ensure round-trip for data.
$retrievedDataset = json_decode((string) $datasetRootedJsonData);

$this->assertEquals(
$retrievedDataset->identifier,
Expand All @@ -280,21 +346,12 @@ private function datastoreImportAndQuery() {

$this->runQueues(['datastore_import']);

$queryString = "[SELECT * FROM {$this->getResourceDatastoreTable($resource)}][WHERE lon = \"61.33\"][ORDER BY lat DESC][LIMIT 1 OFFSET 0];";
$queryString = '[SELECT * FROM ' . $this->getResourceDatastoreTable($resource) . '][WHERE lon = "61.33"][ORDER BY lat DESC][LIMIT 1 OFFSET 0];';
$this->queryResource($resource, $queryString);

/**/
}

private function changeDatasetsResourceOutputPerspective(string $perspective = DataResource::DEFAULT_SOURCE_PERSPECTIVE) {
$configFactory = $this->container->get('config.factory');
$config = $configFactory->getEditable('metastore.settings');
$config->set('resource_perspective_display', $perspective);
$config->save();
}

private function getResourceDatastoreTable(object $resource) {
return "{$resource->identifier}__{$resource->version}";
return $resource->identifier . '__' . $resource->version;
}

private function getResourceFromDataset(object $dataset) {
Expand Down Expand Up @@ -360,7 +417,7 @@ private function getData(string $identifier, string $title, array $downloadUrls)
'JSON Schema requires one or more distributions.'
);
// @todo: Figure out how to assert against $factory->getResult()->getError()
// so we can have a useful test fail message.
// so we can have a useful test fail message.
return $valid_metadata_factory->get(json_encode($data), 'dataset');
}

Expand Down
Loading