Skip to content

Commit

Permalink
Merge pull request #1018 from datopian/ckan/feat/private-datasets
Browse files Browse the repository at this point in the history 
feat(ckan): makes it possible to search private datasets on the ckan api
  • Loading branch information
@demenech
demenech authored Aug 31, 2023
2 parents 91217f3 + 50122cd commit 98d6253
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 108 deletions.
5 changes: 5 additions & 0 deletions .changeset/twenty-mayflies-joke.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@portaljs/ckan': minor
---

package_search method now supports custom headers and include_private parameter
1 change: 1 addition & 0 deletions packages/ckan/src/interfaces/dataset.interface.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ export interface PackageSearchOptions {
query?: string;
resFormat?: Array<string>;
sort?: string;
include_private?: boolean;
}

export interface Tag {
Expand Down
157 changes: 59 additions & 98 deletions packages/ckan/src/lib/ckanapi.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,7 @@ export default class CKAN {

async getDatasetsListWithDetails(options: DatasetListQueryOptions) {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/current_package_list_with_resources?offset=${
options.offset
}&limit=${options.limit}`,
`${this.DMS}/api/3/action/current_package_list_with_resources?offset=${options.offset}&limit=${options.limit}`,
3
);
const responseData = await response.json();
Expand All @@ -44,7 +40,8 @@ export default class CKAN {
}

async packageSearch(
options: PackageSearchOptions
options: PackageSearchOptions,
reqOptions: Partial<RequestInit> = {}
): Promise<{ datasets: Dataset[]; count: number }> {
function buildGroupsQuery(groups: Array<string>) {
if (groups.length > 0) {
Expand Down Expand Up @@ -99,26 +96,26 @@ export default class CKAN {
options.groups,
options?.resFormat
);
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/package_search?start=${options.offset}&rows=${
options.limit
}${fq ? fq : ''}${options.query ? '&q=' + options.query : ''}${
options.sort ? '&sort=' + options.sort : ''
}`,
3
);

let url = `${this.DMS}/api/3/action/package_search?`;
url += `start=${options.offset}`;
url += `&rows=${options.limit}`;
url += fq ? fq : '';
url += options.query ? '&q=' + options.query : '';
url += options.sort ? '&sort=' + options.sort : '';
url += options.include_private
? '&include_private=' + options.include_private
: '';

const response = await fetchRetry(url, 3, reqOptions);
const responseData = await response.json();
const datasets: Array<Dataset> = responseData.result.results;
return { datasets, count: responseData.result.count };
}

async getDatasetDetails(datasetName: string) {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/package_show?id=${datasetName}`,
`${this.DMS}/api/3/action/package_show?id=${datasetName}`,
1
);
const responseData = await response.json();
Expand All @@ -131,9 +128,7 @@ export default class CKAN {

async getDatasetActivityStream(datasetName: string) {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/package_activity_list?id=${datasetName}`,
`${this.DMS}/api/3/action/package_activity_list?id=${datasetName}`,
3
);
const responseData = await response.json();
Expand All @@ -151,9 +146,7 @@ export default class CKAN {
async getUser(userId: string) {
try {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/user_show?id=${userId}`,
`${this.DMS}/api/3/action/user_show?id=${userId}`,
3
);
const responseData = await response.json();
Expand All @@ -166,20 +159,15 @@ export default class CKAN {
}

async getGroupList() {
const response = await fetchRetry(
`${this.DMS}/api/3/action/group_list`,
3
);
const response = await fetchRetry(`${this.DMS}/api/3/action/group_list`, 3);
const responseData = await response.json();
const groups: Array<string> = responseData.result;
return groups;
}

async getGroupsWithDetails() {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/group_list?all_fields=True`,
`${this.DMS}/api/3/action/group_list?all_fields=True`,
3
);
const responseData = await response.json();
Expand All @@ -189,9 +177,7 @@ export default class CKAN {

async getGroupDetails(groupName: string) {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/group_show?id=${groupName}&include_datasets=True`,
`${this.DMS}/api/3/action/group_show?id=${groupName}&include_datasets=True`,
3
);
const responseData = await response.json();
Expand All @@ -201,9 +187,7 @@ export default class CKAN {

async getGroupActivityStream(groupName: string) {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/group_activity_list?id=${groupName}`,
`${this.DMS}/api/3/action/group_activity_list?id=${groupName}`,
3
);
const responseData = await response.json();
Expand All @@ -230,9 +214,7 @@ export default class CKAN {
async getOrgsWithDetails(accrossPages?: boolean) {
if (!accrossPages) {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/organization_list?all_fields=True`,
`${this.DMS}/api/3/action/organization_list?all_fields=True`,
3
);
const responseData = await response.json();
Expand All @@ -251,9 +233,7 @@ export default class CKAN {

for (let i = 0; i < pages; i++) {
let allOrgListResponse = await fetchRetry(
`${
this.DMS
}/api/3/action/organization_list?all_fields=True&offset=${
`${this.DMS}/api/3/action/organization_list?all_fields=True&offset=${
i * 25
}&limit=25`,
3
Expand All @@ -267,9 +247,7 @@ export default class CKAN {

async getOrgDetails(orgName: string) {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/organization_show?id=${orgName}&include_datasets=True`,
`${this.DMS}/api/3/action/organization_show?id=${orgName}&include_datasets=True`,
3
);
const responseData = await response.json();
Expand All @@ -279,9 +257,7 @@ export default class CKAN {

async getOrgActivityStream(orgName: string) {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/organization_activity_list?id=${orgName}`,
`${this.DMS}/api/3/action/organization_activity_list?id=${orgName}`,
3
);
const responseData = await response.json();
Expand All @@ -297,9 +273,7 @@ export default class CKAN {

async getAllTags() {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/tag_list?all_fields=True`,
`${this.DMS}/api/3/action/tag_list?all_fields=True`,
3
);
const responseData = await response.json();
Expand All @@ -308,49 +282,41 @@ export default class CKAN {
}

async getResourcesWithAliasList() {
const response = await fetch(
`${this.DMS}/api/3/action/datastore_search`,
{
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
},
body: JSON.stringify({
id: '_table_metadata',
limit: '32000',
}),
}
);
const response = await fetch(`${this.DMS}/api/3/action/datastore_search`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
},
body: JSON.stringify({
id: '_table_metadata',
limit: '32000',
}),
});
const responseData = await response.json();
const tableMetadata: Array<TableMetadata> = responseData.result.records;
return tableMetadata.filter((item) => item.alias_of);
}

async datastoreSearch(resourceId: string) {
const response = await fetch(
`${this.DMS}/api/3/action/datastore_search`,
{
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
},
body: JSON.stringify({
id: resourceId,
limit: '32000',
}),
}
);
const response = await fetch(`${this.DMS}/api/3/action/datastore_search`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
},
body: JSON.stringify({
id: resourceId,
limit: '32000',
}),
});
const responseData = await response.json();
return responseData.result.records;
}

async getResourceMetadata(resourceId: string) {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/resource_show?id=${resourceId}`,
`${this.DMS}/api/3/action/resource_show?id=${resourceId}`,
3
);
const responseData = await response.json();
Expand All @@ -359,27 +325,22 @@ export default class CKAN {
}

async getResourceInfo(resourceId: string) {
const response = await fetch(
`${this.DMS}/api/3/action/datastore_info`,
{
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
},
body: JSON.stringify({ resource_id: resourceId }),
}
);
const response = await fetch(`${this.DMS}/api/3/action/datastore_info`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
},
body: JSON.stringify({ resource_id: resourceId }),
});
const responseData = await response.json();
const resourceInfo: Array<ResourceInfo> = responseData.result;
return resourceInfo;
}

async getFacetFields(field: 'res_format' | 'tags') {
const response = await fetchRetry(
`${
this.DMS
}/api/3/action/package_search?facet.field=["${field}"]&rows=0`,
`${this.DMS}/api/3/action/package_search?facet.field=["${field}"]&rows=0`,
3
);
const responseData = await response.json();
Expand Down
24 changes: 14 additions & 10 deletions packages/ckan/src/lib/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,14 @@ export function getDaysAgo(date: string) {
return (+today - +createdOn) / msInDay;
}

export default async function fetchRetry(url: string, n: number): Promise<any> {
export default async function fetchRetry(
url: string,
n: number,
opts: Partial<RequestInit> = {}
): Promise<any> {
const abortController = new AbortController();
const id = setTimeout(() => abortController.abort(), 30000);
const res = await fetch(url, { signal: abortController.signal });
const res = await fetch(url, { signal: abortController.signal, ...opts });
clearTimeout(id);
if (!res.ok && n && n > 0) {
return await fetchRetry(url, n - 1);
Expand All @@ -21,13 +25,13 @@ export default async function fetchRetry(url: string, n: number): Promise<any> {
}

export function removeTag(tag?: string) {
if (tag === "{{description}}" || !tag) {
if (tag === '{{description}}' || !tag) {
return undefined;
}
if (typeof window !== "undefined") {
const div = document.createElement("div");
if (typeof window !== 'undefined') {
const div = document.createElement('div');
div.innerHTML = tag;
return div.textContent || div.innerText || "";
return div.textContent || div.innerText || '';
}
return tag;
}
Expand All @@ -38,10 +42,10 @@ export function convertFieldSchema(
) {
function convertToGraphqlString(fieldName: string) {
return fieldName
.replaceAll(" ", "_")
.replaceAll("(", "_")
.replaceAll(")", "_")
.replace(/[^\w\s]|(_)\1/gi, "_");
.replaceAll(' ', '_')
.replaceAll('(', '_')
.replaceAll(')', '_')
.replace(/[^\w\s]|(_)\1/gi, '_');
}
const entries = Object.entries(schema);
return {
Expand Down

0 comments on commit 98d6253

Please sign in to comment.