Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: return error info if Datastore services down #560

Merged
merged 1 commit into from
Sep 12, 2024
Merged

fix: return error info if Datastore services down #560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 34 additions & 9 deletions services/validationService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
getVersionCodelistCommitSha,
getOrgIdPrefixFileName,
validateXMLrecover,
getObjectWithPropertiesAsEnumerable,
} from '../utils/utils.js';
import { client, getStartTime, getElapsedTime } from '../config/appInsights.js';
import validateCodelists from './codelistValidator.js';
Expand Down Expand Up @@ -58,7 +59,7 @@ const groupErrors = (errors, groupKey, additionalKeys) => {
(acc, addKey) => ({ ...acc, [addKey]: grouped[key][0][addKey] }),
{
errors: cleanGroups,
}
},
);
});
};
Expand Down Expand Up @@ -181,7 +182,8 @@ export default async function validate(context, req) {
status: 500,
headers: { 'Content-Type': 'application/json' },
body: {
feedback: 'An unexpected server error occurred preprocessing the XML file. Please contact IATI Secretariat Support.',
feedback:
'An unexpected server error occurred preprocessing the XML file. Please contact IATI Secretariat Support.',
error,
},
};
Expand Down Expand Up @@ -275,7 +277,7 @@ export default async function validate(context, req) {
message: `Version ${
state.iatiVersion
} of the IATI Standard is no longer supported. Supported versions: ${config.VERSIONS.join(
', '
', ',
)}`,
context: [{ text: '' }],
identifier: 'file',
Expand Down Expand Up @@ -309,7 +311,7 @@ export default async function validate(context, req) {
const { errors: codelistResult } = await validateCodelists(
body,
state.iatiVersion,
showDetails
showDetails,
);

state.codelistTime = getElapsedTime(codelistStart);
Expand All @@ -326,7 +328,7 @@ export default async function validate(context, req) {
idSets,
state.schemaErrorsPresent ? getSchema(state.fileType, state.iatiVersion) : '',
showDetails,
showElementMeta
showElementMeta,
);

if (state.schemaErrorsPresent) {
Expand All @@ -341,7 +343,29 @@ export default async function validate(context, req) {

// Advisory Validation
const advisoriesStart = getStartTime();
const advisories = await validateAdvisories(state.iatiVersion, body, adjustLineCountForMissingXmlTag, showDetails);
let advisories;
try {
advisories = await validateAdvisories(
state.iatiVersion,
body,
adjustLineCountForMissingXmlTag,
showDetails,
);
} catch (error) {
context.res = {
status: 500,
headers: { 'Content-Type': 'application/json' },
body: {
feedback:
'There was a problem using the Datastore API to check advisories. Please contact IATI Secretariat Support.',
error: getObjectWithPropertiesAsEnumerable(
error,
Object.getOwnPropertyNames(error).filter((elem) => elem !== 'stack'),
),
},
};
return;
}
state.advisoriesTime = getElapsedTime(advisoriesStart);
context.log({ name: 'Advisory Validate Time (s)', value: state.advisoriesTime });

Expand All @@ -350,7 +374,7 @@ export default async function validate(context, req) {
...schemaErrors,
...flattenErrors(codelistResult),
...flattenErrors(ruleErrors),
...advisories
...advisories,
];

state.exitCategory = 'fullValidation';
Expand All @@ -361,7 +385,7 @@ export default async function validate(context, req) {
combinedErrors,
state,
groupResults,
elementsMeta
elementsMeta,
);

context.res = {
Expand All @@ -377,7 +401,8 @@ export default async function validate(context, req) {
status: 500,
headers: { 'Content-Type': 'application/json' },
body: {
feedback: 'An unexpected server error occurred running validations. Please contact IATI Secretariat Support.',
feedback:
'An unexpected server error occurred running validations. Please contact IATI Secretariat Support.',
error,
},
};
Expand Down
49 changes: 30 additions & 19 deletions utils/datastoreServices.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,51 +1,62 @@
import fetch from 'node-fetch';
import config from '../config/config.js';


async function checkIfBatchIatiIdentifiersInDatastore(iatiIdentifiers) {
const requestHeaders = { Accept: 'application/json' };
requestHeaders[config.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME] = config.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE;
requestHeaders[config.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME] =
config.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE;

const requestBody = { "iati_identifiers" : iatiIdentifiers };
const requestBody = { iati_identifiers: iatiIdentifiers };

const response = await fetch(`${config.DATASTORE_SERVICES_URL}/pub/iati-identifiers/exist`, {
method: 'POST',
headers: requestHeaders,
body: JSON.stringify(requestBody)
body: JSON.stringify(requestBody),
});

const responseJson = await response.json();

if (response.status !== 200) {
throw new Error(
`Error querying the Datastore to check existence of IATI Identifiers. Status: ${response.status}. Message: ${responseJson.error}`
const err = new Error(
`Error querying Datastore services to check existence of IATI Identifiers. Status: ${response.status}.`,
);
err.httpStatusCode = response.status;
err.datastoreServicesResponse = responseJson;
throw err;
}

return responseJson;
};
}

async function checkIfIatiIdentifiersInDatastore(iatiIdentifiers) {

const identifiersPerBatch = config.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS;

const datastoreResults = { num_iati_identifiers_not_found: 0,
total_iati_identifier_occurrences: 0,
const datastoreResults = {
num_iati_identifiers_not_found: 0,
total_iati_identifier_occurrences: 0,
unique_iati_identifiers_found: 0,
iati_identifiers_found: {},
iati_identifiers_not_found: {}
iati_identifiers_not_found: {},
};

for (let i = 0; i < iatiIdentifiers.length; i += identifiersPerBatch) {
const iatiIdentifiersToCheckInBatch = iatiIdentifiers.slice(i, (i + identifiersPerBatch));
const iatiIdentifiersToCheckInBatch = iatiIdentifiers.slice(i, i + identifiersPerBatch);
// eslint-disable-next-line no-await-in-loop
const batchResults = await checkIfBatchIatiIdentifiersInDatastore(iatiIdentifiersToCheckInBatch);
const batchResults = await checkIfBatchIatiIdentifiersInDatastore(
iatiIdentifiersToCheckInBatch,
);

datastoreResults.num_iati_identifiers_not_found += batchResults.num_iati_identifiers_not_found;
datastoreResults.total_iati_identifier_occurrences += batchResults.total_iati_identifier_occurrences;
datastoreResults.unique_iati_identifiers_found += batchResults.unique_iati_identifiers_found;
datastoreResults.num_iati_identifiers_not_found +=
batchResults.num_iati_identifiers_not_found;
datastoreResults.total_iati_identifier_occurrences +=
batchResults.total_iati_identifier_occurrences;
datastoreResults.unique_iati_identifiers_found +=
batchResults.unique_iati_identifiers_found;
Object.assign(datastoreResults.iati_identifiers_found, batchResults.iati_identifiers_found);
Object.assign(datastoreResults.iati_identifiers_not_found, batchResults.iati_identifiers_not_found);
Object.assign(
datastoreResults.iati_identifiers_not_found,
batchResults.iati_identifiers_not_found,
);
}

return datastoreResults;
Expand Down
48 changes: 29 additions & 19 deletions utils/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ const getFileBySha = async (owner, repo, sha, filePath) => {
const body = res.json();
if (res.status !== 200)
throw new Error(
`Error fetching file from github api. Status: ${res.status} Message: ${body.message} `
`Error fetching file from github api. Status: ${res.status} Message: ${body.message} `,
);
return body;
};
Expand All @@ -38,7 +38,7 @@ const getFileCommitSha = async (owner, repo, branch, filePath) => {
const branchBody = await branchRes.json();
if (branchRes.status !== 200)
throw new Error(
`Error fetching sha from github api. Status: ${branchRes.status} Message: ${branchBody.message} `
`Error fetching sha from github api. Status: ${branchRes.status} Message: ${branchBody.message} `,
);
const { sha } = branchBody.commit;
// https://api.github.com/repos/IATI/IATI-Validator-Codelists/commits?sha={sha}&path=codelist_rules.json
Expand All @@ -50,17 +50,17 @@ const getFileCommitSha = async (owner, repo, branch, filePath) => {
Accept: 'application/vnd.github.v3+json',
Authorization: `token ${config.BASIC_GITHUB_TOKEN}`,
},
}
},
);
const fileBody = await fileRes.json();
if (fileRes.status !== 200)
throw new Error(
`Error fetching sha from github api. Status: ${branchRes.status} Message: ${fileBody.message} `
`Error fetching sha from github api. Status: ${branchRes.status} Message: ${fileBody.message} `,
);
// sort to get newest commit
fileBody.sort(
(first, second) =>
new Date(second.commit.committer.date) - new Date(first.commit.committer.date)
new Date(second.commit.committer.date) - new Date(first.commit.committer.date),
);
return fileBody[0].sha;
};
Expand Down Expand Up @@ -122,17 +122,17 @@ config.VERSIONS.forEach(async (version) => {
'IATI',
codelistRepo,
codelistBranch,
'codelist_rules.json'
'codelist_rules.json',
);
codelistRules[version].content = await getFileBySha(
'IATI',
codelistRepo,
codelistRules[version].commitSha,
'codelist_rules.json'
'codelist_rules.json',
);
await redisclient.SET(
`codelistRules${version}`,
JSON.stringify(codelistRules[version])
JSON.stringify(codelistRules[version]),
);
} else {
console.log({
Expand Down Expand Up @@ -160,13 +160,13 @@ config.VERSIONS.forEach(async (version) => {
'IATI',
'IATI-Rulesets',
rulesetBranch,
'rulesets/standard.json'
'rulesets/standard.json',
);
ruleset[version].content = await getFileBySha(
'IATI',
'IATI-Rulesets',
ruleset[version].commitSha,
'rulesets/standard.json'
'rulesets/standard.json',
);
await redisclient.SET(`ruleset${version}`, JSON.stringify(ruleset[version]));
} else {
Expand All @@ -185,13 +185,15 @@ config.VERSIONS.forEach(async (version) => {
['iati-activities', 'iati-organisations'].forEach(async (fileType) => {
schemas[`${fileType}-${version}`] = libxml.parseXml(
(await fs.readFile(`schemas/${version}/${fileType}-schema.xsd`)).toString(),
{ baseUrl: `./schemas/${version}/` }
{ baseUrl: `./schemas/${version}/` },
);
});

// load advisories
try {
const advisoriesJson = (await fs.readFile(`advisory-definitions/advisoryDefinitions-${version}.json`)).toString();
const advisoriesJson = (
await fs.readFile(`advisory-definitions/advisoryDefinitions-${version}.json`)
).toString();
advisoryDefinitions[version] = JSON.parse(advisoriesJson);
console.log(`Loaded advisories for version ${version}`);
} catch (error) {
Expand Down Expand Up @@ -233,7 +235,6 @@ const getAdvisoryDefinitions = (version) => {
throw new Error(`Unable to retrieve advisories in advisoryDefinitions-${version}.json`);
};


const getRulesetCommitSha = (version) => {
if (config.VERSIONS.includes(version)) {
if ('commitSha' in ruleset[version]) {
Expand Down Expand Up @@ -266,7 +267,7 @@ const fetchOrgIdFilename = async () => {
});
if (res.status !== 200) {
console.error(
`HTTP Response from ${ORG_ID_PREFIX_URL}: ${res.status}, while fetching current filename`
`HTTP Response from ${ORG_ID_PREFIX_URL}: ${res.status}, while fetching current filename`,
);
return '';
}
Expand Down Expand Up @@ -321,7 +322,7 @@ const getOrgIdPrefixes = async () => {
const fileName = await fetchOrgIdFilename();
if (fileName === '') {
console.warn(
`No filename in Content-Disposition header from ${ORG_ID_PREFIX_URL}, can't confirm we're using most up-to-date Org-Id prefixes`
`No filename in Content-Disposition header from ${ORG_ID_PREFIX_URL}, can't confirm we're using most up-to-date Org-Id prefixes`,
);
}
if (orgIdPrefixes !== '') {
Expand Down Expand Up @@ -353,7 +354,7 @@ const getOrgIdPrefixes = async () => {
return orgIdPrefixes;
} catch (error) {
console.error(
`Error fetching Organisation ID Prefixes from ${ORG_ID_PREFIX_URL}. Error: ${error}`
`Error fetching Organisation ID Prefixes from ${ORG_ID_PREFIX_URL}. Error: ${error}`,
);
return { fileName: 'not-available', content: new Set() };
}
Expand Down Expand Up @@ -398,7 +399,7 @@ const getOrgIds = async () => {
}, new Set());
} catch (error) {
console.error(
`Error fetching Organsiation IDs from ${PUBLISHERS_URL}. Error: ${error}`
`Error fetching Organsiation IDs from ${PUBLISHERS_URL}. Error: ${error}`,
);
}
}
Expand Down Expand Up @@ -437,8 +438,8 @@ const execXmllint = (input, command) =>
}
return reject(
new Error(
`xmllint exited with code ${code} when executed with ${command}:\n${error}`
)
`xmllint exited with code ${code} when executed with ${command}:\n${error}`,
),
);
});
// pipe input to process
Expand All @@ -454,6 +455,14 @@ const execXmllint = (input, command) =>
*/
const validateXMLrecover = (input) => execXmllint(input, `xmllint --nonet --recover -`);

const getObjectWithPropertiesAsEnumerable = (obj, propertiesToInclude) => {
const enumerableError = {};
Object.values(propertiesToInclude).forEach((propertyName) => {
enumerableError[propertyName] = obj[propertyName];
});
return enumerableError;
};

export {
validateXMLrecover,
getOrgIdPrefixFileName,
Expand All @@ -467,4 +476,5 @@ export {
getAdvisoryDefinitions,
getSchema,
getRulesetCommitSha,
getObjectWithPropertiesAsEnumerable,
};
Loading