diff --git a/package-lock.json b/package-lock.json index f1ab05ac..7938d191 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3396,6 +3396,11 @@ ], "license": "MIT" }, + "node_modules/sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "node_modules/semver": { "version": "6.3.0", "dev": true, @@ -3800,6 +3805,17 @@ "typedarray-to-buffer": "^3.1.5" } }, + "node_modules/xml-js": { + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz", + "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==", + "dependencies": { + "sax": "^1.2.4" + }, + "bin": { + "xml-js": "bin/cli.js" + } + }, "node_modules/y18n": { "version": "5.0.8", "dev": true, @@ -3875,7 +3891,8 @@ "lodash.isobject": "^3.0.2", "lz-string": "^1.4.4", "parse-http-header": "^1.0.1", - "sort-json": "^2.0.1" + "sort-json": "^2.0.1", + "xml-js": "^1.6.11" } } }, @@ -5104,7 +5121,8 @@ "lodash.isobject": "^3.0.2", "lz-string": "^1.4.4", "parse-http-header": "^1.0.1", - "sort-json": "^2.0.1" + "sort-json": "^2.0.1", + "xml-js": "^1.6.11" } }, "debug": { @@ -6154,6 +6172,11 @@ "safe-buffer": { "version": "5.2.1" }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "semver": { "version": "6.3.0", "dev": true @@ -6430,6 +6453,14 @@ "typedarray-to-buffer": "^3.1.5" } }, + "xml-js": { + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz", + "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==", + "requires": { + "sax": "^1.2.4" + } + }, "y18n": { "version": "5.0.8", "dev": true diff --git a/src/api/opensearch.js b/src/api/opensearch.js index 22ea4cc7..f82448f4 100644 --- a/src/api/opensearch.js +++ b/src/api/opensearch.js @@ -62,7 +62,7 @@ function initRequest(path) { }); } -async function search(targets, body) { +async function search(targets, body, optionsQuery = {}) { const endpoint = elasticsearchEndpoint(); const request = new HttpRequest({ @@ -74,9 +74,49 @@ async function search(targets, body) { }, body: body, path: `/${targets}/_search`, + query: optionsQuery, }); return await awsFetch(request); } -module.exports = { getCollection, getFileSet, getSharedLink, getWork, search }; +async function scroll(scrollId) { + const endpoint = elasticsearchEndpoint(); + + const request = new HttpRequest({ + method: "POST", + hostname: endpoint, + headers: { + Host: endpoint, + "Content-Type": "application/json", + }, + body: JSON.stringify({ scroll: "2m" }), + path: `_search/scroll/${scrollId}`, + }); + return await awsFetch(request); +} + +async function deleteScroll(scrollId) { + const endpoint = elasticsearchEndpoint(); + + const request = new HttpRequest({ + method: "DELETE", + hostname: endpoint, + headers: { + Host: endpoint, + "Content-Type": "application/json", + }, + path: `_search/scroll/${scrollId}`, + }); + return await awsFetch(request); +} + +module.exports = { + getCollection, + getFileSet, + getSharedLink, + getWork, + search, + scroll, + deleteScroll, +}; diff --git a/src/handlers/oai.js b/src/handlers/oai.js new file mode 100644 index 00000000..65e5e722 --- /dev/null +++ b/src/handlers/oai.js @@ -0,0 +1,63 @@ +const { processRequest } = require("./middleware"); +const { baseUrl } = require("../helpers"); +const { + getRecord, + identify, + listIdentifiers, + listMetadataFormats, + listRecords, +} = require("./oai/verbs"); +const { invalidOaiRequest } = require("./oai/xml-transformer"); + +const allowedVerbs = [ + "GetRecord", + "Identify", + "ListIdentifiers", + "ListMetadataFormats", + "ListRecords", + "ListSets", +]; + +/** + * A function to support the OAI-PMH harvesting specfication + */ +exports.handler = async (event) => { + event = processRequest(event); + const url = `${baseUrl(event)}oai`; + let verb, identifier, metadataPrefix, resumptionToken; + if (event.requestContext.http.method === "GET") { + verb = event.queryStringParameters?.verb; + identifier = event.queryStringParameters?.identifier; + metadataPrefix = event.queryStringParameters?.metadataPrefix; + resumptionToken = event.queryStringParameters?.resumptionToken; + } else { + const body = new URLSearchParams(event.body); + verb = body.get("verb"); + identifier = body.get("identifier"); + metadataPrefix = body.get("metadataPrefix"); + resumptionToken = body.get("resumptionToken"); + } + + if (!verb) return invalidOaiRequest("badArgument", "Missing required verb"); + + switch (verb) { + case "GetRecord": + return await getRecord(url, identifier, metadataPrefix); + case "Identify": + return await identify(url); + case "ListIdentifiers": + return await listIdentifiers(url, event, metadataPrefix, resumptionToken); + case "ListMetadataFormats": + return await listMetadataFormats(url); + case "ListRecords": + return await listRecords(url, event, metadataPrefix, resumptionToken); + case "ListSets": + return invalidOaiRequest( + "noSetHierarchy", + "This repository does not support Sets", + 401 + ); + default: + return invalidOaiRequest("badVerb", "Illegal OAI verb"); + } +}; diff --git a/src/handlers/oai/search.js b/src/handlers/oai/search.js new file mode 100644 index 00000000..3dee1870 --- /dev/null +++ b/src/handlers/oai/search.js @@ -0,0 +1,56 @@ +const { search } = require("../../api/opensearch"); +const { + extractRequestedModels, + modelsToTargets, +} = require("../../api/request/models"); +const fs = require("fs"); + +async function earliestRecordCreateDate() { + const body = { + size: 1, + _source: "create_date", + query: { + bool: { + must: [ + { term: { api_model: "Work" } }, + { term: { published: true } }, + { term: { visibility: "Public" } }, + ], + }, + }, + sort: [{ create_date: "desc" }], + }; + const esResponse = await search( + modelsToTargets(extractRequestedModels()), + JSON.stringify(body) + ); + const responseBody = JSON.parse(esResponse.body); + return responseBody?.hits?.hits[0]?._source?.create_date; +} + +async function oaiSearch() { + const body = { + size: 5000, + query: { + bool: { + must: [ + { term: { api_model: "Work" } }, + { term: { published: true } }, + { term: { visibility: "Public" } }, + ], + }, + }, + sort: [{ create_date: "desc" }], + }; + const esResponse = await search( + modelsToTargets(extractRequestedModels()), + JSON.stringify(body), + { scroll: "2m" } + ); + return { + ...esResponse, + expiration: new Date(new Date().getTime() + 2 * 60000).toISOString(), + }; +} + +module.exports = { earliestRecordCreateDate, oaiSearch }; diff --git a/src/handlers/oai/verbs.js b/src/handlers/oai/verbs.js new file mode 100644 index 00000000..8aa07c8c --- /dev/null +++ b/src/handlers/oai/verbs.js @@ -0,0 +1,280 @@ +const { invalidOaiRequest, output } = require("../oai/xml-transformer"); +const { earliestRecordCreateDate, oaiSearch } = require("../oai/search"); +const { deleteScroll, getWork, scroll } = require("../../api/opensearch"); + +const fieldMapper = { + contributor: "dc:contributor", + create_date: "dc:date", + description: "dc:description", + title: "dc:title", + id: "dc:identifier", + language: "dc:language", + creator: "dc:creator", + physical_description_material: "dc:format", + publisher: "dc:publisher", + related_material: "dc:relation", + rights_statement: "dc:rights", + source: "dc:source", + subject: "dc:subject", + work_type: "dc:type", +}; + +const oaiAttributes = { + xmlns: "http://www.openarchives.org/OAI/2.0/", + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsi:schemaLocation": + "http://www.openarchives.org/OAI/2.0/\nhttp://www.openarchives.org/OAI/2.0/OAI-PMH.xsd", +}; + +function header(work) { + return { header: { identifier: work.id, datestamp: work.create_date } }; +} + +function transform(work) { + const filteredWork = Object.keys(work) + .filter((key) => Object.keys(fieldMapper).includes(key)) + .reduce((obj, key) => { + obj[fieldMapper[key]] = work[key]; + return obj; + }, {}); + + const metadata = { + metadata: { + "oai_dc:dc": { + _attributes: { + "xmlns:oai_dc": "http://www.openarchives.org/OAI/2.0/oai_dc/", + "xmlns:dc": "http://purl.org/dc/elements/1.1/", + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsi:schemaLocation": + "http://www.openarchives.org/OAI/2.0/oai_dc/\nhttp://www.openarchives.org/OAI/2.0/oai_dc.xsd", + }, + ...filteredWork, + }, + }, + }; + + return { ...header(work), ...metadata }; +} + +const getRecord = async (url, id) => { + if (!id) + return invalidOaiRequest( + "badArgument", + "You must supply an identifier for GetRecord requests" + ); + const esResponse = await getWork(id); + if (esResponse.statusCode == 200) { + const work = JSON.parse(esResponse.body)._source; + const record = transform(work); + const document = { + OAI_PMH: { + _attributes: oaiAttributes, + responseDate: new Date().toISOString(), + request: { + _attributes: { + verb: "GetRecord", + identifier: id, + metadataPrefix: "oai_dc", + }, + _text: url, + }, + GetRecord: { ...record }, + }, + }; + return output(document); + } else { + return invalidOaiRequest( + "idDoesNotExist", + "The specified record does not exist", + 404 + ); + } +}; + +const identify = async (url) => { + let earliestDatestamp = await earliestRecordCreateDate(); + const obj = { + OAI_PMH: { + _attributes: oaiAttributes, + responseDate: new Date().toISOString(), + request: { + _attributes: { + verb: "Identify", + }, + _text: url, + }, + Identify: { + repositoryName: "Northwestern University Libraries", + baseURL: url, + protocolVersion: "2.0", + earliestDatestamp: earliestDatestamp, + deletedRecord: "no", + granularity: "YYYY-MM-DDThh:mm:ssZ", + }, + }, + }; + return output(obj); +}; + +const listIdentifiers = async (url, event, metadataPrefix, resumptionToken) => { + if (!metadataPrefix) { + return invalidOaiRequest( + "badArgument", + "Missing required metadataPrefix argument" + ); + } + const response = + typeof resumptionToken === "string" && resumptionToken.length !== 0 + ? await scroll(resumptionToken) + : await oaiSearch(); + let headers = []; + let resumptionTokenElement; + + if (response.statusCode == 200) { + const responseBody = JSON.parse(response.body); + let scrollId = responseBody._scroll_id; + const hits = responseBody.hits.hits; + + if (hits.length === 0) { + await deleteScroll(scrollId); + scrollId = ""; + } + headers = hits.map((hit) => header(hit._source)); + + resumptionTokenElement = { + _attributes: { + expirationDate: response.expiration, + }, + _text: scrollId, + }; + const obj = { + OAI_PMH: { + _attributes: oaiAttributes, + responseDate: new Date().toISOString(), + request: { + _attributes: { + verb: "ListIdentifiers", + ...(resumptionToken && { resumptionToken: resumptionToken }), + }, + _text: url, + }, + ListIdentifiers: { + header: headers, + resumptionToken: resumptionTokenElement, + }, + }, + }; + + return output(obj); + } else if ( + response.statusCode === 404 && + response.body.match(/No search context found/) + ) { + return invalidOaiRequest( + "badResumptionToken", + "Your resumptionToken is no longer valid", + 401 + ); + } else { + return invalidOaiRequest( + "badRequest", + "An error occurred processing the ListRecords request" + ); + } +}; + +const listMetadataFormats = (url) => { + const obj = { + OAI_PMH: { + _attributes: oaiAttributes, + responseDate: new Date().toISOString(), + request: { + _attributes: { + verb: "ListMetadataFormats", + }, + _text: url, + }, + ListMetadataFormats: { + metadataFormat: { + metadataPrefix: "oai_dc", + schema: "http://www.openarchives.org/OAI/2.0/oai_dc.xsd", + metadataNamespace: "http://www.openarchives.org/OAI/2.0/oai_dc/", + }, + }, + }, + }; + return output(obj); +}; + +const listRecords = async (url, event, metadataPrefix, resumptionToken) => { + if (!metadataPrefix) { + return invalidOaiRequest( + "badArgument", + "Missing required metadataPrefix argument" + ); + } + const response = + typeof resumptionToken === "string" && resumptionToken.length !== 0 + ? await scroll(resumptionToken) + : await oaiSearch(); + let records = []; + let resumptionTokenElement; + + if (response.statusCode == 200) { + const responseBody = JSON.parse(response.body); + const hits = responseBody.hits.hits; + let scrollId = responseBody._scroll_id; + + if (hits.length === 0) { + await deleteScroll(scrollId); + scrollId = ""; + } + records = hits.map((hit) => transform(hit._source)); + resumptionTokenElement = { + _attributes: { + expirationDate: response.expiration, + }, + _text: scrollId, + }; + const obj = { + OAI_PMH: { + _attributes: oaiAttributes, + responseDate: new Date().toISOString(), + request: { + _attributes: { + verb: "ListRecords", + }, + _text: url, + }, + ListRecords: { + record: records, + resumptionToken: resumptionTokenElement, + }, + }, + }; + + return output(obj); + } else if ( + response.statusCode === 404 && + response.body.match(/No search context found/) + ) { + return invalidOaiRequest( + "badResumptionToken", + "Your resumptionToken is no longer valid", + 401 + ); + } else { + return invalidOaiRequest( + "badRequest", + "An error occurred processing the ListRecords request" + ); + } +}; + +module.exports = { + getRecord, + identify, + listIdentifiers, + listMetadataFormats, + listRecords, +}; diff --git a/src/handlers/oai/xml-transformer.js b/src/handlers/oai/xml-transformer.js new file mode 100644 index 00000000..f9960346 --- /dev/null +++ b/src/handlers/oai/xml-transformer.js @@ -0,0 +1,38 @@ +const convert = require("xml-js"); + +const json2xmlOptions = { compact: true, ignoreComment: true, spaces: 4 }; + +const declaration = { + _declaration: { _attributes: { version: "1.0", encoding: "utf-8" } }, +}; + +const invalidOaiRequest = (oaiCode, message, statusCode = 400) => { + const obj = { + OAI_PMH: { + _attributes: { + xmlns: "http://www.openarchives.org/OAI/2.0/", + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsi:schemaLocation": + "http://www.openarchives.org/OAI/2.0/\nhttp://www.openarchives.org/OAI/2.0/OAI-PMH.xsd", + }, + responseDate: new Date().toISOString(), + error: { + _attributes: { + code: oaiCode, + }, + _text: message, + }, + }, + }; + return output(obj, statusCode); +}; + +const output = (obj, statusCode = 200) => { + return { + statusCode: statusCode, + headers: { "content-type": "application/xml" }, + body: convert.js2xml({ ...declaration, ...obj }, json2xmlOptions), + }; +}; + +module.exports = { invalidOaiRequest, output }; diff --git a/src/package-lock.json b/src/package-lock.json index 1015f252..f9aa749d 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -21,7 +21,8 @@ "lodash.isobject": "^3.0.2", "lz-string": "^1.4.4", "parse-http-header": "^1.0.1", - "sort-json": "^2.0.1" + "sort-json": "^2.0.1", + "xml-js": "^1.6.11" } }, "node_modules/@aws-crypto/ie11-detection": { @@ -1417,6 +1418,11 @@ } ] }, + "node_modules/sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "node_modules/semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -1471,6 +1477,17 @@ "engines": { "node": ">= 8" } + }, + "node_modules/xml-js": { + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz", + "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==", + "dependencies": { + "sax": "^1.2.4" + }, + "bin": { + "xml-js": "bin/cli.js" + } } }, "dependencies": { @@ -2724,6 +2741,11 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -2763,6 +2785,14 @@ "version": "3.2.1", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==" + }, + "xml-js": { + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz", + "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==", + "requires": { + "sax": "^1.2.4" + } } } } diff --git a/src/package.json b/src/package.json index cf745ba1..9f2c540b 100644 --- a/src/package.json +++ b/src/package.json @@ -18,6 +18,7 @@ "lodash.isobject": "^3.0.2", "lz-string": "^1.4.4", "parse-http-header": "^1.0.1", - "sort-json": "^2.0.1" + "sort-json": "^2.0.1", + "xml-js": "^1.6.11" } } diff --git a/template.yaml b/template.yaml index 5e16f219..46512627 100644 --- a/template.yaml +++ b/template.yaml @@ -368,6 +368,32 @@ Resources: ApiId: !Ref dcApi Path: /shared-links/{id} Method: GET + oaiFunction: + Type: AWS::Serverless::Function + Properties: + Handler: handlers/oai.handler + Description: Transforms works into OAI Records. + Policies: + Version: 2012-10-17 + Statement: + - Sid: ESHTTPPolicy + Effect: Allow + Action: + - es:ESHttp* + Resource: "*" + Events: + GetApi: + Type: HttpApi + Properties: + ApiId: !Ref dcApi + Path: /oai + Method: GET + PostApi: + Type: HttpApi + Properties: + ApiId: !Ref dcApi + Path: /oai + Method: POST dcApi: Type: AWS::Serverless::HttpApi Properties: diff --git a/test/fixtures/mocks/scroll-empty.json b/test/fixtures/mocks/scroll-empty.json new file mode 100644 index 00000000..5fc9f49d --- /dev/null +++ b/test/fixtures/mocks/scroll-empty.json @@ -0,0 +1,11 @@ +{ + "_scroll_id": "FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB", + "took": 3, + "timed_out": false, + "_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 }, + "hits": { + "total": { "value": 37345, "relation": "eq" }, + "max_score": null, + "hits": [] + } +} diff --git a/test/fixtures/mocks/scroll-missing.json b/test/fixtures/mocks/scroll-missing.json new file mode 100644 index 00000000..d36671e0 --- /dev/null +++ b/test/fixtures/mocks/scroll-missing.json @@ -0,0 +1,29 @@ +{ + "error": { + "root_cause": [ + { + "type": "search_context_missing_exception", + "reason": "No search context found for id [30222]" + } + ], + "type": "search_phase_execution_exception", + "reason": "all shards failed", + "phase": "query", + "grouped": true, + "failed_shards": [ + { + "shard": -1, + "index": null, + "reason": { + "type": "search_context_missing_exception", + "reason": "No search context found for id [30222]" + } + } + ], + "caused_by": { + "type": "search_context_missing_exception", + "reason": "No search context found for id [30222]" + } + }, + "status": 404 +} diff --git a/test/fixtures/mocks/scroll.json b/test/fixtures/mocks/scroll.json new file mode 100644 index 00000000..5be56dd2 --- /dev/null +++ b/test/fixtures/mocks/scroll.json @@ -0,0 +1,996 @@ +{ + "_scroll_id": "FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB", + "took": 3, + "timed_out": false, + "_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 }, + "hits": { + "total": { "value": 37345, "relation": "eq" }, + "max_score": null, + "hits": [ + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "559ca7fb-55d1-45dc-9d8d-bd2ae2de6ae5", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.474227", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.581432Z", + "id": "559ca7fb-55d1-45dc-9d8d-bd2ae2de6ae5", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/559ca7fb-55d1-45dc-9d8d-bd2ae2de6ae5?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/559ca7fb-55d1-45dc-9d8d-bd2ae2de6ae5", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.581418Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "d6e53de6-c0b3-471f-8134-18b751f4ea9e", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Aut error aut nihil exercitationem autem!", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/559ca7fb-55d1-45dc-9d8d-bd2ae2de6ae5/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360581] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "f0ca6d13-3a78-46ae-b5ce-76e329aef30b", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.712781", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.565314Z", + "id": "f0ca6d13-3a78-46ae-b5ce-76e329aef30b", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/f0ca6d13-3a78-46ae-b5ce-76e329aef30b?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/f0ca6d13-3a78-46ae-b5ce-76e329aef30b", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.565304Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "5d901d47-ecfa-43d6-abc8-2e43db519ace", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Et pariatur vel vitae aliquam expedita qui nihil eaque.", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/f0ca6d13-3a78-46ae-b5ce-76e329aef30b/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360565] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "19cc6f17-9c6a-4b00-8371-a283dc9b4d4b", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.819285", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.549450Z", + "id": "19cc6f17-9c6a-4b00-8371-a283dc9b4d4b", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/19cc6f17-9c6a-4b00-8371-a283dc9b4d4b?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/19cc6f17-9c6a-4b00-8371-a283dc9b4d4b", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.549441Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "7ee89ea2-21a2-484d-a94b-c6c40935160d", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Commodi veritatis blanditiis error odio quia quia cupiditate sit atque.", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/19cc6f17-9c6a-4b00-8371-a283dc9b4d4b/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360549] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "c4e1a707-4503-4730-ae00-c5cad69139b0", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.693657", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.533981Z", + "id": "c4e1a707-4503-4730-ae00-c5cad69139b0", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/c4e1a707-4503-4730-ae00-c5cad69139b0?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/c4e1a707-4503-4730-ae00-c5cad69139b0", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.533970Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "55af2192-8cd3-446e-a7d2-7fd292e78ec4", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Aperiam voluptatem et inventore quas inventore!", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/c4e1a707-4503-4730-ae00-c5cad69139b0/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360533] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "ced0dcaa-bcdf-4ecc-b089-863d39672ded", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.676965", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.517686Z", + "id": "ced0dcaa-bcdf-4ecc-b089-863d39672ded", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/ced0dcaa-bcdf-4ecc-b089-863d39672ded?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/ced0dcaa-bcdf-4ecc-b089-863d39672ded", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.517672Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "c4a85615-2b24-4807-aa61-9a766b316ad2", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Nostrum suscipit ea impedit?", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/ced0dcaa-bcdf-4ecc-b089-863d39672ded/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360517] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "1048c796-0624-4f03-a4cb-a98556f15343", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.658654", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.500983Z", + "id": "1048c796-0624-4f03-a4cb-a98556f15343", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/1048c796-0624-4f03-a4cb-a98556f15343?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/1048c796-0624-4f03-a4cb-a98556f15343", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.500974Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "e05c134f-78af-4a7d-ad39-076f8fdebe93", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Eum saepe eaque sit sit at sed voluptatem quam!", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/1048c796-0624-4f03-a4cb-a98556f15343/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360500] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "50e45e7a-9def-4057-9802-ce24d5265d34", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.439382", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.483488Z", + "id": "50e45e7a-9def-4057-9802-ce24d5265d34", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/50e45e7a-9def-4057-9802-ce24d5265d34?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/50e45e7a-9def-4057-9802-ce24d5265d34", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.483473Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "eea7060b-e051-40c9-9878-8f603814b4b5", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Asperiores iure possimus eaque?", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/50e45e7a-9def-4057-9802-ce24d5265d34/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360483] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "e5abf5ba-8dba-4981-b7bf-d0a14d29cec1", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.414020", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.467385Z", + "id": "e5abf5ba-8dba-4981-b7bf-d0a14d29cec1", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/e5abf5ba-8dba-4981-b7bf-d0a14d29cec1?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/e5abf5ba-8dba-4981-b7bf-d0a14d29cec1", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.467373Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "9387eaf1-fb3f-4e42-867f-11b1e220d1c7", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Dolor illo dolor voluptate laudantium praesentium molestiae?", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/e5abf5ba-8dba-4981-b7bf-d0a14d29cec1/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360467] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "8d7ece78-80ed-4b2c-be8e-9b41fbd30dfd", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.641800", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.449679Z", + "id": "8d7ece78-80ed-4b2c-be8e-9b41fbd30dfd", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/8d7ece78-80ed-4b2c-be8e-9b41fbd30dfd?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/8d7ece78-80ed-4b2c-be8e-9b41fbd30dfd", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.449665Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "0f9285ab-311b-44c4-8846-6b6f5c896750", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Quia nobis aliquid dicta et corporis.", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/8d7ece78-80ed-4b2c-be8e-9b41fbd30dfd/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360449] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "ec5ed76e-587a-4116-a6ba-4d246e2b936a", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.176537", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.431943Z", + "id": "ec5ed76e-587a-4116-a6ba-4d246e2b936a", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/ec5ed76e-587a-4116-a6ba-4d246e2b936a?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/ec5ed76e-587a-4116-a6ba-4d246e2b936a", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.431933Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "75218fca-ec38-4a84-a2b4-5b9dc99cf5dc", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Possimus ipsa et velit unde.", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/ec5ed76e-587a-4116-a6ba-4d246e2b936a/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360431] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "c32eabf6-926c-4750-ad74-42e106327208", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:28.134997", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.415046Z", + "id": "c32eabf6-926c-4750-ad74-42e106327208", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/c32eabf6-926c-4750-ad74-42e106327208?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/c32eabf6-926c-4750-ad74-42e106327208", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.415035Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "2807c9a8-eff8-41a6-bbf1-40c46ea88517", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Et nobis et illum repellat.", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/c32eabf6-926c-4750-ad74-42e106327208/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360415] + }, + { + "_index": "bmq-dev-dc-v2-work-1668533786915", + "_type": "_doc", + "_id": "7ba02d5f-a58d-47c7-839e-37fb916eaaa8", + "_score": null, + "_source": { + "visibility": "Public", + "indexed_at": "2022-11-22T20:36:27.947365", + "contributor": [], + "keywords": [], + "box_number": [], + "ingest_project": {}, + "caption": [], + "physical_description_material": [], + "abstract": [], + "terms_of_use": null, + "modified_date": "2022-11-22T20:36:00.395900Z", + "id": "7ba02d5f-a58d-47c7-839e-37fb916eaaa8", + "subject": [], + "preservation_level": null, + "cultural_context": [], + "rights_statement": {}, + "table_of_contents": [], + "batch_ids": [], + "box_name": [], + "collection": {}, + "ingest_sheet": {}, + "iiif_manifest": "https://dcapi.rdc-staging.library.northwestern.edu/works/7ba02d5f-a58d-47c7-839e-37fb916eaaa8?as=iiif", + "creator": [], + "technique": [], + "library_unit": null, + "legacy_identifier": [], + "scope_and_contents": [], + "provenance": [], + "rights_holder": [], + "date_created": [], + "series": [], + "language": [], + "api_link": "https://dcapi.rdc-staging.library.northwestern.edu/works/7ba02d5f-a58d-47c7-839e-37fb916eaaa8", + "file_sets": [], + "physical_description_size": [], + "catalog_key": [], + "location": [], + "representative_file_set": { + "aspect_ratio": 1.0, + "id": null, + "url": "https://iiif.dev.rdc.library.northwestern.edu/iiif/2/bmq-dev/00000000-0000-0000-0000-000000000001" + }, + "source": [], + "create_date": "2022-11-22T20:36:00.395890Z", + "description": [], + "publisher": [], + "published": true, + "folder_names": [], + "notes": [], + "genre": [], + "csv_metadata_update_jobs": [], + "related_material": [], + "related_url": [], + "identifier": [], + "style_period": [], + "alternate_title": [], + "accession_number": "7e883205-793a-4953-9c4c-d57bae5a6753", + "project": { + "cycle": null, + "desc": null, + "manager": null, + "name": null, + "proposer": null, + "task_number": null + }, + "license": null, + "folder_numbers": [], + "title": "Officia fugiat suscipit enim sunt ratione qui.", + "work_type": "Image", + "thumbnail": "https://dcapi.rdc-staging.library.northwestern.edu/works/7ba02d5f-a58d-47c7-839e-37fb916eaaa8/thumbnail", + "status": null, + "api_model": "Work", + "ark": null + }, + "sort": [1669149360395] + } + ] + } +} diff --git a/test/fixtures/mocks/search-earliest-create-date.json b/test/fixtures/mocks/search-earliest-create-date.json new file mode 100644 index 00000000..009daaa7 --- /dev/null +++ b/test/fixtures/mocks/search-earliest-create-date.json @@ -0,0 +1,29 @@ +{ + "took": 3, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 10000, + "relation": "gte" + }, + "max_score": null, + "hits": [ + { + "_index": "dc-v2-work-1658438320350940", + "_type": "_doc", + "_id": "559ca7fb-55d1-45dc-9d8d-bd2ae2de6ae5", + "_score": null, + "_source": { + "create_date": "2022-11-22T20:36:00.581418Z" + }, + "sort": [1669149360581] + } + ] + } +} diff --git a/test/integration/oai.test.js b/test/integration/oai.test.js new file mode 100644 index 00000000..adc0bec8 --- /dev/null +++ b/test/integration/oai.test.js @@ -0,0 +1,420 @@ +"use strict"; + +const chai = require("chai"); +const expect = chai.expect; +const { handler } = require("../../src/handlers/oai"); +const convert = require("xml-js"); +chai.use(require("chai-http")); + +const xmlOpts = { compact: true, alwaysChildren: true }; + +describe("Oai routes", () => { + helpers.saveEnvironment(); + const mock = helpers.mockIndex(); + + describe("POST /oai", () => { + it("supports the GetRecord verb", async () => { + const body = "verb=GetRecord&identifier=1234&metadataPrefix=oai_dc"; + mock + .get("/dc-v2-work/_doc/1234") + .reply(200, helpers.testFixture("mocks/work-1234.json")); + const event = helpers.mockEvent("POST", "/oai").body(body).render(); + const result = await handler(event); + expect(result.statusCode).to.eq(200); + expect(result).to.have.header("content-type", /application\/xml/); + + const resultBody = convert.xml2js(result.body, xmlOpts); + let metadata = resultBody.OAI_PMH.GetRecord.metadata["oai_dc:dc"]; + expect(metadata) + .to.be.an("object") + .and.to.deep.include.keys( + "dc:contributor", + "dc:creator", + "dc:date", + "dc:description", + "dc:format", + "dc:identifier", + "dc:language", + "dc:publisher", + "dc:relation", + "dc:rights", + "dc:source", + "dc:subject", + "dc:title", + "dc:type" + ); + }); + + it("enforces the id parameter for the GetRecord verb", async () => { + const body = "verb=GetRecord&metadataPrefix=oai_dc"; + const event = helpers.mockEvent("POST", "/oai").body(body).render(); + const result = await handler(event); + expect(result.statusCode).to.eq(400); + expect(result).to.have.header("content-type", /application\/xml/); + + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error["_attributes"]["code"]).to.eq( + "badArgument" + ); + expect(resultBody.OAI_PMH.error["_text"]).to.eq( + "You must supply an identifier for GetRecord requests" + ); + }); + + it("provides the correct error code when GetRecord does not find a matching work", async () => { + const body = "verb=GetRecord&identifier=1234&metadataPrefix=oai_dc"; + mock + .get("/dc-v2-work/_doc/1234") + .reply(404, helpers.testFixture("mocks/missing-work-1234.json")); + const event = helpers.mockEvent("POST", "/oai").body(body).render(); + const result = await handler(event); + expect(result.statusCode).to.eq(404); + expect(result).to.have.header("content-type", /application\/xml/); + + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error["_attributes"]["code"]).to.eq( + "idDoesNotExist" + ); + expect(resultBody.OAI_PMH.error["_text"]).to.eq( + "The specified record does not exist" + ); + }); + + it("supports the ListRecords verb", async () => { + const body = "verb=ListRecords&metadataPrefix=oai_dc"; + mock + .post("/dc-v2-work/_search?scroll=2m") + .reply(200, helpers.testFixture("mocks/scroll.json")); + const event = helpers.mockEvent("POST", "/oai").body(body).render(); + const result = await handler(event); + expect(result.statusCode).to.eq(200); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.ListRecords.record) + .to.be.an("array") + .and.to.have.lengthOf(12); + }); + + it("uses an empty resumptionToken to tell harvesters that list requests are complete", async () => { + mock + .post( + "/_search/scroll/FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB" + ) + .reply(200, helpers.testFixture("mocks/scroll-empty.json")); + + mock + .delete( + "/_search/scroll/FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB" + ) + .reply(200, {}); + + const body = + "verb=ListRecords&metadataPrefix=oai_dc&resumptionToken=FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB"; + const event = helpers.mockEvent("POST", "/oai").body(body).render(); + const result = await handler(event); + expect(result.statusCode).to.eq(200); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + const resumptionToken = resultBody.OAI_PMH.ListRecords.resumptionToken; + expect(resumptionToken).to.not.haveOwnProperty("_text"); + }); + + it("returns a badResumptionToken error when a resumptionToken expires", async () => { + mock + .post( + "/_search/scroll/FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB" + ) + .reply(404, helpers.testFixture("mocks/scroll-missing.json")); + + const body = + "verb=ListRecords&metadataPrefix=oai_dc&resumptionToken=FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB"; + const event = helpers.mockEvent("POST", "/oai").body(body).render(); + const result = await handler(event); + expect(result.statusCode).to.eq(401); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error["_attributes"]["code"]).to.eq( + "badResumptionToken" + ); + expect(resultBody.OAI_PMH.error["_text"]).to.eq( + "Your resumptionToken is no longer valid" + ); + }); + + it("fails gracefully", async () => { + mock + .post( + "/_search/scroll/FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB" + ) + .reply(404, helpers.testFixture("mocks/missing-index.json")); + + const body = + "verb=ListRecords&metadataPrefix=oai_dc&resumptionToken=FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB"; + const event = helpers.mockEvent("POST", "/oai").body(body).render(); + const result = await handler(event); + expect(result.statusCode).to.eq(400); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error["_attributes"]["code"]).to.eq( + "badRequest" + ); + expect(resultBody.OAI_PMH.error["_text"]).to.eq( + "An error occurred processing the ListRecords request" + ); + }); + + it("requires a metadataPrefix", async () => { + const body = "verb=ListRecords"; + mock + .post("/dc-v2-work/_search?scroll=2m") + .reply(200, helpers.testFixture("mocks/scroll.json")); + const event = helpers.mockEvent("POST", "/oai").body(body).render(); + const result = await handler(event); + expect(result.statusCode).to.eq(400); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error["_attributes"]["code"]).to.eq( + "badArgument" + ); + expect(resultBody.OAI_PMH.error["_text"]).to.eq( + "Missing required metadataPrefix argument" + ); + }); + + it("supports the ListMetadataFormats verb", async () => { + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ verb: "ListMetadataFormats", metadataPrefix: "oai_dc" }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(200); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + const listMetadataFormatsElement = + resultBody.OAI_PMH.ListMetadataFormats.metadataFormat; + expect(listMetadataFormatsElement.metadataNamespace._text).to.eq( + "http://www.openarchives.org/OAI/2.0/oai_dc/" + ); + }); + }); + + describe("GET /oai", () => { + it("requires a verb", async () => { + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ metadataPrefix: "oai_dc" }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(400); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error._attributes).to.include({ + code: "badArgument", + }); + expect(resultBody.OAI_PMH.error._text).to.eq("Missing required verb"); + }); + + it("supports the Identify verb", async () => { + const query = { + size: 1, + _source: "create_date", + query: { + bool: { + must: [ + { term: { api_model: "Work" } }, + { term: { published: true } }, + { term: { visibility: "Public" } }, + ], + }, + }, + sort: [{ create_date: "desc" }], + }; + mock + .post("/dc-v2-work/_search", query) + .reply( + 200, + helpers.testFixture("mocks/search-earliest-create-date.json") + ); + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ verb: "Identify", metadataPrefix: "oai_dc" }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(200); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + const identifyElement = resultBody.OAI_PMH.Identify; + expect(identifyElement.earliestDatestamp._text).to.eq( + "2022-11-22T20:36:00.581418Z" + ); + expect(identifyElement.deletedRecord._text).to.eq("no"); + expect(identifyElement.granularity._text).to.eq("YYYY-MM-DDThh:mm:ssZ"); + }); + + it("supports the ListRecords verb", async () => { + mock + .post("/dc-v2-work/_search?scroll=2m") + .reply(200, helpers.testFixture("mocks/scroll.json")); + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ verb: "ListRecords", metadataPrefix: "oai_dc" }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(200); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.ListRecords.record) + .to.be.an("array") + .to.have.lengthOf(12); + }); + + it("does not support the ListSets verb", async () => { + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ verb: "ListSets", metadataPrefix: "oai_dc" }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(401); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error._attributes).to.include({ + code: "noSetHierarchy", + }); + }); + + it("supports the ListIdentifiers verb", async () => { + mock + .post("/dc-v2-work/_search?scroll=2m") + .reply(200, helpers.testFixture("mocks/scroll.json")); + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ verb: "ListIdentifiers", metadataPrefix: "oai_dc" }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(200); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + const resumptionToken = + resultBody.OAI_PMH.ListIdentifiers.resumptionToken; + expect(resumptionToken["_text"]).to.have.lengthOf(120); + }); + + it("requires a metadataPrefix for the ListIdentifiers verb", async () => { + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ verb: "ListIdentifiers" }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(400); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error._attributes).to.include({ + code: "badArgument", + }); + expect(resultBody.OAI_PMH.error._text).to.eq( + "Missing required metadataPrefix argument" + ); + }); + + it("uses an empty resumptionToken to tell harvesters that list requests are complete", async () => { + mock + .post( + "/_search/scroll/FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB" + ) + .reply(200, helpers.testFixture("mocks/scroll-empty.json")); + + mock + .delete( + "/_search/scroll/FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB" + ) + .reply(200, {}); + + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ + verb: "ListIdentifiers", + metadataPrefix: "oai_dc", + resumptionToken: + "FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB", + }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(200); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + const resumptionToken = + resultBody.OAI_PMH.ListIdentifiers.resumptionToken; + expect(resumptionToken).to.not.haveOwnProperty("_text"); + }); + + it("returns a badResumptionToken error when a resumptionToken expires", async () => { + mock + .post( + "/_search/scroll/FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB" + ) + .reply(404, helpers.testFixture("mocks/scroll-missing.json")); + + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ + verb: "ListIdentifiers", + metadataPrefix: "oai_dc", + resumptionToken: + "FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB", + }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(401); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error["_attributes"]["code"]).to.eq( + "badResumptionToken" + ); + expect(resultBody.OAI_PMH.error["_text"]).to.eq( + "Your resumptionToken is no longer valid" + ); + }); + + it("fails gracefully", async () => { + mock + .post( + "/_search/scroll/FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB" + ) + .reply(404, helpers.testFixture("mocks/missing-index.json")); + + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ + verb: "ListIdentifiers", + metadataPrefix: "oai_dc", + resumptionToken: + "FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFm1jN3ZCajdnUURpbUhad1hIYnNsQmcAAAAAAAB2DhZXbmtMZVF5Q1JsMi1ScGRsYUlHLUtB", + }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(400); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error["_attributes"]["code"]).to.eq( + "badRequest" + ); + expect(resultBody.OAI_PMH.error["_text"]).to.eq( + "An error occurred processing the ListRecords request" + ); + }); + + it("provides an error when an incorrect verb is submitted", async () => { + const event = helpers + .mockEvent("GET", "/oai") + .queryParams({ verb: "BadVerb", metadataPrefix: "oai_dc" }) + .render(); + const result = await handler(event); + expect(result.statusCode).to.eq(400); + expect(result).to.have.header("content-type", /application\/xml/); + const resultBody = convert.xml2js(result.body, xmlOpts); + expect(resultBody.OAI_PMH.error._attributes).to.include({ + code: "badVerb", + }); + }); + }); +});