Skip to content

Commit

Permalink
Update handling of Palace Project collections
Browse files Browse the repository at this point in the history
- Automatically load Palace Project Collections into Aspen and allow the name of the collection to be changed for display within Aspen.
- Allow Palace Project Collections to be omitted from Aspen.
- Reduce the update interval for Palace Project Collections that do not have circulation.
- rebuild jars
  • Loading branch information
mdnoble73 committed Apr 4, 2024
1 parent 7b94d4b commit 479620a
Show file tree
Hide file tree
Showing 28 changed files with 322 additions and 40 deletions.
Binary file modified code/axis_360_export/axis_360_export.jar
Binary file not shown.
Binary file modified code/carlx_export/carlx_export.jar
Binary file not shown.
Binary file modified code/cloud_library_export/cloud_library_export.jar
Binary file not shown.
Binary file modified code/course_reserves_indexer/course_reserves_indexer.jar
Binary file not shown.
Binary file modified code/cron/cron.jar
Binary file not shown.
Binary file modified code/events_indexer/events_indexer.jar
Binary file not shown.
Binary file modified code/evergreen_export/evergreen_export.jar
Binary file not shown.
Binary file modified code/evolve_export/evolve_export.jar
Binary file not shown.
Binary file modified code/hoopla_export/hoopla_export.jar
Binary file not shown.
Binary file modified code/horizon_export/horizon_export.jar
Binary file not shown.
Binary file modified code/koha_export/koha_export.jar
Binary file not shown.
Binary file modified code/oai_indexer/oai_indexer.jar
Binary file not shown.
Binary file modified code/overdrive_extract/overdrive_extract.jar
Binary file not shown.
Binary file modified code/palace_project_export/palace_project_export.jar
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package org.aspendiscovery.palace_project;

public class PalaceProjectCollection {
public long id;
public String palaceProjectName;
public String displayName;
public boolean hasCirculation;
public boolean includeInAspen;
public long lastIndexed;
}
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,9 @@ private static boolean exportPalaceProjectData() {
boolean updatesRun = false;
try{
PreparedStatement getSettingsStmt = aspenConn.prepareStatement("SELECT * from palace_project_settings");
PreparedStatement getCollectionsForSettingStmt = aspenConn.prepareStatement("SELECT * from palace_project_collections where settingId = ?", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
PreparedStatement insertCollectionStmt = aspenConn.prepareStatement("INSERT INTO palace_project_collections (settingId, palaceProjectName, displayName, hasCirculation, includeInAspen) VALUES (?, ?, ?, ?, ?)", PreparedStatement.RETURN_GENERATED_KEYS);
PreparedStatement setLastIndexedStmt = aspenConn.prepareStatement("UPDATE palace_project_collections set lastIndexed = ? where id = ?", PreparedStatement.RETURN_GENERATED_KEYS);

ResultSet getSettingsRS = getSettingsStmt.executeQuery();
int numSettings = 0;
Expand Down Expand Up @@ -248,6 +251,21 @@ private static boolean exportPalaceProjectData() {
//Get a list of all existing records in the database
loadExistingTitles();

//Get a list of collections within Aspen
getCollectionsForSettingStmt.setLong(1, settingsId);
ResultSet collectionsForSettingsRS = getCollectionsForSettingStmt.executeQuery();
HashMap <String, PalaceProjectCollection> palaceProjectCollections = new HashMap<>();
while (collectionsForSettingsRS.next()) {
PalaceProjectCollection collection = new PalaceProjectCollection();
collection.id = collectionsForSettingsRS.getLong("id");
collection.palaceProjectName = collectionsForSettingsRS.getString("palaceProjectName");
collection.displayName = collectionsForSettingsRS.getString("displayName");
collection.hasCirculation = collectionsForSettingsRS.getBoolean("hasCirculation");
collection.includeInAspen = collectionsForSettingsRS.getBoolean("includeInAspen");
collection.lastIndexed = collectionsForSettingsRS.getLong("lastIndexed");
palaceProjectCollections.put(collection.palaceProjectName, collection);
}

String url = palaceProjectBaseUrl + "/" + palaceProjectLibraryId + "/crawlable";
HashMap<String, String> headers = new HashMap<>();
headers.put("Accept", "application/opds+json");
Expand Down Expand Up @@ -276,40 +294,71 @@ private static boolean exportPalaceProjectData() {
continue;
}
validCollections.put(linkTitle, link.getString("href"));
if (!palaceProjectCollections.containsKey(linkTitle)) {
//Add the collection to the database
insertCollectionStmt.setLong(1, settingsId);
insertCollectionStmt.setString(2, linkTitle);
insertCollectionStmt.setString(3, linkTitle);
insertCollectionStmt.setBoolean(4, linkTitle.toLowerCase().contains("marketplace"));
insertCollectionStmt.setBoolean(5, true);
insertCollectionStmt.executeUpdate();
ResultSet generatedKeys = insertCollectionStmt.getGeneratedKeys();
if (generatedKeys.next()){
long collectionId = generatedKeys.getLong(1);
PalaceProjectCollection collection = new PalaceProjectCollection();
collection.id = collectionId;
collection.palaceProjectName = linkTitle;
collection.displayName = linkTitle;
collection.hasCirculation = linkTitle.toLowerCase().contains("marketplace");
collection.includeInAspen = true;
palaceProjectCollections.put(collection.palaceProjectName, collection);
}
}
}
}
}
}
}

long nowInSeconds = new Date().getTime() / 1000;
long yesterdayInSeconds = nowInSeconds - 24 * 60 * 60;
for (String collectionName : validCollections.keySet()) {
String collectionUrl = validCollections.get(collectionName);
while (collectionUrl != null) {
WebServiceResponse responseForCollection = NetworkUtils.getURL(collectionUrl, logger, headers);
if (!response.isSuccess()) {
logEntry.incErrors("Could not get titles from " + collectionUrl + " " + responseForCollection.getMessage());
} else {
JSONObject collectionResponseJSON = new JSONObject(responseForCollection.getMessage());
if (collectionResponseJSON.has("publications")) {
JSONArray responseTitles = collectionResponseJSON.getJSONArray("publications");
if (responseTitles != null && !responseTitles.isEmpty()) {
updateTitlesInDB(collectionName, responseTitles, doFullReload);
logEntry.saveResults();
}
}
collectionUrl = null;
//Get the next URL
if (collectionResponseJSON.has("links")) {
JSONArray links = collectionResponseJSON.getJSONArray("links");
for (int i = 0; i < links.length(); i++) {
JSONObject curLink = links.getJSONObject(i);
if (curLink.getString("rel").equals("next")) {
collectionUrl = curLink.getString("href");
break;
//Index the collection if the collection has circulation or the collection has not been updated for 24 hours
PalaceProjectCollection collection = palaceProjectCollections.get(collectionName);
if (collection.includeInAspen) {
if (collection.hasCirculation || collection.lastIndexed < yesterdayInSeconds) {
//Index all records in the collection
String collectionUrl = validCollections.get(collectionName);
while (collectionUrl != null) {
WebServiceResponse responseForCollection = NetworkUtils.getURL(collectionUrl, logger, headers);
if (!response.isSuccess()) {
logEntry.incErrors("Could not get titles from " + collectionUrl + " " + responseForCollection.getMessage());
} else {
JSONObject collectionResponseJSON = new JSONObject(responseForCollection.getMessage());
if (collectionResponseJSON.has("publications")) {
JSONArray responseTitles = collectionResponseJSON.getJSONArray("publications");
if (responseTitles != null && !responseTitles.isEmpty()) {
updateTitlesInDB(collectionName, responseTitles, doFullReload);
logEntry.saveResults();
}
}
collectionUrl = null;
//Get the next URL
if (collectionResponseJSON.has("links")) {
JSONArray links = collectionResponseJSON.getJSONArray("links");
for (int i = 0; i < links.length(); i++) {
JSONObject curLink = links.getJSONObject(i);
if (curLink.getString("rel").equals("next")) {
collectionUrl = curLink.getString("href");
break;
}
}
}
}
}
}
}else{
//TODO: Remove all currently indexed porjects from solr
}
}
}
Expand Down Expand Up @@ -466,23 +515,23 @@ private static void exportSinglePalaceProjectTitle(String singleWorkId) {
palaceProjectBaseUrl = getSettingsRS.getString("apiUrl");
String palaceProjectLibraryId = getSettingsRS.getString("libraryId");

String url = palaceProjectBaseUrl + "/" + palaceProjectLibraryId + "/crawlable";
HashMap<String, String> headers = new HashMap<>();
headers.put("Accept", "application/opds+json");
headers.put("User-Agent", "Aspen Discovery");
WebServiceResponse response = NetworkUtils.getURL(url, logger, headers);
if (!response.isSuccess()){
logEntry.incErrors("Could not get titles from " + url + " " + response.getMessage());
}else {
JSONObject responseJSON = new JSONObject(response.getMessage());
if (responseJSON.has("publications")) {
JSONArray responseTitles = responseJSON.getJSONArray("publications");
if (responseTitles != null && !responseTitles.isEmpty()) {
//updateTitlesInDB(responseTitles, false);
logEntry.saveResults();
}
}
}
// String url = palaceProjectBaseUrl + "/" + palaceProjectLibraryId + "/crawlable";
// HashMap<String, String> headers = new HashMap<>();
// headers.put("Accept", "application/opds+json");
// headers.put("User-Agent", "Aspen Discovery");
// WebServiceResponse response = NetworkUtils.getURL(url, logger, headers);
// if (!response.isSuccess()){
// logEntry.incErrors("Could not get titles from " + url + " " + response.getMessage());
// }else {
// JSONObject responseJSON = new JSONObject(response.getMessage());
// if (responseJSON.has("publications")) {
// JSONArray responseTitles = responseJSON.getJSONArray("publications");
// if (responseTitles != null && !responseTitles.isEmpty()) {
// //updateTitlesInDB(responseTitles, false);
// logEntry.saveResults();
// }
// }
// }
}
if (numSettings == 0){
logger.error("Unable to find settings for Palace Project, please add settings to the database");
Expand Down
Binary file modified code/polaris_export/polaris_export.jar
Binary file not shown.
Binary file modified code/reindexer/reindexer.jar
Binary file not shown.
Binary file modified code/sideload_processing/sideload_processing.jar
Binary file not shown.
Binary file modified code/sierra_export_api/sierra_export_api.jar
Binary file not shown.
Binary file modified code/symphony_export/symphony_export.jar
Binary file not shown.
Binary file modified code/user_list_indexer/user_list_indexer.jar
Binary file not shown.
9 changes: 9 additions & 0 deletions code/web/release_notes/24.04.00.MD
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@

## Aspen Discovery Updates
// mark
### Axis 360 Updates
- Allow scoping of Axis 360 titles by audience. (*MDN*)

### Evergreen Updates
- When updating from unapi, check the holdable attribute for the copy and if it is set to false set the subfield x to unholdable. (Ticket 126257) (*MDN*)
- When indexing MARC records, mark any item with subfield x of unholdable as not holdable within Aspen. (Ticket 126257) (*MDN*)
Expand All @@ -22,6 +25,12 @@
### Marc holdings updates
- Update logic for determining the display name for an owning library for a marc holding. (Ticket 124256) (*MDN*)

### Palace Project Updates
- Automatically load Palace Project Collections into Aspen and allow the name of the collection to be changed for display within Aspen. (*MDN*)
- Allow Palace Project Collections to be omitted from Aspen. (*MDN*)
- Reduce the update interval for Palace Project Collections that do not have circulation. (*MDN*)
- Allow scoping of Palace Project titles by audience. (*MDN*)

### Search Updates
- When determining the active search location, match the rules for determining scope. First check the subdomain against the active subdomain and then check the location code but only if the subdomain is blank. (Ticket 128603) (*MDN*)

Expand Down
94 changes: 94 additions & 0 deletions code/web/services/PalaceProject/Collections.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
<?php

require_once ROOT_DIR . '/Action.php';
require_once ROOT_DIR . '/sys/PalaceProject/PalaceProjectSetting.php';
require_once ROOT_DIR . '/sys/PalaceProject/PalaceProjectCollection.php';
require_once ROOT_DIR . '/services/Admin/ObjectEditor.php';

class PalaceProject_Collections extends ObjectEditor {
function getObjectType(): string {
return 'PalaceProjectCollections';
}

function getToolName(): string {
return 'Collections';
}

function getModule(): string {
return 'PalaceProject';
}

function getPageTitle(): string {
return 'Palace Project Collections';
}

function getAllObjects($page, $recordsPerPage): array {
$object = new PalaceProjectCollection();
if (isset($_REQUEST['settingId'])) {
$settingId = $_REQUEST['settingId'];
$object->settingId = $settingId;
}
$object->limit(($page - 1) * $recordsPerPage, $recordsPerPage);
$this->applyFilters($object);
$object->orderBy($this->getSort());
$object->find();
$objectList = [];
while ($object->fetch()) {
$objectList[$object->id] = clone $object;
}
return $objectList;
}

function getDefaultSort(): string {
return 'palaceProjectName asc';
}

function getObjectStructure($context = ''): array {
return PalaceProjectCollection::getObjectStructure($context);
}

function getPrimaryKeyColumn(): string {
return 'id';
}

function getIdKeyColumn(): string {
return 'id';
}

function canAddNew() {
return true;
}

function canDelete() {
return true;
}

function getAdditionalObjectActions($existingObject): array {
return [];
}

function getInstructions(): string {
return 'https://help.aspendiscovery.org/help/integration/econtent';
}

function getBreadcrumbs(): array {
$breadcrumbs = [];
$breadcrumbs[] = new Breadcrumb('/Admin/Home', 'Administration Home');
$breadcrumbs[] = new Breadcrumb('/Admin/Home#palace_project', 'Palace Project');
if (isset($_REQUEST['settingId'])) {
$breadcrumbs[] = new Breadcrumb('/PalaceProject/Settings?objectAction=edit&id=' . $this->activeObject->settingId, 'Settings');
$breadcrumbs[] = new Breadcrumb('/PalaceProject/Collections?settingId=' . $this->activeObject->settingId, 'All Collections');
}else{
$breadcrumbs[] = new Breadcrumb('/PalaceProject/Settings', 'All Settings');
}
return $breadcrumbs;
}

function getActiveAdminSection(): string {
return 'palace_project';
}

function canView(): bool {
return UserAccount::userHasPermission('Administer Palace Project');
}
}
6 changes: 6 additions & 0 deletions code/web/sys/Account/User.php
Original file line number Diff line number Diff line change
Expand Up @@ -3669,6 +3669,12 @@ public function getAdminActions() {
} else {
$sections['palace_project']->addAction($palaceProjectScopesAction, 'Administer Palace Project');
}
$palaceProjectCollectionsAction = new AdminAction('Collections', 'Defines the collections within a Palace Project Account.', '/PalaceProject/Collections');
if ($sections['palace_project']->addAction($palaceProjectCollectionsAction, 'Administer Palace Project')) {
$palaceProjectSettingsAction->addSubAction($palaceProjectCollectionsAction, 'Administer Palace Project');
} else {
$sections['palace_project']->addAction($palaceProjectCollectionsAction, 'Administer Palace Project');
}
$sections['palace_project']->addAction(new AdminAction('Indexing Log', 'View the indexing log for Palace Project.', '/PalaceProject/IndexingLog'), [
'View System Reports',
'View Indexing Logs',
Expand Down
79 changes: 79 additions & 0 deletions code/web/sys/PalaceProject/PalaceProjectCollection.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
<?php
require_once ROOT_DIR . '/sys/PalaceProject/PalaceProjectSetting.php';

class PalaceProjectCollection extends DataObject {
public $__table = 'palace_project_collections'; // table name
public $id;
public $settingId;
public $palaceProjectName;
public $displayName;
public $hasCirculation;
public $lastIndexed;

public function getUniquenessFields(): array {
return [
'id',
];
}

public static function getObjectStructure($context = ''): array {
$palaceProjectSettings = [];
$palaceProjectSetting = new PalaceProjectSetting();
$palaceProjectSetting->find();
while ($palaceProjectSetting->fetch()) {
$palaceProjectSettings[$palaceProjectSetting->id] = (string)$palaceProjectSetting;
}

$structure = [
'id' => [
'property' => 'id',
'type' => 'label',
'label' => 'Id',
'description' => 'The unique id within the database',
],
'settingId' => [
'property' => 'settingId',
'type' => 'enum',
'values' => $palaceProjectSettings,
'label' => 'Setting Id',
'readOnly' => true,
],
'palaceProjectName' => [
'property' => 'palaceProjectName',
'type' => 'text',
'label' => 'Palace Project Name',
'description' => 'The name of the collection within Palace Project',
'readOnly' => true,
],
'displayName' => [
'property' => 'displayName',
'type' => 'text',
'label' => 'Aspen Display Name',
'description' => 'The name of the collection for display within Asepn',
],
'hasCirculation' => [
'property' => 'hasCirculation',
'type' => 'checkbox',
'label' => 'Has Circulation',
'description' => 'If the collection has circulation. Collections with circulation will be indexed continuously.',
],
'includeInAspen' => [
'property' => 'includeInAspen',
'type' => 'checkbox',
'label' => 'Include In Aspen',
'description' => 'Whether the collection is included within Aspen.',
],
'lastIndexed' => [
'property' => 'lastIndexed',
'type' => 'timestamp',
'label' => 'Last Indexed',
'description' => 'When the collection was indexed last. Collections without circulation will index every 24 hours',
],
];
return $structure;
}

public function getEditLink($context): string {
return '/PalaceProject/Collections?objectAction=edit&id=' . $this->id;
}
}
Loading

0 comments on commit 479620a

Please sign in to comment.