Add additional configuration options to System Variables to control h…

…ow solr commits are done during record deletions

code/reindexer/src/org/aspen_discovery/reindexer/GroupedWorkIndexer.java

@@ -33,6 +33,8 @@ public class GroupedWorkIndexer {
 	private final BaseIndexingLogEntry logEntry;
 	private final Logger logger;
 	private final Long indexStartTime;
+	private int deletionCommitInterval = 1000;
+	private boolean waitAfterDeleteCommit = false;
 	private int totalRecordsHandled = 0;
 	private ConcurrentUpdateHttp2SolrClient updateServer;
 	private RecordGroupingProcessor recordGroupingProcessor;
@@ -207,14 +209,16 @@ public GroupedWorkIndexer(String serverName, Connection dbConn, Ini configIni, b
 
 		//Check to see if we should store record details in Solr
 		try{
-			PreparedStatement systemVariablesStmt = dbConn.prepareStatement("SELECT storeRecordDetailsInSolr, storeRecordDetailsInDatabase, indexVersion, searchVersion, processEmptyGroupedWorks, enableNovelistSeriesIntegration from system_variables");
+			PreparedStatement systemVariablesStmt = dbConn.prepareStatement("SELECT storeRecordDetailsInSolr, storeRecordDetailsInDatabase, indexVersion, searchVersion, processEmptyGroupedWorks, enableNovelistSeriesIntegration, deletionCommitInterval, waitAfterDeleteCommit from system_variables");
 			ResultSet systemVariablesRS = systemVariablesStmt.executeQuery();
 			if (systemVariablesRS.next()){
 				this.storeRecordDetailsInSolr = systemVariablesRS.getBoolean("storeRecordDetailsInSolr");
 				this.storeRecordDetailsInDatabase = systemVariablesRS.getBoolean("storeRecordDetailsInDatabase");
 				this.indexVersion = systemVariablesRS.getInt("indexVersion");
 				this.searchVersion = systemVariablesRS.getInt("searchVersion");
 				this.enableNovelistSeriesIntegration = systemVariablesRS.getBoolean("enableNovelistSeriesIntegration");
+				this.deletionCommitInterval = systemVariablesRS.getInt("deletionCommitInterval");
+				this.waitAfterDeleteCommit = systemVariablesRS.getBoolean("waitAfterDeleteCommit");
 				if (fullReindex) {
 					this.processEmptyGroupedWorks = systemVariablesRS.getBoolean("processEmptyGroupedWorks");
 				}
@@ -605,8 +609,12 @@ public synchronized void deleteRecord(String permanentId) {
 			//With this commit, we get errors in the log "Previous SolrRequestInfo was not closed!"
 			//Allow auto commit functionality to handle this
 			totalRecordsHandled++;
-			if (totalRecordsHandled % 1000 == 0) {
-				this.commitChanges();
+			if (totalRecordsHandled % this.deletionCommitInterval == 0) {
+				if (this.waitAfterDeleteCommit) {
+					this.commitChangesWithWait();
+				}else{
+					this.commitChanges();
+				}
 			}
 
 			/*
@@ -670,6 +678,14 @@ public void commitChanges(){
 		}
 	}
 
+	public void commitChangesWithWait(){
+		try {
+			updateServer.commit(true, false, true);
+		}catch (Exception e) {
+			logEntry.incErrors("Error committing changes ", e);
+		}
+	}
+
 	/**
 	 * This is called from all the indexers, so we would like to prevent scheduled works from being processed multiple times.
 	 * Rather than getting a list of all the scheduled works, we will process up to max works to process by getting the oldest record

code/web/release_notes/24.07.03.MD

@@ -0,0 +1,7 @@
+## Aspen Discovery Updates
+### Index Updates
+- Add additional configuration options to System Variables to control how solr commits are done during record deletions. (*MDN*)
+
+## This release includes code contributions from
+- ByWater Solutions
+  - Mark Noble (MDN)

code/web/sys/DBMaintenance/version_updates/24.07.03.php

@@ -0,0 +1,28 @@
+<?php
+
+/** @noinspection PhpUnused */
+function getUpdates24_07_03(): array {
+	/** @noinspection SqlWithoutWhere */
+	return [
+		/*'name' => [
+			 'title' => '',
+			 'description' => '',
+			 'continueOnError' => false,
+			 'sql' => [
+				 ''
+			 ]
+		 ], //name*/
+
+		//mark - ByWater
+		'add_configuration_for_index_deletions' => [
+			'title' => 'Add configuration for index deletions',
+			'description' => 'Add additional configuration for how records are deleted from solr',
+			'continueOnError' => true,
+			'sql' => [
+				"ALTER TABLE system_variables ADD COLUMN deletionCommitInterval INT DEFAULT 1000",
+				"ALTER TABLE system_variables ADD COLUMN waitAfterDeleteCommit TINYINT DEFAULT 0",
+			]
+		], //add_configuration_for_index_deletions
+
+	];
+}

code/web/sys/SystemVariables.php

@@ -17,6 +17,8 @@ class SystemVariables extends DataObject {
 	public $useHtmlEditorRatherThanMarkdown;
 	public $storeRecordDetailsInSolr;
 	public $storeRecordDetailsInDatabase;
+	public $deletionCommitInterval;
+	public $waitAfterDeleteCommit;
 	public $indexVersion;
 	public $searchVersion;
 	public $enableNovelistSeriesIntegration;
@@ -198,6 +200,22 @@ static function getObjectStructure($context = ''): array {
 						'description' => 'Whether or not covers can be loaded from the 020z',
 						'default' => false,
 					],
+					'deletionCommitInterval' => [
+						'property' => 'deletionCommitInterval',
+						'type' => 'integer',
+						'label' => 'Deletion Commit Interval (# of records)',
+						'description' => 'Based on this setting, Aspen will call a solr commit after the specified number of records are marked for deletion',
+						'required' => true,
+						'default' => 1000,
+						'min' => 250,
+					],
+					'waitAfterDeleteCommit' => [
+						'property' => 'waitAfterDeleteCommit',
+						'type' => 'checkbox',
+						'label' => 'Wait after delete commit',
+						'description' => 'Whether or not to wait for Solr to finish processing the commit before deleting more records',
+						'default' => false,
+					]
 				],
 			],