Skip to content
This repository has been archived by the owner on Aug 30, 2022. It is now read-only.

Commit

Permalink
#150: added a index_version field to the conversation index that al…
Browse files Browse the repository at this point in the history 
…lows to query on startup if all documents in the index where indexed with the current version. If not a full re-build of the index is triggered. If not a partial update (via the last_sync field) is enabled. Changed the default for the `smarti.index.rebuildOnStartup` property to `false` as with the new functionality this is no longer required under normal operational circumstances.
  • Loading branch information
@westei
westei committed Dec 29, 2017
1 parent 0f22d23 commit 81f2602
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 37 deletions.
11 changes: 11 additions & 0 deletions ...on/src/main/java/io/redlink/smarti/query/conversation/ConversationIndexConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,18 @@ public class ConversationIndexConfiguration {

public final static String CONVERSATION_INDEX = "conversation";

/**
* The current conversation index version. Needs to be increased on schema.xml
* or software updates that do require a full re-index
*/
public static final int CONVERSATION_INDEX_VERSION = 1; //start with 1 after introducing this feature with #150

public static final String FIELD_ID = "id";
/**
* Field used to store the current {@link #CONVERSATION_INDEX_VERSION} so that
* updates that require a full re-index can be detected on startup
*/
public static final String FIELD_INDEX_VERSION = "index_version";
public static final String FIELD_TYPE = "type";
public static final String TYPE_MESSAGE = "message";
public static final String TYPE_CONVERSATION = "conversation";
Expand Down
66 changes: 29 additions & 37 deletions ...-conversation/src/main/java/io/redlink/smarti/query/conversation/ConversationIndexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,30 +17,7 @@

package io.redlink.smarti.query.conversation;

import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_CONTEXT;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_CONVERSATION_ID;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_DOMAIN;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_END_TIME;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_ENVIRONMENT;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_ID;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_MESSAGE;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_MESSAGES;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_MESSAGE_COUNT;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_MESSAGE_ID;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_MESSAGE_IDX;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_MODIFIED;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_OWNER;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_START_TIME;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_SYNC_DATE;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_TIME;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_TYPE;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_USER_ID;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_USER_NAME;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.FIELD_VOTE;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.TYPE_CONVERSATION;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.TYPE_MESSAGE;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.getEnvironmentField;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.getMetaField;
import static io.redlink.smarti.query.conversation.ConversationIndexConfiguration.*;

import java.io.IOException;
import java.time.Instant;
Expand Down Expand Up @@ -137,8 +114,8 @@ public class ConversationIndexer implements ConversytionSyncCallback {

private final ExecutorService indexerPool;

@Value("${smarti.index.rebuildOnStartup:true}")
private boolean rebuildOnStartup = true;
@Value("${smarti.index.rebuildOnStartup:false}")
private boolean rebuildOnStartup = false;

@Autowired
public ConversationIndexer(SolrCoreContainer solrServer, StoreService storeService){
Expand All @@ -163,28 +140,37 @@ protected void startup() {
indexTask = cloudSync == null ? null : new ConversationIndexTask(cloudSync);
if(indexTask != null){
log.info("initialize ConversationIndex after startup ...");
Date syncDate = null;
Date syncDate = null; //null triggers a full rebuild (default)
if(!rebuildOnStartup){
try (SolrClient solr = solrServer.getSolrClient(conversationCore)){
//read (1) FIELD_SYNC_DATE from index
//search for conversations indexed with an earlier version of the index
SolrQuery query = new SolrQuery("*:*");
query.addSort(FIELD_SYNC_DATE, ORDER.desc);
query.setFields(FIELD_SYNC_DATE);
query.setRows(1);
query.setStart(0);
QueryResponse result = solr.query(query);
if(result.getResults() != null && result.getResults().getNumFound() > 0){
syncDate = (Date)result.getResults().get(0).getFieldValue(FIELD_SYNC_DATE);
log.debug("set lastSync date to {}", syncDate);
query.addFilterQuery(String.format("!%s:%s",FIELD_INDEX_VERSION,CONVERSATION_INDEX_VERSION));
query.setRows(0); //we only need the count
if(solr.query(query).getResults().getNumFound() > 0){
log.info("conversation index contains documents indexed with an outdated version - full re-build required");
} else { //partial update possible. Search for the last sync date ...
query = new SolrQuery("*:*");
query.addSort(FIELD_SYNC_DATE, ORDER.desc);
query.setFields(FIELD_SYNC_DATE);
query.setRows(1);
query.setStart(0);
QueryResponse result = solr.query(query);
if(result.getResults() != null && result.getResults().getNumFound() > 0){
syncDate = (Date)result.getResults().get(0).getFieldValue(FIELD_SYNC_DATE);
log.info("Perform partial update of conversation index (lastSync date:{})", syncDate);
}
}
} catch (IOException | SolrServerException e) {
log.warn("Updating Conversation index on startup failed ({} - {})", e.getClass().getSimpleName(), e.getMessage());
log.debug("STACKTRACE:",e);
}
} else {
log.info("full re-build on startup required via configuration");
}
indexTask.setLastSync(syncDate);
indexerPool.execute(indexTask);
} else { //manual initialization
} else { //manual initialization (performs a full re-index to be up-to-date)
Iterators.partition(storeService.listConversationIDs().iterator(), 100).forEachRemaining(
ids -> {
ids.stream()
Expand Down Expand Up @@ -281,6 +267,9 @@ private SolrInputDocument toSolrInputDocument(Conversation conversation) {
final SolrInputDocument solrConversation = new SolrInputDocument();

solrConversation.setField(FIELD_ID, conversation.getId().toHexString());
//#150 index the current version of the index so that we can detect the need of a
//full re-index after a software update on startup
solrConversation.setField(FIELD_INDEX_VERSION, CONVERSATION_INDEX_VERSION);
solrConversation.setField(FIELD_TYPE, TYPE_CONVERSATION);
solrConversation.setField(FIELD_MODIFIED, conversation.getLastModified());

Expand Down Expand Up @@ -338,6 +327,9 @@ private SolrInputDocument toSolrInputDocument(Message message, int i, Conversati
solrMsg.setField(FIELD_CONVERSATION_ID, conversation.getId());
solrMsg.setField(FIELD_MESSAGE_ID, message.getId());
solrMsg.setField(FIELD_MESSAGE_IDX, i);
//#150 index the current version of the index so that we can detect the need of a
//full re-index after a software update on startup
solrMsg.setField(FIELD_INDEX_VERSION, CONVERSATION_INDEX_VERSION);
solrMsg.setField(FIELD_TYPE, TYPE_MESSAGE);
if (message.getUser() != null) {
solrMsg.setField(FIELD_USER_ID, message.getUser().getId());
Expand Down
2 changes: 2 additions & 0 deletions lib/query-conversation/src/main/resources/solr/core/conversation/conf/schema.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
<!-- generic -->
<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="true"/>
<field name="type" type="type" indexed="true" stored="true" multiValued="false" required="true"/>
<field name="index_version" type="int" indexed="true" stored="false" multiValued="false" required="true"/>
<field name="owner" type="string" indexed="true" stored="true" multiValued="false"/>
<field name="user_id" type="string" indexed="true" stored="true" multiValued="false"/>
<field name="user_name" type="string" indexed="true" stored="true" multiValued="false"/>
Expand Down Expand Up @@ -82,6 +83,7 @@
<defaultSearchField>text</defaultSearchField>

<fieldType name="string" class="solr.StrField" sortMissingLast="true" docValues="true" />
<fieldType name="int" class="solr.TrieIntField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="long" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="date" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
<fieldType name="text_iterms_de" class="solr.TextField" positionIncrementGap="100">
Expand Down

0 comments on commit 81f2602

Please sign in to comment.