Merge pull request #193 from diging/develop

prepare release

Conceptpower+Spring/pom.xml

@@ -9,7 +9,7 @@
 	<version>2.0-BUILD-SNAPSHOT</version>
 	<properties>
 		<java-version>1.8</java-version>
-		<org.springframework-version>4.3.7.RELEASE</org.springframework-version>
+		<org.springframework-version>4.3.18.RELEASE</org.springframework-version>
 		<org.aspectj-version>1.7.3</org.aspectj-version>
 		<org.slf4j-version>1.7.5</org.slf4j-version>
 		<tomcat.deploy.path>http://localhost:8080/manager/text</tomcat.deploy.path>
@@ -19,21 +19,21 @@
 		<db.files.path>/Users/jdamerow/Software/Conceptpower/db_files</db.files.path>
 		<email.username></email.username>
 		<email.password></email.password>
-		<email.server.port></email.server.port>
+		<email.server.port>8081</email.server.port>
 		<email.server.host></email.server.host>
 		<email.from></email.from>
 		<admin.username>admin</admin.username>
 		<admin.password>admin</admin.password>
 		<wordnet.path>/usr/local/WordNet-3.0</wordnet.path>
 		<spring-security-version>3.2.8.RELEASE</spring-security-version>
-		<lucene.index>/Users/karthikeyanmohan/Software/Conceptpower/indexFiles</lucene.index>
+		<lucene.index></lucene.index>
 		<pullrequest></pullrequest>
 		<debug_level>debug</debug_level>
 		<skip.integration.tests>false</skip.integration.tests>
 		<skip.unit.tests>false</skip.unit.tests>
-		<concept.list.url>https://www.dropbox.com/s/48gftjus29qyp2q/conceptLists.db?raw=1</concept.list.url>
-		<concept.types.url>https://www.dropbox.com/s/mla61f63fgfil3y/conceptTypes.db?raw=1</concept.types.url>
-		<users.url>https://www.dropbox.com/s/n0vgpc5ow1xko4v/users.db?raw=1</users.url>
+		<concept.list.url>https://www.dropbox.com/s/r82nymnt2247zz4/conceptLists.db?raw=1</concept.list.url>
+		<concept.types.url>https://www.dropbox.com/s/nwn15s84fzz37s5/conceptTypes.db?raw=1</concept.types.url>
+		<users.url>https://www.dropbox.com/s/p6h3974hmlouq0i/users.db?raw=1</users.url>
 	</properties>
 
 	<repositories>

Conceptpower+Spring/src/main/java/edu/asu/conceptpower/app/core/impl/ConceptManager.java

@@ -31,6 +31,7 @@
 import edu.asu.conceptpower.app.wordnet.WordNetManager;
 import edu.asu.conceptpower.core.ConceptEntry;
 import edu.asu.conceptpower.core.ConceptList;
+import edu.asu.conceptpower.rest.SearchParamters;
 import edu.asu.conceptpower.servlet.core.ChangeEvent;
 import edu.asu.conceptpower.servlet.core.ChangeEvent.ChangeEventTypes;
 
@@ -72,7 +73,14 @@ public class ConceptManager implements IConceptManager {
     @Override
     public ConceptEntry getConceptEntry(String id) {
 
-        ConceptEntry[] entries = client.getEntriesByFieldContains(SearchFieldNames.MERGED_IDS, id);
+        Map<String, String> fieldMap = new HashMap<>();
+        fieldMap.put(SearchFieldNames.MERGED_IDS, id);
+        ConceptEntry[] entries;
+        try {
+            entries = indexService.searchForConcepts(fieldMap, SearchParamters.OP_OR);
+        } catch (IllegalAccessException | LuceneException | IndexerRunningException e) {
+            return null;
+        }
         if (entries != null && entries.length > 0) {
             fillConceptEntry(entries[0]);
             alternativeIdService.addAlternativeIds(id, entries[0]);

Conceptpower+Spring/src/main/java/edu/asu/conceptpower/app/lucene/impl/LuceneUtility.java

@@ -2,6 +2,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.StringReader;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.FileSystems;
@@ -22,10 +23,19 @@
 import javax.annotation.PreDestroy;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.custom.CustomAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
 import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.standard.StandardTokenizerFactory;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedDocValuesField;
@@ -36,6 +46,7 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.classic.QueryParser;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
@@ -53,7 +64,9 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PagedBytes.Reader;
 import org.apache.lucene.util.QueryBuilder;
+import org.apache.lucene.util.Version;
 import org.jsoup.Jsoup;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,7 +75,6 @@
 import org.springframework.context.annotation.PropertySource;
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Component;
-
 import edu.asu.conceptpower.app.constants.LuceneFieldNames;
 import edu.asu.conceptpower.app.constants.SearchFieldNames;
 import edu.asu.conceptpower.app.db4o.IConceptDBManager;
@@ -124,6 +136,7 @@ public class LuceneUtility implements ILuceneUtility {
 
     private String lucenePath;
 
+
     private int numberOfResults;
 
     private IndexWriter writer = null;
@@ -132,19 +145,23 @@ public class LuceneUtility implements ILuceneUtility {
     private Directory index;
     private Path relativePath = null;
     private IndexSearcher searcher = null;
-
+    private Analyzer customAnalyzer = null;
+
     /**
      * 
      * @throws LuceneException
      */
     @PostConstruct
-    public void init() throws LuceneException {
+    public void init() throws LuceneException, IOException {
+        customAnalyzer = CustomAnalyzer.builder().withTokenizer("keyword").addTokenFilter("asciifolding").addTokenFilter("worddelimiter").
+                addTokenFilter("lowercase").build();
         lucenePath = env.getProperty("lucenePath");
         numberOfResults = Integer.parseInt(env.getProperty("numberOfLuceneResults"));
         try {
             relativePath = FileSystems.getDefault().getPath(lucenePath, "index");
+
             index = FSDirectory.open(relativePath);
-            configWhiteSpace = new IndexWriterConfig(standardAnalyzer);
+            configWhiteSpace = new IndexWriterConfig(customAnalyzer);
             writer = new IndexWriter(index, configWhiteSpace);
             reader = DirectoryReader.open(writer, true);
             searcher = new IndexSearcher(reader);
@@ -256,7 +273,7 @@ private ConceptEntry getConceptFromDocument(Document d) throws IllegalAccessExce
             LuceneField luceneFieldAnnotation = field.getAnnotation(LuceneField.class);
             field.setAccessible(true);
             if (luceneFieldAnnotation != null && d.get(luceneFieldAnnotation.lucenefieldName()) != null)
-                if (!luceneFieldAnnotation.isMultiple()) {
+                if (luceneFieldAnnotation.isMultiple()) {
                     IndexableField[] indexableFields = d.getFields(luceneFieldAnnotation.lucenefieldName() + LuceneFieldNames.NOT_LOWERCASED);
                     if (indexableFields == null || indexableFields.length == 0) {
                         indexableFields = d.getFields(luceneFieldAnnotation.lucenefieldName());
@@ -497,7 +514,7 @@ public ConceptEntry[] queryIndex(Map<String, String> fieldMap, String operator,
         if (operator == null || operator.equalsIgnoreCase(SearchParamters.OP_AND)) {
             occur = BooleanClause.Occur.MUST;
         }
-
+        
         java.lang.reflect.Field[] fields = ConceptEntry.class.getDeclaredFields();
 
         for (java.lang.reflect.Field field : fields) {
@@ -517,9 +534,8 @@ public ConceptEntry[] queryIndex(Map<String, String> fieldMap, String operator,
 
         }
 
-        PerFieldAnalyzerWrapper perFieldAnalyzerWrapper = new PerFieldAnalyzerWrapper(standardAnalyzer,
-                analyzerPerField);
-
+
+        PerFieldAnalyzerWrapper perFieldAnalyzerWrapper = new PerFieldAnalyzerWrapper(customAnalyzer, analyzerPerField);
         QueryBuilder qBuild = new QueryBuilder(perFieldAnalyzerWrapper);
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
 
@@ -579,32 +595,25 @@ public ConceptEntry[] queryIndex(Map<String, String> fieldMap, String operator,
                 ConceptEntry entry = getConceptFromDocument(d);
                 concepts.add(entry);
             }
+            return concepts.toArray(new ConceptEntry[concepts.size()]);
         }
 
         catch (IOException ex) {
             throw new LuceneException("Issues in querying lucene index. Please retry", ex);
         }
-        logger.debug("Number of concepts retrieved from lucene = " + concepts.size());
-        return concepts.toArray(new ConceptEntry[concepts.size()]);
-
     }
 
-    private void buildQuery(BooleanClause.Occur occur, PerFieldAnalyzerWrapper perFieldAnalyzerWrapper,
-            QueryBuilder qBuild, BooleanQuery.Builder builder, LuceneField luceneFieldAnnotation, String searchString) {
+    private void buildQuery(BooleanClause.Occur occur, PerFieldAnalyzerWrapper perFieldAnalyzerWrapper,QueryBuilder qBuild, BooleanQuery.Builder builder, LuceneField luceneFieldAnnotation, String searchString) {
         if (luceneFieldAnnotation.isTokenized()) {
             BooleanQuery.Builder tokenizedQueryBuilder = new BooleanQuery.Builder();
-            buildTokenizedOrWildCardQuery(luceneFieldAnnotation, searchString, tokenizedQueryBuilder);
+            buildTokenizedOrWildCardQuery(luceneFieldAnnotation, searchString, qBuild, tokenizedQueryBuilder);
 
             if (luceneFieldAnnotation.isShortPhraseSearchable()) {
                 BooleanQuery.Builder rootQueryBuilder = new BooleanQuery.Builder();
                 rootQueryBuilder.add(tokenizedQueryBuilder.build(), Occur.SHOULD);
                 // Short word searching
                 BooleanQuery.Builder shortWordSearchQueryBuilder = new BooleanQuery.Builder();
-                shortWordSearchQueryBuilder.add(
-                        new PhraseQuery(luceneFieldAnnotation.lucenefieldName() + LuceneFieldNames.UNTOKENIZED_SUFFIX,
-                                searchString),
-                        Occur.SHOULD);
-
+                shortWordSearchQueryBuilder.add(new PhraseQuery(luceneFieldAnnotation.lucenefieldName() + LuceneFieldNames.UNTOKENIZED_SUFFIX, searchString), Occur.SHOULD);
                 rootQueryBuilder.add(shortWordSearchQueryBuilder.build(), Occur.SHOULD);
                 tokenizedQueryBuilder = rootQueryBuilder;
             }
@@ -613,24 +622,25 @@ private void buildQuery(BooleanClause.Occur occur, PerFieldAnalyzerWrapper perFi
         } else {
             if (luceneFieldAnnotation.isWildCardSearchEnabled()) {
                 createWildCardSearchQuery(luceneFieldAnnotation, searchString, builder, occur);
-            } else {
-                builder.add(new BooleanClause(
-                        new TermQuery(new Term(luceneFieldAnnotation.lucenefieldName(), searchString)), occur));
             }
-        }
+            builder.add(new BooleanClause((qBuild.createPhraseQuery(luceneFieldAnnotation.lucenefieldName(), searchString)), occur));
+        }        
     }
 
-    private void buildTokenizedOrWildCardQuery(LuceneField luceneFieldAnnotation, String searchString,
-            BooleanQuery.Builder tokenizedQueryBuilder) {
-        for (String searchValue : searchString.split(" ")) {
+    private void buildTokenizedOrWildCardQuery(LuceneField luceneFieldAnnotation, String searchString, QueryBuilder qBuild,
+            BooleanQuery.Builder tokenizedQueryBuilder) {       
             if (luceneFieldAnnotation.isWildCardSearchEnabled()) {
-                createWildCardSearchQuery(luceneFieldAnnotation, searchValue, tokenizedQueryBuilder, Occur.MUST);
+                BooleanQuery.Builder analyzedBuilder = new BooleanQuery.Builder();             
+                createWildCardSearchQuery(luceneFieldAnnotation, searchString, analyzedBuilder, Occur.SHOULD);
+                analyzedBuilder.add(new BooleanClause(
+                        (qBuild.createPhraseQuery(luceneFieldAnnotation.lucenefieldName(), searchString)), Occur.SHOULD));
+                tokenizedQueryBuilder.add(analyzedBuilder.build(), Occur.MUST);
             } else {
-                tokenizedQueryBuilder.add(new PhraseQuery(luceneFieldAnnotation.lucenefieldName(), searchValue),
+                tokenizedQueryBuilder.add(qBuild.createPhraseQuery(luceneFieldAnnotation.lucenefieldName(), searchString),
                         Occur.MUST);
             }
         }
-    }
+
 
     /**
      * This method adds the wild card query to the query builder when the

Conceptpower+Spring/src/main/java/edu/asu/conceptpower/app/wordnet/WordNetManager.java

@@ -68,7 +68,7 @@ public ConceptEntry getConcept(String id) {
         try {
             wordId = WordID.parseWordID(id);
         } catch (IllegalArgumentException e) {
-            logger.error("Could not find id '" + id + "' in WordNet.", e);
+            logger.warn("Could not find id '" + id + "' in WordNet.", e);
             return null;
         }
 

Conceptpower+Spring/src/main/java/edu/asu/conceptpower/rest/Concepts.java

@@ -7,6 +7,7 @@
 import java.util.List;
 import java.util.ListIterator;
 
+import org.apache.commons.lang3.StringEscapeUtils;
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
@@ -167,7 +168,7 @@ public class Concepts {
         HttpHeaders responseHeaders = new HttpHeaders();
         responseHeaders.add("Content-Type", MediaType.APPLICATION_JSON_VALUE + "; charset=utf-8");
 
-        return new ResponseEntity<String>(jsonObject.toJSONString(), responseHeaders,
+        return new ResponseEntity<String>(StringEscapeUtils.unescapeJson(jsonObject.toJSONString()), responseHeaders,
                 HttpStatus.OK);
     }
 
@@ -242,7 +243,7 @@ public ResponseEntity<String> addConcepts(@RequestBody String body, Principal pr
         HttpHeaders responseHeaders = new HttpHeaders();
         responseHeaders.add("Content-Type", MediaType.APPLICATION_JSON_VALUE + "; charset=utf-8");
 
-        return new ResponseEntity<String>(responseArray.toJSONString(), responseHeaders,
+        return new ResponseEntity<String>(StringEscapeUtils.unescapeJson(responseArray.toJSONString()), responseHeaders,
                 HttpStatus.OK);
     }
 

Conceptpower+Spring/src/main/java/edu/asu/conceptpower/web/ConceptSearchController.java

@@ -87,6 +87,7 @@ public String search(HttpServletRequest req, ModelMap model, @RequestParam(defau
         if (results.hasErrors()) {
             return "conceptsearch";
         }
+        conceptSearchBean.setWord(conceptSearchBean.getWord().trim());
 
         if (conceptIdsToMerge != null) {
             model.addAttribute("conceptIdsToMerge", conceptIdsToMerge);

Conceptpower+Spring/src/main/java/edu/asu/conceptpower/web/ConceptWrapperAddController.java

@@ -27,6 +27,7 @@
 
 import edu.asu.conceptpower.app.core.IConceptManager;
 import edu.asu.conceptpower.app.core.IIndexService;
+import edu.asu.conceptpower.app.core.impl.ConceptManager;
 import edu.asu.conceptpower.app.exceptions.DictionaryDoesNotExistException;
 import edu.asu.conceptpower.app.exceptions.DictionaryModifyException;
 import edu.asu.conceptpower.app.exceptions.IndexerRunningException;
@@ -82,10 +83,23 @@ private void initBinder(WebDataBinder binder) {
      *         page
      */
     @RequestMapping(value = "auth/conceptlist/addconceptwrapper")
-    public String prepareConceptWrapperAdd(@ModelAttribute ConceptWrapperAddBean conceptWrapperAddBean,
-            HttpServletRequest req, ModelMap model) {
+    public String prepareConceptWrapperAdd(@RequestParam(value = "wrapperId", required = false) String wrapperId,
+            HttpServletRequest req, ModelMap model){
         model.addAttribute("types", conceptWrapperService.fetchAllConceptTypes());
         model.addAttribute("lists", conceptWrapperService.fetchAllConceptLists());
+        ConceptEntry entry = null;
+        try {
+            entry = conceptManager.getWordnetConceptEntry(wrapperId);
+        } catch (LuceneException ex) {
+            logger.error("Error while fetching concepts based on concept id", ex);
+            return "/auth/conceptlist/addconceptwrapper";
+        }
+        ConceptWrapperAddBean conceptWrapperAddBean = new ConceptWrapperAddBean();
+        conceptWrapperAddBean.setDescription(entry.getDescription());
+        conceptWrapperAddBean.setWord(entry.getWord());
+        conceptWrapperAddBean.setSelectedConceptList(entry.getConceptList());
+        conceptWrapperAddBean.setPos(entry.getPos());
+        conceptWrapperAddBean.setWrapperids(wrapperId);
         model.addAttribute("conceptWrapperAddBean", conceptWrapperAddBean);
         return "/auth/conceptlist/addconceptwrapper";
     }

Conceptpower+Spring/src/main/java/edu/asu/conceptpower/web/backing/ConceptWrapperAddBean.java

@@ -10,6 +10,7 @@ public class ConceptWrapperAddBean {
     private String equals;
     private String similar;
     private String wrapperids;
+    private String pos;
 
     public String getWord() {
         return word;
@@ -75,4 +76,12 @@ public void setSynonymids(String synonymids) {
         this.synonymids = synonymids;
     }
 
+    public String getPos() {
+        return pos;
+    }
+
+    public void setPos(String pos) {
+        this.pos = pos;
+    }
+
 }

Conceptpower+Spring/src/main/webapp/WEB-INF/views/home.jsp

@@ -205,8 +205,8 @@ function prepareMergeConcept(conceptId) {
     }
 }
 
-var createWrapper = function(word, pos, conceptList, description, conceptType, wrapperId) {
-  window.location = '${pageContext.servletContext.contextPath}/auth/conceptlist/addconceptwrapper?word=' + word + '&selectedConceptList=' + conceptList + '&description=' + description + '&selectedType=' + conceptType + '&wrapperids=' + wrapperId;
+function createWrapper(wrapperId) {
+  window.location = '${pageContext.servletContext.contextPath}/auth/conceptlist/addconceptwrapper?wrapperId=' + wrapperId;
 }
 
 </script>
@@ -324,7 +324,7 @@ var createWrapper = function(word, pos, conceptList, description, conceptType, w
                   </a>
                 </c:when>
                 <c:otherwise>
-                  <a href="#" onclick="createWrapper('${concept.entry.word}', '${concept.entry.pos}', '${concept.entry.conceptList}', '${concept.entry.description}', '${concept.entry.typeId}', '${concept.entry.id}');">
+                  <a href="#" onclick="createWrapper('${concept.entry.id}');">
                     <i title='Create wrapper' class="fa fa-pencil-square-o"></i>
                   </a>
                 </c:otherwise>