From bc7993fb17d221320fcd744ed09377fece9ed979 Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Mon, 16 Oct 2017 13:23:32 -0700 Subject: [PATCH 01/16] [VGNWB-207]Added way to filter identities for duplicates and added a new concept to the template when a identity is found. --- .../static/annotations/js/annotators/text.js | 60 +++++++++++++++++-- annotations/views/rest_views.py | 53 +++++++++++++++- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/annotations/static/annotations/js/annotators/text.js b/annotations/static/annotations/js/annotators/text.js index fde127d6..41b41788 100644 --- a/annotations/static/annotations/js/annotators/text.js +++ b/annotations/static/annotations/js/annotators/text.js @@ -6,16 +6,67 @@ var ConceptListItem = { props: ['concept'], - template: `
+ template: `
+ `, +
+ +
+
+ {{ concept.label }} ({{ z }}) + +
+
{{ concept.description }}
+
+
`, + data: function() { + return { + identi: false, + identi2: false, + ide: this.concept, + z :"", + concept2: "" + } + }, methods: { select: function() { this.$emit('selectconcept', this.concept); }, + ident: function(identi, concept) { + if(typeof this.concept.identities[0].concepts[0] !== 'undefined'){ + this.identi = true; + } + }, + ident2: function(identi2, concept,z, concept2) { + if(typeof this.concept.identities[0].concepts[1] !== 'undefined'){ + this.identi2 = true; + var regex = /digitalhps/g; + var str = this.concept.identities[0].concepts[1]; + var m; + + while ((m = regex.exec(str)) !== null) { + // This is necessary to avoid infinite loops with zero-width matches + if (m.index === regex.lastIndex) { + regex.lastIndex++; + } + + // The result can be accessed through the `m`-variable. + m.forEach((match, groupIndex) => { + this.concept2 = str; + this.z = "CHPS"; + console.log("match"); + console.log(this.concept2); + }); + } + } + } + }, + created() { + this.ident(), + this.ident2() } } @@ -37,7 +88,7 @@ var ConceptSearch = {
- + @@ -101,7 +152,7 @@ var ConceptSearch = { self.error = true; self.searching = false; }); - } + }, }, components: { @@ -1112,6 +1163,7 @@ Appellator = new Vue({ self.updateSwimRef(); self.handleScroll(); } + }, destroyed () { window.removeEventListener('scroll', this.handleScroll); diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index da4cc2f5..10ea5b89 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -24,6 +24,8 @@ import uuid +import unicodedata + import goat goat.GOAT = settings.GOAT goat.GOAT_APP_TOKEN = settings.GOAT_APP_TOKEN @@ -33,6 +35,7 @@ logger = logging.getLogger(__name__) logger.setLevel(settings.LOGLEVEL) +import json # http://stackoverflow.com/questions/17769814/django-rest-framework-model-serializers-read-nested-write-flat @@ -480,12 +483,18 @@ def create(self, request, *args, **kwargs): @list_route() def search(self, request, **kwargs): + print "starting" q = request.GET.get('search', None) if not q: return Response({'results': []}) pos = request.GET.get('pos', None) + concepts = goat.Concept.search(q=q, pos=pos, limit=50) + + + + def _relabel(datum): _fields = { 'name': 'label', @@ -493,8 +502,48 @@ def _relabel(datum): 'identifier': 'uri' } - return {_fields.get(k, k): v for k, v in datum.iteritems()} - return Response({'results': map(_relabel, [c.data for c in concepts])}) + return {_fields.get(k, k): v for k, v in datum.iteritems() } + results = map(_relabel, [c.data for c in concepts]) + + i = 0 + mat = [] + di = len(results) + con1 = [] + con2 = [] + + + print results + while (i != di): + test = results[i]["identities"] + print len(results[i]["identities"]) + if results[i]["identities"]: + z = 1 + while (z != len(results[i]["identities"])): + con1 = results[i]["identities"][0]["concepts"] + #print "this is con1: %s", con1 + '''if z != len(results[i]["identities"]): + z = z + 1 + else: + break''' + if z != len(results[i]["identities"]): + con2 = results[i]["identities"][z]["concepts"] + #print "this is con2: %s", con2 + if set(con1) == set(con2): + print "Should delete this: ", results[i]["identities"][z] + results[i]["identities"].pop(z) + + + + + else: + break + + #print results[i]["identities"] + print results[i]["identities"] + i = i + 1 + print results + return Response({'results': results}) + def get_queryset(self, *args, **kwargs): """ From 65ca63f0d3c8db635aa8497903aa4401625aae17 Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Tue, 7 Nov 2017 17:11:19 -0700 Subject: [PATCH 02/16] [VGNWB-207] refactored function to removed duplicate idenetities and added xml parser to get indentity information. --- .../static/annotations/js/annotators/text.js | 50 +++++------ annotations/views/rest_views.py | 84 ++++++++++--------- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/annotations/static/annotations/js/annotators/text.js b/annotations/static/annotations/js/annotators/text.js index 41b41788..bfa40755 100644 --- a/annotations/static/annotations/js/annotators/text.js +++ b/annotations/static/annotations/js/annotators/text.js @@ -14,60 +14,52 @@ var ConceptListItem = {
{{ concept.description }}
-
+
- {{ concept.label }} ({{ z }}) + {{ concept_name}} ({{ concept_auth }})
-
{{ concept.description }}
+
{{ concept_desc }}
`, data: function() { return { - identi: false, identi2: false, ide: this.concept, - z :"", - concept2: "" } }, methods: { select: function() { this.$emit('selectconcept', this.concept); }, - ident: function(identi, concept) { - if(typeof this.concept.identities[0].concepts[0] !== 'undefined'){ - this.identi = true; - } - }, - ident2: function(identi2, concept,z, concept2) { - if(typeof this.concept.identities[0].concepts[1] !== 'undefined'){ + filter: function(identi2, concept) { + if(typeof this.concept.identities[0].concepts[0] !== 'undefined'){ this.identi2 = true; var regex = /digitalhps/g; - var str = this.concept.identities[0].concepts[1]; - var m; + var string_to_match = this.concept.identities[0].concepts[0]; + var match; - while ((m = regex.exec(str)) !== null) { + while ((match = regex.exec(string_to_match)) !== null) { // This is necessary to avoid infinite loops with zero-width matches - if (m.index === regex.lastIndex) { + if (match.index === regex.lastIndex) { regex.lastIndex++; } - // The result can be accessed through the `m`-variable. - m.forEach((match, groupIndex) => { - this.concept2 = str; - this.z = "CHPS"; - console.log("match"); - console.log(this.concept2); + // The result can be accessed through the `match`-variable. + match.forEach((match, groupIndex) => { + this.concept2 = string_to_match; + this.concept_auth = "CHPS"; + this.concept_name = this.concept.identities[0].concept_name; + this.concept_desc = this.concept.identities[0].concept_desc; + this.concept_uri = this.identities[0].concept_uri; }); } } - } + } + }, created() { - this.ident(), - this.ident2() - + this.filter() } } @@ -88,7 +80,7 @@ var ConceptSearch = {
- + @@ -152,7 +144,7 @@ var ConceptSearch = { self.error = true; self.searching = false; }); - }, + } }, components: { diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index 10ea5b89..9c477ef1 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -24,7 +24,8 @@ import uuid -import unicodedata +import xml.etree.ElementTree as ET +import urllib2 import goat goat.GOAT = settings.GOAT @@ -35,7 +36,6 @@ logger = logging.getLogger(__name__) logger.setLevel(settings.LOGLEVEL) -import json # http://stackoverflow.com/questions/17769814/django-rest-framework-model-serializers-read-nested-write-flat @@ -483,17 +483,17 @@ def create(self, request, *args, **kwargs): @list_route() def search(self, request, **kwargs): - print "starting" + print "**************tttttt" q = request.GET.get('search', None) - if not q: - return Response({'results': []}) - pos = request.GET.get('pos', None) - concepts = goat.Concept.search(q=q, pos=pos, limit=50) + if not q: + return Response({'results': []}) + pos = request.GET.get('pos', None) + concepts = goat.Concept.search(q=q, pos=pos, limit=50) def _relabel(datum): _fields = { @@ -505,43 +505,51 @@ def _relabel(datum): return {_fields.get(k, k): v for k, v in datum.iteritems() } results = map(_relabel, [c.data for c in concepts]) - i = 0 - mat = [] - di = len(results) - con1 = [] - con2 = [] - - - print results - while (i != di): - test = results[i]["identities"] - print len(results[i]["identities"]) - if results[i]["identities"]: - z = 1 - while (z != len(results[i]["identities"])): - con1 = results[i]["identities"][0]["concepts"] - #print "this is con1: %s", con1 - '''if z != len(results[i]["identities"]): - z = z + 1 + + for result in results: + length = len(result["identities"]) + if length != 0: + length = length - 1 + identities = [] + identities.append(result["identities"][0]) + while (length != 0): + if (length != 0): + concept1 = result["identities"][length]["concepts"] + if (length != 0): + length = length - 1 + else: + break + concept2 = result["identities"][length]["concepts"] + if set(concept1) != set(concept2): + identities.append(result["identities"][length]) else: - break''' - if z != len(results[i]["identities"]): - con2 = results[i]["identities"][z]["concepts"] - #print "this is con2: %s", con2 - if set(con1) == set(con2): - print "Should delete this: ", results[i]["identities"][z] - results[i]["identities"].pop(z) + break + result["identities"] = identities + concepts = result["identities"][0]["concepts"] + uri = result["uri"] + if uri in concepts: concepts.remove(uri) + concept_copy = result["identities"][0] + for concept in concepts: + url = concept + data = urllib2.urlopen(url) + + tree = ET.parse(data) + root = tree.getroot() + + namespace = {'hps': 'http://www.digitalhps.org/'} + + for entry in root.findall('hps:conceptEntry', namespace): + description1 = entry.find('hps:description', namespace) + name1 = entry.find('hps:lemma', namespace) + concept_description = description1.text + concept_name = name1.text + concept_parsed = {'concept_name': concept_name, 'concept_desc':concept_description, 'concept_uri':concept} + concept_copy.update(concept_parsed) - else: - break - #print results[i]["identities"] - print results[i]["identities"] - i = i + 1 - print results return Response({'results': results}) From 6a28a3c6dd49ab8ace4678ef8478b8226808fccc Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Mon, 13 Nov 2017 15:29:09 -0700 Subject: [PATCH 03/16] [VGNWB-207] Removed regex from template and added it to view, view now filters dplicate concepts and fetchs data for non-duplicate concepts in the form of xml. The template can now add up to three extra concepts to list for each concept. --- .../static/annotations/js/annotators/text.js | 86 +++++++++++++------ annotations/views/rest_views.py | 83 +++++++++++++----- 2 files changed, 124 insertions(+), 45 deletions(-) diff --git a/annotations/static/annotations/js/annotators/text.js b/annotations/static/annotations/js/annotators/text.js index bfa40755..949cc8cb 100644 --- a/annotations/static/annotations/js/annotators/text.js +++ b/annotations/static/annotations/js/annotators/text.js @@ -14,17 +14,35 @@ var ConceptListItem = {
{{ concept.description }}
-
+
{{ concept_name}} ({{ concept_auth }})
-
{{ concept_desc }}
+
{{ concept_desc }}
+ +
+
+ {{ concept2_name }} ({{ concept2_auth }}) + +
+
{{ concept2_desc }}
+
+ +
+
+ {{ concept3_name }} ({{ concept3_auth }}) + +
+
{{ concept3_desc }}
+
`, data: function() { return { - identi2: false, + concept1: false, + concept2: false, + concept3: false, ide: this.concept, } }, @@ -32,34 +50,54 @@ var ConceptListItem = { select: function() { this.$emit('selectconcept', this.concept); }, - filter: function(identi2, concept) { - if(typeof this.concept.identities[0].concepts[0] !== 'undefined'){ - this.identi2 = true; - var regex = /digitalhps/g; - var string_to_match = this.concept.identities[0].concepts[0]; - var match; + filter: function(concept1, concept) { + if(this.concept.identities[0].concepts[0] !== undefined){ + this.concept1 = true; + if(this.concept.identities[0].concept0 !== undefined) { - while ((match = regex.exec(string_to_match)) !== null) { - // This is necessary to avoid infinite loops with zero-width matches - if (match.index === regex.lastIndex) { - regex.lastIndex++; - } + this.concept_name = this.concept.identities[0].concept0[0]; + this.concept_desc = this.concept.identities[0].concept0[1]; + this.concept_uri = this.concept.identities[0].concept0[2]; + this.concept_auth = this.concept.identities[0].concept0[3]; + + } - // The result can be accessed through the `match`-variable. - match.forEach((match, groupIndex) => { - this.concept2 = string_to_match; - this.concept_auth = "CHPS"; - this.concept_name = this.concept.identities[0].concept_name; - this.concept_desc = this.concept.identities[0].concept_desc; - this.concept_uri = this.identities[0].concept_uri; - }); - } } + }, + filter2: function(concept2, concept) { + if(this.concept.identities[0].concepts[1] !== undefined){ + this.concept2 = true; + if(this.concept.identities[0].concept1 !== undefined) { + + this.concept2_name = this.concept.identities[0].concept1[0]; + this.concept2_desc = this.concept.identities[0].concept1[1]; + this.concept2_uri = this.concept.identities[0].concept1[2]; + this.concept2_auth = this.concept.identities[0].concept1[3]; + + } + + } + }, + filter3: function(concept3, concept) { + if(this.concept.identities[0].concepts[2] !== undefined){ + this.concept3 = true; + if(this.concept.identities[0].concept2 !== undefined) { + + this.concept3_name = this.concept.identities[0].concept2[0]; + this.concept3_desc = this.concept.identities[0].concept2[1]; + this.concept3_uri = this.concept.identities[0].concept2[2]; + this.concept3_auth = this.concept.identities[0].concept2[3]; + + } + + } } }, created() { - this.filter() + this.filter(), + this.filter2(), + this.filter3() } } diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index 9c477ef1..2281e7b5 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -26,6 +26,9 @@ import xml.etree.ElementTree as ET import urllib2 +import re +from lxml import etree as e + import goat goat.GOAT = settings.GOAT @@ -483,12 +486,8 @@ def create(self, request, *args, **kwargs): @list_route() def search(self, request, **kwargs): - print "**************tttttt" q = request.GET.get('search', None) - - - if not q: return Response({'results': []}) pos = request.GET.get('pos', None) @@ -505,8 +504,9 @@ def _relabel(datum): return {_fields.get(k, k): v for k, v in datum.iteritems() } results = map(_relabel, [c.data for c in concepts]) - for result in results: + # Filter results to remove dupilcates and any non-duplicates to concepts + new_concepts = {} length = len(result["identities"]) if length != 0: length = length - 1 @@ -524,30 +524,71 @@ def _relabel(datum): identities.append(result["identities"][length]) else: break + result["identities"] = identities concepts = result["identities"][0]["concepts"] uri = result["uri"] if uri in concepts: concepts.remove(uri) + i = 0 # used to generate concept name + for concept in concepts: + ''' go through all the concepts and parse xml data for each concept + then append info to list and then append list to dictionary so that + list can be referenced as con0, con1, etc + ''' + + hps = re.search( r'www.digitalhps.org', concept, re.M|re.I) + viaf = re.search( r'viaf.org', concept, re.M|re.I) + if hps: + url = concept + data = urllib2.urlopen(url) + + tree = ET.parse(data) + root = tree.getroot() + + namespace = {'hps': 'http://www.digitalhps.org/'} + + concept_list = [] + for entry in root.findall('hps:conceptEntry', namespace): + description1 = entry.find('hps:description', namespace) + name1 = entry.find('hps:lemma', namespace) + concept_description = description1.text + concept_name = name1.text + + + concept_list.append(concept_name) + concept_list.append(concept_description) + concept_list.append(concept) + concept_list.append("Concept Power") + new_concepts['concept' + str(i)] = concept_list + i = i + 1 + + elif viaf: + url = concept + '/viaf.xml' # have to append /viaf.xml to viaf url's in order to access xml + data = urllib2.urlopen(url) + tree = e.parse(data) + root = tree.getroot() + + namespace = {'foaf': 'http://xmlns.com/foaf/0.1/', 'ns1': 'http://viaf.org/viaf/terms#', 'owl': 'http://www.w3.org/2002/07/owl#', 'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', 'void': 'http://rdfs.org/ns/void#'} + + concept_list = [] + find = e.XPath("//ns1:text", namespaces = namespace) + concept_name = find(root)[0].text + concept_description = find(root)[0].text + + concept_list.append(concept_name) + concept_list.append(concept_description) + concept_list.append(concept) + concept_list.append("VIAF") + new_concepts['concept' + str(i)] = concept_list + i = i + 1 + else: + print "Nothing found!!" - concept_copy = result["identities"][0] - - for concept in concepts: - url = concept - data = urllib2.urlopen(url) - - tree = ET.parse(data) - root = tree.getroot() - namespace = {'hps': 'http://www.digitalhps.org/'} + result["identities"][0].update(new_concepts) + print result["identities"][0] - for entry in root.findall('hps:conceptEntry', namespace): - description1 = entry.find('hps:description', namespace) - name1 = entry.find('hps:lemma', namespace) - concept_description = description1.text - concept_name = name1.text - concept_parsed = {'concept_name': concept_name, 'concept_desc':concept_description, 'concept_uri':concept} - concept_copy.update(concept_parsed) return Response({'results': results}) From 89cefc2e95dd1bf8895f9f37add62a0c492a80c6 Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Tue, 14 Nov 2017 13:53:01 -0700 Subject: [PATCH 04/16] [VGNWB-207] removed print statement and fixed spaceing --- annotations/views/rest_views.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index 2281e7b5..b2d89b9b 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -29,7 +29,6 @@ import re from lxml import etree as e - import goat goat.GOAT = settings.GOAT goat.GOAT_APP_TOKEN = settings.GOAT_APP_TOKEN @@ -585,12 +584,7 @@ def _relabel(datum): else: print "Nothing found!!" - result["identities"][0].update(new_concepts) - print result["identities"][0] - - - return Response({'results': results}) From a4cb327ba5e38faa9c7c99e650809aded5024990 Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Tue, 28 Nov 2017 15:32:16 -0700 Subject: [PATCH 05/16] [VGNWB-207] updated jean bacon main loop. --- .../static/annotations/js/annotators/text.js | 41 ++++++-------- annotations/views/rest_views.py | 54 +++++++++---------- 2 files changed, 40 insertions(+), 55 deletions(-) diff --git a/annotations/static/annotations/js/annotators/text.js b/annotations/static/annotations/js/annotators/text.js index 949cc8cb..0c37a6ce 100644 --- a/annotations/static/annotations/js/annotators/text.js +++ b/annotations/static/annotations/js/annotators/text.js @@ -51,46 +51,35 @@ var ConceptListItem = { this.$emit('selectconcept', this.concept); }, filter: function(concept1, concept) { - if(this.concept.identities[0].concepts[0] !== undefined){ + if(this.concept.identities[1].concept0[0] !== undefined){ this.concept1 = true; - if(this.concept.identities[0].concept0 !== undefined) { - - this.concept_name = this.concept.identities[0].concept0[0]; - this.concept_desc = this.concept.identities[0].concept0[1]; - this.concept_uri = this.concept.identities[0].concept0[2]; - this.concept_auth = this.concept.identities[0].concept0[3]; - - } + this.concept_name = this.concept.identities[1].concept0[0]; + this.concept_desc = this.concept.identities[1].concept0[1]; + this.concept_uri = this.concept.identities[1].concept0[2]; + this.concept_auth = this.concept.identities[1].concept0[3]; } }, filter2: function(concept2, concept) { - if(this.concept.identities[0].concepts[1] !== undefined){ + if( this.concept.identities[1].concept1[0] !== undefined){ this.concept2 = true; - if(this.concept.identities[0].concept1 !== undefined) { - - this.concept2_name = this.concept.identities[0].concept1[0]; - this.concept2_desc = this.concept.identities[0].concept1[1]; - this.concept2_uri = this.concept.identities[0].concept1[2]; - this.concept2_auth = this.concept.identities[0].concept1[3]; + this.concept_name1 = this.concept.identities[1].concept1[0]; + this.concept_desc1 = this.concept.identities[1].concept1[1]; + this.concept_uri1 = this.concept.identities[1].concept1[2]; + this.concept_auth1 = this.concept.identities[1].concept1[3]; - } } }, filter3: function(concept3, concept) { - if(this.concept.identities[0].concepts[2] !== undefined){ + if(this.concept.identities[1].concept2[0] !== undefined){ this.concept3 = true; - if(this.concept.identities[0].concept2 !== undefined) { - - this.concept3_name = this.concept.identities[0].concept2[0]; - this.concept3_desc = this.concept.identities[0].concept2[1]; - this.concept3_uri = this.concept.identities[0].concept2[2]; - this.concept3_auth = this.concept.identities[0].concept2[3]; + this.concept_name2 = this.concept.identities[1].concept2[0]; + this.concept_desc2 = this.concept.identities[1].concept2[1]; + this.concept_uri2 = this.concept.identities[1].concept2[2]; + this.concept_auth2 = this.concept.identities[1].concept2[3]; } - - } } }, diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index b2d89b9b..68558c80 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -504,41 +504,35 @@ def _relabel(datum): results = map(_relabel, [c.data for c in concepts]) for result in results: - # Filter results to remove dupilcates and any non-duplicates to concepts - new_concepts = {} - length = len(result["identities"]) - if length != 0: - length = length - 1 - identities = [] - identities.append(result["identities"][0]) - while (length != 0): - if (length != 0): - concept1 = result["identities"][length]["concepts"] - if (length != 0): - length = length - 1 - else: - break - concept2 = result["identities"][length]["concepts"] - if set(concept1) != set(concept2): - identities.append(result["identities"][length]) - else: - break + identities = [] + if result["identities"]: + identities.append(result["identities"][0]["concepts"]) + for ident in result["identities"]: - result["identities"] = identities - concepts = result["identities"][0]["concepts"] - uri = result["uri"] - if uri in concepts: concepts.remove(uri) + for identity in identities: + if set(identity) != set(ident["concepts"]): + identities.append(ident) + + result["identities"] = identities + if result["identities"]: + concepts = result["identities"] + uri = result["uri"] + if uri in concepts[0]: concepts[0].remove(uri) + print concepts i = 0 # used to generate concept name - for concept in concepts: - ''' go through all the concepts and parse xml data for each concept - then append info to list and then append list to dictionary so that - list can be referenced as con0, con1, etc - ''' + new_concepts = {} + for concept in concepts[0]: + print concept + #go through all the concepts and parse xml data for each concept + #then append info to list and then append list to dictionary so that + #list can be referenced as con0, con1, etc + hps = re.search( r'www.digitalhps.org', concept, re.M|re.I) viaf = re.search( r'viaf.org', concept, re.M|re.I) if hps: + print "found HPS" url = concept data = urllib2.urlopen(url) @@ -563,6 +557,7 @@ def _relabel(datum): i = i + 1 elif viaf: + print "found VIAF" url = concept + '/viaf.xml' # have to append /viaf.xml to viaf url's in order to access xml data = urllib2.urlopen(url) tree = e.parse(data) @@ -584,7 +579,8 @@ def _relabel(datum): else: print "Nothing found!!" - result["identities"][0].update(new_concepts) + result["identities"].append(new_concepts) + print results return Response({'results': results}) From 1a363f103833587f0bf9149ac3be67ba7eae591b Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Wed, 29 Nov 2017 12:50:35 -0700 Subject: [PATCH 06/16] [VGNWB-207] removed print statements --- annotations/views/rest_views.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index 68558c80..15299b01 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -519,16 +519,12 @@ def _relabel(datum): uri = result["uri"] if uri in concepts[0]: concepts[0].remove(uri) - print concepts i = 0 # used to generate concept name new_concepts = {} for concept in concepts[0]: - print concept #go through all the concepts and parse xml data for each concept #then append info to list and then append list to dictionary so that #list can be referenced as con0, con1, etc - - hps = re.search( r'www.digitalhps.org', concept, re.M|re.I) viaf = re.search( r'viaf.org', concept, re.M|re.I) if hps: @@ -580,7 +576,6 @@ def _relabel(datum): print "Nothing found!!" result["identities"].append(new_concepts) - print results return Response({'results': results}) From 6eada27c604bc3c917a32be876541bbad2aadb21 Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Wed, 29 Nov 2017 12:51:44 -0700 Subject: [PATCH 07/16] [VGNWB-207]removed print statements --- annotations/views/rest_views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index 15299b01..82f607f3 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -528,7 +528,6 @@ def _relabel(datum): hps = re.search( r'www.digitalhps.org', concept, re.M|re.I) viaf = re.search( r'viaf.org', concept, re.M|re.I) if hps: - print "found HPS" url = concept data = urllib2.urlopen(url) @@ -553,7 +552,6 @@ def _relabel(datum): i = i + 1 elif viaf: - print "found VIAF" url = concept + '/viaf.xml' # have to append /viaf.xml to viaf url's in order to access xml data = urllib2.urlopen(url) tree = e.parse(data) From 07e289a449024e66b2d2496799bd762078aaf610 Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Fri, 12 Jan 2018 11:43:13 -0700 Subject: [PATCH 08/16] [VGNWB-207-3] Added responsive height to max-height of appellation-creator to prevent two scroll boxes. --- .../static/annotations/js/annotators/text.js | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/annotations/static/annotations/js/annotators/text.js b/annotations/static/annotations/js/annotators/text.js index 0c37a6ce..aa084bc6 100644 --- a/annotations/static/annotations/js/annotators/text.js +++ b/annotations/static/annotations/js/annotators/text.js @@ -17,25 +17,22 @@ var ConceptListItem = {
-
{{ concept_desc }}
-
+ +
{{ concept_desc }}
+
- -
{{ concept2_desc }}
-
+
{{ concept3_desc }}
`, data: function() { @@ -413,7 +410,7 @@ AppellationCreator = { } }, - template: `
+ template: `
What is this? Date: Tue, 16 Jan 2018 10:23:18 -0700 Subject: [PATCH 09/16] [VGNWB-203-7] fixed code layout. --- annotations/views/rest_views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index 82f607f3..12dd45a1 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -504,14 +504,17 @@ def _relabel(datum): results = map(_relabel, [c.data for c in concepts]) for result in results: + identities = [] if result["identities"]: identities.append(result["identities"][0]["concepts"]) + for ident in result["identities"]: - for identity in identities: if set(identity) != set(ident["concepts"]): identities.append(ident) + + result["identities"] = identities if result["identities"]: From 319c61f072213c3944a823089feb30bda1243442 Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Fri, 26 Jan 2018 15:06:49 -0700 Subject: [PATCH 10/16] [VGNWB-207-3] Cleaned up the jean bacon loops and added more comments. --- annotations/views/rest_views.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index 12dd45a1..a7c55545 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -504,30 +504,24 @@ def _relabel(datum): results = map(_relabel, [c.data for c in concepts]) for result in results: - - identities = [] - if result["identities"]: + identities = [] # list to hold duplicate identities + if result["identities"]: # if identities exist append the first identitiy to the list so that we can filter out other identities against it identities.append(result["identities"][0]["concepts"]) - - for ident in result["identities"]: - for identity in identities: - if set(identity) != set(ident["concepts"]): + for ident in result["identities"]: # go through the identities in each result + for identity in identities: # go through the identities in the identities list + if set(identity) != set(ident["concepts"]): # if the ideneities list does not contain the identity from the result then add it to the list identities.append(ident) - - - - result["identities"] = identities + result["identities"] = identities # replace the identities list if result["identities"]: concepts = result["identities"] - uri = result["uri"] - if uri in concepts[0]: concepts[0].remove(uri) + if uri in concepts[0]: concepts[0].remove(uri) # remove original uri from the list if it exists. i = 0 # used to generate concept name new_concepts = {} - for concept in concepts[0]: - #go through all the concepts and parse xml data for each concept - #then append info to list and then append list to dictionary so that - #list can be referenced as con0, con1, etc + for concept in concepts[0]: # determine if the concept is a viaf or concept power uri + #go through all the concepts and parse xml data for each concept + #then append info to list and then append list to dictionary so that + #list can be referenced as con0, con1, etc hps = re.search( r'www.digitalhps.org', concept, re.M|re.I) viaf = re.search( r'viaf.org', concept, re.M|re.I) if hps: @@ -576,7 +570,7 @@ def _relabel(datum): else: print "Nothing found!!" - result["identities"].append(new_concepts) + result["identities"].append(new_concepts) # add the concept data back to the identities list return Response({'results': results}) From 86f43cb847177eb78a7967c2e0c8a2c700187543 Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Fri, 26 Jan 2018 15:26:39 -0700 Subject: [PATCH 11/16] [VGNWB-207-3] updated six and pg requirements to prevent build from failing. --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d93d6da4..73dda157 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ kombu==3.0.37 lxml==3.6.4 networkx==1.11 oauthlib==1.0.3 -psycopg2==2.6.1 +psycopg2==2.7.1 PyJWT==1.4.2 python-dateutil==2.4.2 python-openid==2.2.5 @@ -39,3 +39,4 @@ Unidecode==0.4.20 urllib3==1.19.1 xmltodict==0.10.2 dicttoxml==1.6.6 +six==1.10.0 From 697dbc8880f934ba3594d76987f4a0c64b9bed2b Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Tue, 30 Jan 2018 10:32:38 -0700 Subject: [PATCH 12/16] [VGNWB-207] updated formatting. --- annotations/views/rest_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index a7c55545..11d8fd0b 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -504,7 +504,7 @@ def _relabel(datum): results = map(_relabel, [c.data for c in concepts]) for result in results: - identities = [] # list to hold duplicate identities + identities = [] # list to hold non-duplicate identities if result["identities"]: # if identities exist append the first identitiy to the list so that we can filter out other identities against it identities.append(result["identities"][0]["concepts"]) for ident in result["identities"]: # go through the identities in each result From 96edda920a2fcae1635b76b2fc6859316d2625e9 Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Thu, 1 Feb 2018 12:30:21 -0700 Subject: [PATCH 13/16] [VGNWB-237] added code to ignore unicode errors so that text will still display regardless if there are encoding errors. --- annotations/annotators.py | 3 ++- annotations/templates/annotations/vue.html | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/annotations/annotators.py b/annotations/annotators.py index e7a85dfe..e6418d3e 100644 --- a/annotations/annotators.py +++ b/annotations/annotators.py @@ -153,11 +153,12 @@ def render_display(self, context={}): def get_context(self): resource = self.get_resource() request = self.context.get('request') + content = self.get_content(resource) return { 'text': self.text, 'textid': self.text.id, 'title': 'Annotate Text', - 'content': self.get_content(resource), + 'content': unicode(content, errors='ignore'), 'baselocation' : basepath(request), 'userid': request.user.id, 'title': self.text.title, diff --git a/annotations/templates/annotations/vue.html b/annotations/templates/annotations/vue.html index 3e6dc76d..ea88c7d8 100644 --- a/annotations/templates/annotations/vue.html +++ b/annotations/templates/annotations/vue.html @@ -231,7 +231,7 @@ var USER_NAME = '{{ user.username }}'; var PROJECT_ID = '{{ project.id }}'; var PROJECT_NAME = '{{ project.name }}'; - var TEXT_CONTENT = `{{ content|safe }}`; + var TEXT_CONTENT = `{{ content|safe }}`; // the safe tag suppresses unicode errors var BASE_URL = '{{ base_url }}'; // Appellator is the text annotation application. Even though we load From 867dc2a0e9b549b0cec14a4ce78e0ebb6b9c080d Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Thu, 8 Feb 2018 10:35:22 -0700 Subject: [PATCH 14/16] [VGNWB-237] removed files that should not have been in there. --- .../static/annotations/js/annotators/text.js | 74 +---------------- annotations/views/rest_views.py | 82 +------------------ 2 files changed, 5 insertions(+), 151 deletions(-) diff --git a/annotations/static/annotations/js/annotators/text.js b/annotations/static/annotations/js/annotators/text.js index e6422c9d..65d3a314 100644 --- a/annotations/static/annotations/js/annotators/text.js +++ b/annotations/static/annotations/js/annotators/text.js @@ -6,84 +6,17 @@ var ConceptListItem = { props: ['concept'], - template: `
-
+ template: ` - -
- -
{{ concept_desc }}
-
- -
- -
{{ concept2_desc }}
-
- -
- -
{{ concept3_desc }}
-
-
`, - data: function() { - return { - concept1: false, - concept2: false, - concept3: false, - ide: this.concept, - } - }, +
`, methods: { select: function() { this.$emit('selectconcept', this.concept); }, - filter: function(concept1, concept) { - if(this.concept.identities[1].concept0[0] !== undefined){ - this.concept1 = true; - this.concept_name = this.concept.identities[1].concept0[0]; - this.concept_desc = this.concept.identities[1].concept0[1]; - this.concept_uri = this.concept.identities[1].concept0[2]; - this.concept_auth = this.concept.identities[1].concept0[3]; - } - }, - filter2: function(concept2, concept) { - if( this.concept.identities[1].concept1[0] !== undefined){ - this.concept2 = true; - this.concept_name1 = this.concept.identities[1].concept1[0]; - this.concept_desc1 = this.concept.identities[1].concept1[1]; - this.concept_uri1 = this.concept.identities[1].concept1[2]; - this.concept_auth1 = this.concept.identities[1].concept1[3]; - - - } - }, - filter3: function(concept3, concept) { - if(this.concept.identities[1].concept2[0] !== undefined){ - this.concept3 = true; - this.concept_name2 = this.concept.identities[1].concept2[0]; - this.concept_desc2 = this.concept.identities[1].concept2[1]; - this.concept_uri2 = this.concept.identities[1].concept2[2]; - this.concept_auth2 = this.concept.identities[1].concept2[3]; - - } - } - - }, - created() { - this.filter(), - this.filter2(), - this.filter3() } } @@ -411,7 +344,7 @@ AppellationCreator = { } }, - template: `
+ template: `
What is this? Date: Thu, 1 Mar 2018 12:54:02 -0700 Subject: [PATCH 15/16] [VGNWB-237] added more flexibility to character encoding so that regardless of what encoding is uploaded to giles vogon should be able to detected and recode in utf-8 --- annotations/annotators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/annotations/annotators.py b/annotations/annotators.py index e6418d3e..3219cadb 100644 --- a/annotations/annotators.py +++ b/annotations/annotators.py @@ -58,6 +58,7 @@ def my_view(request, text_id): from annotations.utils import basepath from annotations.models import TextCollection, VogonUserDefaultProject from urlparse import urlparse +import chardet class Annotator(object): @@ -154,11 +155,12 @@ def get_context(self): resource = self.get_resource() request = self.context.get('request') content = self.get_content(resource) + detect = chardet.detect(content) return { 'text': self.text, 'textid': self.text.id, 'title': 'Annotate Text', - 'content': unicode(content, errors='ignore'), + 'content': content.decode(detect['encoding']).encode('utf-8'), 'baselocation' : basepath(request), 'userid': request.user.id, 'title': self.text.title, From d7b13142af6ed427b589825c79b2bb3d852b7bdd Mon Sep 17 00:00:00 2001 From: Tjquinn1 Date: Wed, 7 Mar 2018 15:06:56 -0700 Subject: [PATCH 16/16] [VGNWB-237] added comment about why we are using chardet. --- annotations/annotators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/annotations/annotators.py b/annotations/annotators.py index 3219cadb..fb68c5ee 100644 --- a/annotations/annotators.py +++ b/annotations/annotators.py @@ -160,7 +160,7 @@ def get_context(self): 'text': self.text, 'textid': self.text.id, 'title': 'Annotate Text', - 'content': content.decode(detect['encoding']).encode('utf-8'), + 'content': content.decode(detect['encoding']).encode('utf-8'), # We are using chardet to guess the encoding becuase giles is returning everyting with a utf-8 header even if it is not utf-8 'baselocation' : basepath(request), 'userid': request.user.id, 'title': self.text.title,