diff --git a/.env_app-example b/.env_app-example index ff2ecc82..9b937388 100644 --- a/.env_app-example +++ b/.env_app-example @@ -15,3 +15,5 @@ export BASE_URL="http://localhost:8000/" # Make sure APP_ROOT variable ends with a / # this is the prefix that vogon will be deployed under (e.g. "vogon-2" for deployment at my-server.org/vogon-2) export APP_ROOT= + +export QUADRIGA_ENDPOINT= \ No newline at end of file diff --git a/annotations/admin.py b/annotations/admin.py index 888c5a97..052ed509 100755 --- a/annotations/admin.py +++ b/annotations/admin.py @@ -1,7 +1,6 @@ from django.contrib import admin from annotations.forms import * from annotations.models import * -from annotations.tasks import submit_relationsets_to_quadriga from itertools import groupby @@ -50,81 +49,12 @@ def created_by(self, obj): return obj.createdBy -def submit_relationsets(modeladmin, request, queryset): - """ - Submit selected :class:`.RelationSet`\s to Quadriga. - - Will quietly skip any :class:`.RelationSet`\s that have already been - submitted. - """ - - queryset = queryset.filter(submitted=False, pending=False) - - # Do not submit a relationset to Quadriga if the constituent interpretations - # involve concepts that are not resolved. - all_rsets = [rs for rs in queryset if rs.ready()] - - project_grouper = lambda rs: getattr(rs.project, 'quadriga_id', -1) - for project_id, project_group in groupby(sorted(all_rsets, key=project_grouper), key=project_grouper): - for text_id, text_group in groupby(project_group, key=lambda rs: rs.occursIn.id): - text = Text.objects.get(pk=text_id) - for user_id, user_group in groupby(text_group, key=lambda rs: rs.createdBy.id): - user = VogonUser.objects.get(pk=user_id) - # We lose the iterator after the first pass, so we want a list here. - rsets = [] - for rs in user_group: - rsets.append(rs.id) - rs.pending = True - rs.save() - kwargs = {} - if project_id: - kwargs.update({ - 'project_id': project_id, - }) - submit_relationsets_to_quadriga.delay(rsets, text.id, user.id, **kwargs) - - -def submit_relationsets_synch(modeladmin, request, queryset): - """ - Submit selected :class:`.RelationSet`\s to Quadriga. - - Will quietly skip any :class:`.RelationSet`\s that have already been - submitted. - """ - - queryset = queryset.filter(submitted=False, pending=False) - - # Do not submit a relationset to Quadriga if the constituent interpretations - # involve concepts that are not resolved. - all_rsets = [rs for rs in queryset if rs.ready()] - - project_grouper = lambda rs: getattr(rs.project, 'quadriga_id', -1) - for project_id, project_group in groupby(sorted(all_rsets, key=project_grouper), key=project_grouper): - for text_id, text_group in groupby(project_group, key=lambda rs: rs.occursIn.id): - text = Text.objects.get(pk=text_id) - for user_id, user_group in groupby(text_group, key=lambda rs: rs.createdBy.id): - user = VogonUser.objects.get(pk=user_id) - # We lose the iterator after the first pass, so we want a list here. - rsets = [] - for rs in user_group: - rsets.append(rs.id) - rs.pending = True - rs.save() - kwargs = {} - if project_id and project_id > 0: - kwargs.update({ - 'project_id': project_id, - }) - submit_relationsets_to_quadriga(rsets, text.id, user.id, **kwargs) - - class RelationSetAdmin(admin.ModelAdmin): class Meta: model = RelationSet list_display = ('id', 'createdBy', 'occursIn', 'created', 'ready', - 'pending', 'submitted', ) - actions = (submit_relationsets, submit_relationsets_synch) + 'submitted', 'status',) class AppellationAdmin(admin.ModelAdmin): diff --git a/annotations/annotators.py b/annotations/annotators.py index cff338ca..d617d7f8 100644 --- a/annotations/annotators.py +++ b/annotations/annotators.py @@ -50,15 +50,9 @@ def my_view(request, text_id): """ - -import requests from django.shortcuts import get_object_or_404, render from django.http import Http404 -from annotations.tasks import tokenize -from annotations.utils import basepath -from annotations.models import TextCollection, VogonUserDefaultProject, Text -from urllib.parse import urlparse -import chardet +from annotations.models import TextCollection, Text class Annotator(object): diff --git a/annotations/migrations/0042_relationset_status.py b/annotations/migrations/0042_relationset_status.py new file mode 100644 index 00000000..9bf691b3 --- /dev/null +++ b/annotations/migrations/0042_relationset_status.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.20 on 2024-10-29 16:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('annotations', '0041_delete_importedcitesphereitem'), + ] + + operations = [ + migrations.AddField( + model_name='relationset', + name='status', + field=models.CharField(choices=[('not_ready', 'Not Ready'), ('ready_to_submit', 'Ready to Submit'), ('submitted', 'Submitted')], default='not_ready', max_length=20), + ), + ] diff --git a/annotations/migrations/0043_remove_relationset_pending.py b/annotations/migrations/0043_remove_relationset_pending.py new file mode 100644 index 00000000..4e4d63db --- /dev/null +++ b/annotations/migrations/0043_remove_relationset_pending.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.20 on 2024-10-29 18:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('annotations', '0042_relationset_status'), + ] + + operations = [ + migrations.RemoveField( + model_name='relationset', + name='pending', + ), + ] diff --git a/annotations/migrations/0044_merge_20241213_1539.py b/annotations/migrations/0044_merge_20241213_1539.py new file mode 100644 index 00000000..c78608f1 --- /dev/null +++ b/annotations/migrations/0044_merge_20241213_1539.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.20 on 2024-12-13 15:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('annotations', '0042_textcollection_collaborators'), + ('annotations', '0043_remove_relationset_pending'), + ] + + operations = [ + ] diff --git a/annotations/models.py b/annotations/models.py index 4508e5b2..f89fd25f 100644 --- a/annotations/models.py +++ b/annotations/models.py @@ -776,6 +776,10 @@ class RelationSet(models.Model): A :class:`.RelationSet` organizes :class:`.Relation`\s into complete statements. """ + # RelationSet statuses + STATUS_NOT_READY = 'not_ready' + STATUS_READY_TO_SUBMIT = 'ready_to_submit' + STATUS_SUBMITTED = 'submitted' project = models.ForeignKey('TextCollection', related_name='relationsets', null=True, blank=True, on_delete=models.CASCADE) @@ -799,12 +803,15 @@ class RelationSet(models.Model): occursIn = models.ForeignKey('Text', related_name='relationsets', on_delete=models.CASCADE) """The text on which this RelationSet is based.""" - pending = models.BooleanField(default=False) - """ - A :class:`.RelationSet` is pending if it has been selected for submission, - but the submission process has not yet completed. The primary purpose of - this field is to prevent duplicate submissions. - """ + STATUS_CHOICES = [ + (STATUS_NOT_READY, 'Not Ready'), + (STATUS_READY_TO_SUBMIT, 'Ready to Submit'), + (STATUS_SUBMITTED, 'Submitted'), + ] + + status = models.CharField( + max_length=20, choices=STATUS_CHOICES, default=STATUS_NOT_READY + ) submitted = models.BooleanField(default=False) """ @@ -920,6 +927,20 @@ def ready(self): return all(map(criteria, values)) ready.boolean = True # So that we can display a nifty icon in changelist. + def update_status(self): + """ + Check if the RelationSet is ready and update the status accordingly. + """ + if self.ready(): # Check readiness based on the concepts + if self.status != self.STATUS_SUBMITTED: # Avoid overriding submitted status + self.status = self.STATUS_READY_TO_SUBMIT + self.submitted = False + else: + self.status = self.STATUS_NOT_READY + self.submitted = False + + self.save() + def appellations(self): """ Get all non-predicate appellations in child :class:`.Relation`\s. @@ -1160,4 +1181,4 @@ class DocumentPosition(models.Model): If :attr:`.position_type` is :attr:`.WHOLE_DOCUMENT`\, then this can be blank. - """ + """ \ No newline at end of file diff --git a/annotations/quadriga.py b/annotations/quadriga.py index 606950db..627fe9f4 100644 --- a/annotations/quadriga.py +++ b/annotations/quadriga.py @@ -1,5 +1,6 @@ from django.contrib.contenttypes.models import ContentType from django.conf import settings +from django.utils import timezone from annotations.models import Relation, Appellation, DateAppellation, DocumentPosition @@ -7,9 +8,6 @@ import datetime import re import uuid -import requests -from requests.auth import HTTPBasicAuth - def _created_element(element, annotation): ET.SubElement(element, 'id') @@ -245,26 +243,6 @@ def to_quadruples(relationsets, text, user, network_label=None, return project, params -def submit_relationsets(relationsets, text, user, - userid=settings.QUADRIGA_USERID, - password=settings.QUADRIGA_PASSWORD, - endpoint=settings.QUADRIGA_ENDPOINT, **kwargs): - """ - Submit the :class:`.RelationSet`\s in ``relationsets`` to Quadriga. - """ - payload, params = to_quadruples(relationsets, text, user, toString=True, **kwargs) - auth = HTTPBasicAuth(userid, password) - headers = {'Accept': 'application/xml'} - r = requests.post(endpoint, data=payload, auth=auth, headers=headers) - - if r.status_code == requests.codes.ok: - response_data = parse_response(r.text) - response_data.update(params) - return True, response_data - - return False, r.text - - def parse_response(raw_response): QDNS = '{http://www.digitalhps.org/Quadriga}' root = ET.fromstring(raw_response) @@ -275,3 +253,112 @@ def parse_response(raw_response): tag = child.tag.replace(QDNS, '') data[tag] = child.text return data + + + +def build_concept_node(concept, user, creation_time, source_uri): + """ + Helper function to build a concept node dictionary. + """ + return { + "label": concept.label or "", + "metadata": { + "type": "appellation_event", + "interpretation": concept.uri, + "termParts": [ + { + "position": 1, + "expression": concept.label or "", + "normalization": "", + "formattedPointer": "", + "format": "" + } + ] + }, + "context": { + "creator": user.username, + "creationTime": creation_time.strftime('%Y-%m-%d'), + "creationPlace": "", + "sourceUri": source_uri + } + } + +def get_relation_node(user, creation_time, source_uri): + """ + Helper function to build a relation node. + """ + return { + "label": "", + "metadata": { + "type": "relation_event" + }, + "context": { + "creator": user.username, + "creationTime": creation_time.strftime('%Y-%m-%d'), + "creationPlace": "", + "sourceUri": source_uri + } + } + +def generate_graph_data(relationset, user): + nodes = {} + edges = [] + node_counter = 0 + source_uri = relationset.occursIn.uri + + def get_node_id(): + nonlocal node_counter + node_id = str(node_counter) + node_counter += 1 + return node_id + + for relation in relationset.constituents.all(): + subject = getattr(relation.source_content_object, 'interpretation', None) + obj = getattr(relation.object_content_object, 'interpretation', None) + predicate = getattr(relation.predicate, 'interpretation', None) + + # Extract the creation time from the appellation + creation_time = relationset.occursIn.created + + if subject and subject.uri not in nodes: + subject_id = get_node_id() + nodes[subject_id] = build_concept_node(subject, user, creation_time, source_uri) + + if obj and obj.uri not in nodes: + obj_id = get_node_id() + nodes[obj_id] = build_concept_node(obj, user, creation_time, source_uri) + + if predicate and predicate.uri not in nodes: + predicate_id = get_node_id() + nodes[predicate_id] = build_concept_node(predicate, user, creation_time, source_uri) + + relation_id = get_node_id() + nodes[relation_id] = get_relation_node(user, creation_time, source_uri) + + if subject and predicate: + edges.append({"source": subject_id, "relation": "subject", "target": predicate_id}) + + if predicate and obj: + edges.append({"source": predicate_id, "relation": "predicate", "target": obj_id}) + + edges.append({"source": relation_id, "relation": "object", "target": obj_id}) + + return { + "graph": { + "metadata": { + "defaultMapping": { + "subject": {"type": "REF", "reference": "0"}, + "predicate": {"type": "URI", "uri": "", "label": ""}, + "object": {"type": "REF", "reference": "3"} + }, + "context": { + "creator": user.username, + "creationTime": timezone.now().strftime('%Y-%m-%d'), + "creationPlace": "", + "sourceUri": source_uri + } + }, + "nodes": nodes, + "edges": edges + } + } diff --git a/annotations/serializers.py b/annotations/serializers.py index 996a0923..dfd5e69d 100644 --- a/annotations/serializers.py +++ b/annotations/serializers.py @@ -1,5 +1,6 @@ from rest_framework import serializers from annotations.models import VogonUser +from annotations.utils import tokenize from .models import * from concepts.models import Concept, Type @@ -122,7 +123,7 @@ class Meta: model = RelationSet fields = ('id', 'label', 'created', 'template', 'createdBy', 'occursIn', 'appellations', 'concepts', 'project', - 'representation', 'date_appellations') # + 'representation', 'date_appellations', 'status') # class TemporalBoundsSerializer(serializers.ModelSerializer): diff --git a/annotations/static/annotations/css/annotators/text.css b/annotations/static/annotations/css/annotators/text.css index 3b6a4b2d..e5a4039d 100644 --- a/annotations/static/annotations/css/annotators/text.css +++ b/annotations/static/annotations/css/annotators/text.css @@ -198,6 +198,10 @@ background-color: rgb(250, 250, 250); } + .relation-submitted { + border-left: 4px solid #7aef7a; + } + a[data-tooltip] { position: relative; cursor: pointer; diff --git a/annotations/static/annotations/js/annotators/relationlist.js b/annotations/static/annotations/js/annotators/relationlist.js index 40169417..46dd103e 100644 --- a/annotations/static/annotations/js/annotators/relationlist.js +++ b/annotations/static/annotations/js/annotators/relationlist.js @@ -1,64 +1,202 @@ -RelationListItem = { +// RelationListItem Component +const RelationListItem = { props: ['relation'], - template: `
  • - - - - - -
    {{ getRepresentation(relation) }}
    -
    Created by {{ getCreatorName(relation.createdBy) }} on {{ getFormattedDate(relation.created) }}
    -
  • `, - - methods: { - select: function () { - this.$emit('selectrelation', this.relation); + template: ` +
  • +
    + + {{ getRepresentation(relation) }} +
    +
    + Created by {{ getCreatorName(relation.createdBy) }} on {{ getFormattedDate(relation.created) }} +
    +
  • + `, + data() { + return { + isChecked: this.isAlreadySubmitted + }; + }, + computed: { + isReadyToSubmit() { + // This matches RelationSet.STATUS_READY_TO_SUBMIT = 'ready_to_submit' + // See annotations/models.py RelationSet model + return this.relation.status === 'ready_to_submit'; }, - isSelected: function () { - return this.relation.selected; + isAlreadySubmitted() { + // This matches RelationSet.STATUS_SUBMITTED = 'submitted' + // See annotations/models.py RelationSet model + return this.relation.status === 'submitted'; }, - getRepresentation: function (relation) { + tooltipText() { + if (this.isAlreadySubmitted) return 'Already submitted'; + if (!this.isReadyToSubmit) return 'Quadruple is not ready to submit'; + return ''; + } + }, + methods: { + toggleSelection() { + if (!this.isAlreadySubmitted) { + this.$emit('toggleSelection', { relation: this.relation, selected: this.isChecked }); + } + }, + getRepresentation(relation) { if (relation.representation) { return relation.representation; } else { - return relation.appellations.map(function (appellation) { - return appellation.interpretation.label; - }).join('; '); + return relation.appellations.map(appellation => appellation.interpretation.label).join('; '); } }, - getCreatorName: function (creator) { - if (creator.id == USER_ID) { - return 'you'; - } else { - return creator.username; - } + getCreatorName(creator) { + return creator.id == USER_ID ? 'you' : creator.username; }, - getFormattedDate: function (isodate) { + getFormattedDate(isodate) { return moment(isodate).format('dddd LL [at] LT'); } - } -} +}; -RelationList = { +// RelationList Component +const RelationList = { props: ['relations'], - template: ``, + template: `
    +
    +
    Relations
    + +
    + + + + + +
    `, components: { 'relation-list-item': RelationListItem }, + data() { + return { + selectedQuadruples: [], + message: '', + messageClass: '' + }; + }, + computed: { + canSubmit() { + return this.selectedQuadruples.length > 0 && this.selectedQuadruples.every(id => { + const relation = this.relations.find(rel => rel.id === id); + return relation && relation.status === 'ready_to_submit'; + }); + } + }, methods: { - selectRelation: function (relation) { - this.$emit('selectrelation', relation); + selectRelation(relation) { + this.$emit('selectRelation', relation); + }, + toggleSelection({ relation, selected }) { + if (selected) { + this.selectedQuadruples.push(relation.id); + } else { + this.selectedQuadruples = this.selectedQuadruples.filter(id => id !== relation.id); + } + }, + submitSelected() { + if (this.selectedQuadruples.length === 0) { + this.setMessage("No quadruples selected", "alert-warning"); + return; + } + + let submissionPromises = this.selectedQuadruples.map((quadrupleId) => { + return this.submitQuadruple(quadrupleId); + }); + + Promise.allSettled(submissionPromises).then((results) => { + let successes = results.filter(r => r.status === 'fulfilled').length; + let failures = results.filter(r => r.status === 'rejected').length; + + if (successes > 0) { + this.setMessage(`${successes} quadruple(s) submitted successfully.`, "alert-success"); + } + + if (failures > 0) { + let failureReasons = results + .filter(r => r.status === 'rejected') + .map(r => r.reason) + .join(', '); + this.setMessage(`Failed to submit ${failures} quadruple(s). Reasons: ${failureReasons}`, "alert-danger"); + } + + this.fetchRelations(); + }); + }, + + submitQuadruple(quadrupleId) { + const csrfToken = getCookie('csrftoken'); + + const param = { + 'pk': quadrupleId, + 'project_id': self.project.id + }; + + return axios.post(`/vogon/rest/relationset/submit`, param, { + headers: { + 'X-CSRFToken': csrfToken, + 'Content-type': 'application/json' + }, + withCredentials: true, + }) + .then(() => { + // Update local status to 'submitted' + let relation = this.relations.find(r => r.id === quadrupleId); + if (relation) { + relation.status = 'submitted'; + } + this.selectedQuadruples = this.selectedQuadruples.filter(id => id !== quadrupleId); + }) + .catch((error) => { + console.error(`Failed to submit quadruple ${quadrupleId}:`, error); + throw error.response ? error.response.data.error : 'Unknown error'; + }); + }, + + fetchRelations() { + axios.get('/vogon/rest/relation') + .then(response => { + this.relations = response.data; + }) + .catch(error => { + console.error('Failed to fetch relations:', error); + }); + }, + + setMessage(message, type) { + this.message = message; + this.messageClass = type; + setTimeout(() => { + this.message = ''; + this.messageClass = ''; + }, 5000); } } -} \ No newline at end of file +}; diff --git a/annotations/tasks.py b/annotations/tasks.py deleted file mode 100644 index 9c7ca32e..00000000 --- a/annotations/tasks.py +++ /dev/null @@ -1,280 +0,0 @@ -""" -We should probably write some documentation. -""" - - - -from django.contrib.auth.models import Group -from django.utils.safestring import SafeText -from django.contrib.contenttypes.models import ContentType - -import requests, uuid, re -from datetime import datetime, timedelta -from django.utils import timezone -from itertools import groupby, chain -from collections import defaultdict - -from annotations.models import * -from annotations import quadriga - -from celery import shared_task - -from django.conf import settings -import logging -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(settings.LOGLEVEL) - - - - -def tokenize(content, delimiter=' '): - """ - In order to annotate a text, we must first wrap "annotatable" tokens - in tags, with arbitrary IDs. - - Parameters - ---------- - content : unicode - delimiter : unicode - Character or sequence by which to split and join tokens. - - Returns - ------- - tokenizedContent : unicode - """ - chunks = content.split(delimiter) if content is not None else [] - pattern = '{1}' - return delimiter.join([pattern.format(i, c) for i, c in enumerate(chunks)]) - - -def retrieve(repository, resource): - """ - Get the content of a resource. - - Parameters - ---------- - repository : :class:`.Repository` - resource : unicode or int - Identifier by which the resource can be retrieved from ``repository``. - - Returns - ------- - content : unicode - """ - return repository.getContent(resource) - - -# TODO: this should be retired. -def scrape(url): - """ - Retrieve text content from a website. - - Parameters - ---------- - url : unicode - Location of web resource. - - Returns - ------- - textData : dict - Metadata and text content retrieved from ``url``. - """ - response = requests.get(uri, allow_redirects=True) - - # TODO : consider plugging in something like DiffBot. - soup = "" #BeautifulSoup(response.text, "html.parser") - - textData = { - 'title': soup.title.string, - 'content': response.text, - 'content-type': response.headers['content-type'], - } - return textData - - -# TODO: this should be retired. -def extract_text_file(uploaded_file): - """ - Extract the text file, and return its content - - Parameters - ---------- - uploaded_file : InMemoryUploadedFile - The uploaded text file - - Returns - ------- - Content of the text file as a string - """ - if uploaded_file.content_type != 'text/plain': - raise ValueError('uploaded_file file should be a plain text file') - filecontent = '' - for line in uploaded_file: - filecontent += line + ' ' - return filecontent - - -# TODO: this should be retired. -def extract_pdf_file(uploaded_file): - """ - Extract a PDF file and return its content - - Parameters - ---------- - uploaded_file : InMemoryUploadedFile - The uploaded PDF file - - Returns - ------- - Content of the PDF file as a string - """ - if uploaded_file.content_type != 'application/pdf': - raise ValueError('uploaded_file file should be a PDF file') - doc = slate.PDF(uploaded_file) - filecontent = '' - for content in doc: - filecontent += content.decode('utf-8') + '\n\n' - return filecontent - - -# TODO: this should be retired. -# TODO: refactor!! This signature stinks. -def save_text_instance(tokenized_content, text_title, date_created, is_public, user, uri=None): - """ - This method creates and saves the text instance based on the parameters passed - - Parameters - ---------- - tokenized_content : String - The tokenized text - text_title : String - The title of the text instance - date_created : Date - The date to be associated with text instance - is_public : Boolean - Whether the text content is public or not - user : User - The user who saved the text content - """ - if not uri: - uri = 'http://vogonweb.net/' + str(uuid.uuid1()) - text = Text(tokenizedContent=tokenized_content, - title=text_title, - created=date_created, - public=is_public, - addedBy=user, - uri=uri) - text.save() - if is_public: - group = Group.objects.get_or_create(name='Public')[0] - return text - - -@shared_task -def submit_relationsets_to_quadriga(rset_ids, text_id, user_id, **kwargs): - logger.debug('Submitting %i relations to Quadriga' % len(rset_ids)) - rsets = RelationSet.objects.filter(pk__in=rset_ids) - text = Text.objects.get(pk=text_id) - user = VogonUser.objects.get(pk=user_id) - status, response = quadriga.submit_relationsets(rsets, text, user, **kwargs) - - if status: - qsr = RelationSet.objects.filter(pk__in=rset_ids) - project_id = response.get('projectId') - workspace_id = response.get('workspaceId') - network_id = response.get('networkId') - accession = QuadrigaAccession.objects.create(**{ - 'createdBy': user, - 'project_id': project_id, - 'workspace_id': workspace_id, - 'network_id': network_id - }) - logger.debug('Submitted %i relations as network %s to project %s workspace %s' % (qsr.count(), network_id, project_id, workspace_id)) - - for relationset in qsr: - relationset.submitted = True - relationset.submittedOn = accession.created - relationset.submittedWith = accession - relationset.save() - for relation in relationset.constituents.all(): - relation.submitted = True - relation.submittedOn = accession.created - relation.submittedWith = accession - relation.save() - for appellation in relationset.appellations(): - appellation.submitted = True - appellation.submittedOn = accession.created - appellation.submittedWith = accession - appellation.save() - else: - logger.debug('Quadriga submission failed with %s' % str(response)) - -@shared_task -def accession_ready_relationsets(): - logger.debug('Looking for relations to accession to Quadriga...') - # print 'processing %i relationsets' % len(all_rsets) - # project_grouper = lambda rs: getattr(rs.occursIn.partOf.first(), 'quadriga_id', -1) - - # for project_id, project_group in groupby(sorted(all_rsets, key=project_grouper), key=project_grouper): - kwargs = {} - - for project_id in chain([None], TextCollection.objects.values_list('quadriga_id', flat=True).distinct('quadriga_id')): - if project_id: - kwargs.update({ - 'project_id': project_id, - }) - - qs = RelationSet.objects.filter(submitted=False, pending=False) - if project_id: - qs = qs.filter(project_id__quadriga_id=project_id) - else: # Don't grab relations that actually do belong to a project. - qs = qs.filter(project_id__isnull=True) - - # Do not submit a relationset to Quadriga if the constituent interpretations - # involve concepts that are not resolved. - qs = [o for o in qs if o.ready()] - relationsets = defaultdict(lambda: defaultdict(list)) - - for relationset in qs: - timeCreated = relationset.created - if timeCreated + timedelta(days=settings.SUBMIT_WAIT_TIME['days'], hours=settings.SUBMIT_WAIT_TIME['hours'], minutes=settings.SUBMIT_WAIT_TIME['minutes']) < datetime.now(timezone.utc): - relationsets[relationset.occursIn.id][relationset.createdBy.id].append(relationset) - for text_id, text_rsets in list(relationsets.items()): - for user_id, user_rsets in list(text_rsets.items()): - # Update state. - def _state(obj): - obj.pending = True - obj.save() - list(map(_state, user_rsets)) - submit_relationsets_to_quadriga.delay([o.id for o in user_rsets], text_id, user_id, **kwargs) - - -# TODO: this should be retired. -def handle_file_upload(request, form): - """ - Handle the uploaded file and route it to corresponding handlers - - Parameters - ---------- - request : `django.http.requests.HttpRequest` - form : `django.forms.Form` - The form with uploaded content - - """ - uploaded_file = request.FILES['filetoupload'] - uri = form.cleaned_data['uri'] - text_title = form.cleaned_data['title'] - date_created = form.cleaned_data['datecreated'] - is_public = form.cleaned_data['ispublic'] - user = request.user - file_content = None - if uploaded_file.content_type == 'text/plain': - file_content = extract_text_file(uploaded_file) - elif uploaded_file.content_type == 'application/pdf': - file_content = extract_pdf_file(uploaded_file) - - # Save the content if the above extractors extracted something - if file_content != None: - tokenized_content = tokenize(file_content) - return save_text_instance(tokenized_content, text_title, date_created, is_public, user, uri) diff --git a/annotations/templates/annotations/vue.html b/annotations/templates/annotations/vue.html index d6160a4b..1fb633ed 100644 --- a/annotations/templates/annotations/vue.html +++ b/annotations/templates/annotations/vue.html @@ -4,6 +4,8 @@ {% block extrahead %} + + @@ -466,4 +468,4 @@
    Assignment Failed!
    {% endblock %} - + \ No newline at end of file diff --git a/annotations/utils.py b/annotations/utils.py index fa453a5e..db832a50 100644 --- a/annotations/utils.py +++ b/annotations/utils.py @@ -8,6 +8,26 @@ from django.db.models import Q, Count, F import re import math +import re + +def tokenize(content, delimiter=' '): + """ + In order to annotate a text, we must first wrap "annotatable" tokens + in tags, with arbitrary IDs. + + Parameters + ---------- + content : unicode + delimiter : unicode + Character or sequence by which to split and join tokens. + + Returns + ------- + tokenizedContent : unicode + """ + chunks = content.split(delimiter) if content is not None else [] + pattern = '{1}' + return delimiter.join([pattern.format(i, c) for i, c in enumerate(chunks)]) from annotations import models @@ -175,4 +195,4 @@ def _annotate_project_counts(queryset): Q(texts__relationsets__createdBy=F('ownedBy')), distinct=True), num_collaborators=Count('collaborators', distinct=True) - ) \ No newline at end of file + ) diff --git a/annotations/views/__init__.py b/annotations/views/__init__.py index 671d6c1a..4bccfd18 100644 --- a/annotations/views/__init__.py +++ b/annotations/views/__init__.py @@ -1,4 +1,4 @@ from . import \ aws_views, data_views, main_views, network_views, project_views, \ - quadruple_views, relationtemplate_views, rest_views, search_views, \ + relationtemplate_views, rest_views, search_views, \ text_views, user_views, repository_views, annotation_views diff --git a/annotations/views/quadruple_views.py b/annotations/views/quadruple_views.py deleted file mode 100644 index 56496b47..00000000 --- a/annotations/views/quadruple_views.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -These views are mainly for debugging purposes; they provide quad-xml from -various scenarios. -""" - -from django.http import HttpResponse - -from annotations import quadriga -from annotations.models import (RelationSet, Appellation, Relation, VogonUser, - Text) - - -def appellation_xml(request, appellation_id): - """ - Return partial quad-xml for an :class:`.Appellation`\. - - Parameters - ---------- - request : `django.http.requests.HttpRequest` - appellation_id : int - - Returns - ---------- - :class:`django.http.response.HttpResponse` - """ - - appellation = Appellation.objects.get(pk=appellation_id) - appellation_xml = quadriga.to_appellationevent(appellation, toString=True) - return HttpResponse(appellation_xml, content_type='application/xml') - - -def relation_xml(request, relation_id): - """ - Return partial quad-xml for an :class:`.Appellation`\. - - Parameters - ---------- - request : `django.http.requests.HttpRequest` - relation_id : int - - Returns - ---------- - :class:`django.http.response.HttpResponse` - """ - - relation = Relation.objects.get(pk=relation_id) - relation_xml = quadriga.to_relationevent(relation, toString=True) - return HttpResponse(relation_xml, content_type='application/xml') - - -def relationset_xml(request, relationset_id): - """ - Return partial quad-xml for an :class:`.Appellation`\. - - Parameters - ---------- - request : `django.http.requests.HttpRequest` - relationset_id : int - - Returns - ---------- - :class:`django.http.response.HttpResponse` - """ - - relationset = RelationSet.objects.get(pk=relationset_id) - relation_xml = quadriga.to_relationevent(relationset.root, toString=True) - return HttpResponse(relation_xml, content_type='application/xml') - - -def text_xml(request, text_id, user_id): - """ - Return complete quad-xml for the annotations in a :class:`.Text`\. - - Parameters - ---------- - request : `django.http.requests.HttpRequest` - text_id : int - - Returns - ---------- - :class:`django.http.response.HttpResponse` - """ - - text = Text.objects.get(pk=text_id) - user = VogonUser.objects.get(pk=user_id) - relationsets = RelationSet.objects.filter(occursIn_id=text_id, createdBy_id=user_id) - text_xml, _ = quadriga.to_quadruples(relationsets, text, user, toString=True) - return HttpResponse(text_xml, content_type='application/xml') diff --git a/annotations/views/repository_views.py b/annotations/views/repository_views.py index d36bc96d..0c360fcd 100644 --- a/annotations/views/repository_views.py +++ b/annotations/views/repository_views.py @@ -11,14 +11,12 @@ from django.conf import settings from annotations.forms import RepositorySearchForm -from annotations.tasks import tokenize from repository.models import Repository from repository.auth import * from repository.managers import * from annotations.models import Text, TextCollection from annotations.annotators import supported_content_types -from annotations.tasks import tokenize -from annotations.utils import get_pagination_metadata +from annotations.utils import get_pagination_metadata, tokenize from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage diff --git a/annotations/views/rest_views.py b/annotations/views/rest_views.py index cdd197eb..1c62ff0e 100644 --- a/annotations/views/rest_views.py +++ b/annotations/views/rest_views.py @@ -20,13 +20,17 @@ from annotations.serializers import * from annotations.models import * +from annotations.quadriga import generate_graph_data from concepts.models import Concept, Type from concepts.lifecycle import * +from external_accounts.models import CitesphereAccount + import uuid import requests from django.conf import settings +from django.utils import timezone import json @@ -347,7 +351,6 @@ class PredicateViewSet(AnnotationFilterMixin, viewsets.ModelViewSet): serializer_class = AppellationSerializer permission_classes = (ProjectOwnerOrCollaboratorAccessOrReadOnly, ) - class RelationSetViewSet(viewsets.ModelViewSet): queryset = RelationSet.objects.all() serializer_class = RelationSetSerializer @@ -356,6 +359,10 @@ class RelationSetViewSet(viewsets.ModelViewSet): def get_queryset(self, *args, **kwargs): queryset = super(RelationSetViewSet, self).get_queryset(*args, **kwargs) + # Perform readiness checks only for fetched RelationSets + for relationset in queryset: + relationset.update_status() + textid = self.request.query_params.getlist('text') userid = self.request.query_params.getlist('user') @@ -374,6 +381,70 @@ def get_queryset(self, *args, **kwargs): queryset = queryset.filter(project_id=project_id) return queryset.order_by('-created') + + @action(detail=False, methods=['post'], permission_classes=[IsAuthenticated], url_name='submit') + def submit(self, request): + + user = request.user + pk = request.data.get('pk') + + project_id = request.data.get('project_id') + project = TextCollection.objects.get(pk=project_id) + + try: + relationset = RelationSet.objects.get(pk=pk) + + except RelationSet.DoesNotExist: + return Response({'error': 'RelationSet not found.'}, status=status.HTTP_404_NOT_FOUND) + + if relationset.createdBy != user: + return Response({'error': 'You are not authorized to submit this RelationSet.'}, + status=status.HTTP_403_FORBIDDEN) + + if relationset.status != RelationSet.STATUS_READY_TO_SUBMIT: + return Response({'error': 'Quadruple(s) is not ready to submit.'}, + status=status.HTTP_400_BAD_REQUEST) + + if relationset.submitted: + return Response({'error': 'Quadruple(s) has already been submitted.'}, + status=status.HTTP_400_BAD_REQUEST) + + if not project.quadriga_id: + return Response({'error': 'Project does not have a Quadriga ID configured. Please configure a Quadriga ID in the project settings.'}, + status=status.HTTP_400_BAD_REQUEST) + + try: + citesphere_account = CitesphereAccount.objects.get(user=user, repository=relationset.occursIn.repository) + access_token = citesphere_account.access_token + + headers = { + 'Authorization': f'Bearer {access_token}', + 'Content-Type': 'application/json', + } + + collection_id = project.quadriga_id + endpoint = f"{settings.QUADRIGA_ENDPOINT}/api/v1/collection/{collection_id}/network/add/" + + graph_data = generate_graph_data(relationset, user) + + response = requests.post(endpoint, json=graph_data, headers=headers) + response.raise_for_status() + + # Update the status of the RelationSet + relationset.status = 'submitted' + relationset.submitted = True + relationset.submittedOn = timezone.now() + relationset.save() + + return Response({'success': 'Quadruples submitted successfully.'}, status=status.HTTP_200_OK) + + except CitesphereAccount.DoesNotExist: + return Response({'error': 'No Citesphere account found.'}, + status=status.HTTP_400_BAD_REQUEST) + except requests.RequestException as e: + print("ERROR", e) + return Response({'error': 'Internal Server Error Occured. Please try again later!'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) class RelationViewSet(viewsets.ModelViewSet): @@ -421,6 +492,7 @@ def get_queryset(self, *args, **kwargs): return queryset + # TODO: do we need this anymore? class TemporalBoundsViewSet(viewsets.ModelViewSet, AnnotationFilterMixin): queryset = TemporalBounds.objects.all() diff --git a/annotations/views/text_views.py b/annotations/views/text_views.py index b9d57806..299c0e1a 100644 --- a/annotations/views/text_views.py +++ b/annotations/views/text_views.py @@ -15,7 +15,6 @@ from annotations.forms import UploadFileForm from annotations.models import Text, Appellation, RelationSet from annotations.utils import basepath -from annotations.tasks import handle_file_upload from annotations.display_helpers import get_appellation_summaries @@ -181,45 +180,6 @@ def text(request, textid): return render(request, template, context) -#TODO: retire this view. -@login_required -def upload_file(request): - """ - Upload a file and save the text instance. - - Parameters - ---------- - request : `django.http.requests.HttpRequest` - - Returns - ---------- - :class:`django.http.response.HttpResponse` - """ - - project_id = request.GET.get('project', None) - - if request.method == 'POST': - form = UploadFileForm(request.POST, request.FILES) - form.fields['project'].queryset = form.fields['project'].queryset.filter(ownedBy_id=request.user.id) - if form.is_valid(): - - text = handle_file_upload(request, form) - return HttpResponseRedirect(reverse('text', args=[text.id]) + '?mode=annotate') - else: - form = UploadFileForm() - - form.fields['project'].queryset = form.fields['project'].queryset.filter(ownedBy_id=request.user.id) - if project_id: - form.fields['project'].initial = project_id - - template = "annotations/upload_file.html" - context = { - 'user': request.user, - 'form': form, - 'subpath': settings.SUBPATH, - } - return render(request, template, context) - def texts(request): qs = Text.objects.filter(Q(addedBy=request.user)) diff --git a/concepts/signals.py b/concepts/signals.py index c3c04088..53403268 100755 --- a/concepts/signals.py +++ b/concepts/signals.py @@ -1,7 +1,6 @@ from django.db.models.signals import post_save from django.dispatch import receiver -from concepts.tasks import resolve_concept from concepts.models import Concept, Type from django.conf import settings import logging @@ -21,7 +20,6 @@ def concept_post_save_receiver(sender, **kwargs): instance = kwargs.get('instance', None) if instance: logger.debug("Received post_save signal for Concept %s" % instance.uri) - resolve_concept.delay(instance.id) # # diff --git a/concepts/tasks.py b/concepts/tasks.py deleted file mode 100644 index 1301bfd6..00000000 --- a/concepts/tasks.py +++ /dev/null @@ -1,36 +0,0 @@ - - -from django.conf import settings -import logging -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(settings.LOGLEVEL) - -import requests - -from concepts.authorities import resolve, search, add -from concepts.models import Concept -from concepts.lifecycle import * - -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. -from celery import shared_task - - -@shared_task -def resolve_concept(instance_id): - """ - Since resolving concepts can involve several API calls, we handle it - asynchronously. - """ - try: - manager = ConceptLifecycle(Concept.objects.get(pk=instance_id)) - except Concept.DoesNotExist: - return - - try: - manager.resolve() - except ConceptLifecycleException as E: - logger.debug("Resolve concept failed: %s" % str(E)) - return - logger.debug("Resolved concept %s" % manager.instance.uri) diff --git a/requirements.txt b/requirements.txt index d83a4d67..e275ce66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ amqp==1.4.9 anyjson==0.3.3 +asgiref==3.2.10 billiard==3.3.0.23 black-goat-client==0.2.6 bleach==1.4.2 diff --git a/util/util.py b/util/util.py index 025b833d..f8b371ae 100644 --- a/util/util.py +++ b/util/util.py @@ -1,3 +1,5 @@ +from annotations.utils import tokenize + def unescape(s): return s.replace('&', '&')\ .replace('<', '<')\ @@ -40,7 +42,6 @@ def correctPosition(position): print(escaped[start:end], unescape(escaped)[start_o:end_o]) -from annotations.tasks import tokenize def correctTaggedPosition(position): text = position.occursIn manager = text.repository.manager(user) diff --git a/vogon/__init__.py b/vogon/__init__.py index b64e43e8..f45b4107 100644 --- a/vogon/__init__.py +++ b/vogon/__init__.py @@ -1,5 +1,4 @@ from __future__ import absolute_import # This will make sure the app is always imported when -# Django starts so that shared_task will use this app. -from .celery import app as celery_app +# Django starts so that shared_task will use this app. \ No newline at end of file diff --git a/vogon/celery.py b/vogon/celery.py deleted file mode 100644 index 2615eb52..00000000 --- a/vogon/celery.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import absolute_import - -import os - -from celery import Celery - -# set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'vogon.settings') - -from django.conf import settings - -app = Celery('vogon') - -# Using a string here means the worker will not have to -# pickle the object when using Windows. -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) -app.conf.update(BROKER_URL=os.environ.get('REDIS_URL', 'redis://localhost:6379/2'), - CELERY_RESULT_BACKEND=os.environ.get('REDIS_URL', 'redis://localhost:6379/2')) diff --git a/vogon/settings.py b/vogon/settings.py index 5f06a12c..d19c8129 100644 --- a/vogon/settings.py +++ b/vogon/settings.py @@ -196,17 +196,6 @@ } } -CONCEPTPOWER_USERID = os.environ.get('CONCEPTPOWER_USERID') -CONCEPTPOWER_PASSWORD = os.environ.get('CONCEPTPOWER_PASSWORD') -CONCEPTPOWER_ENDPOINT = os.environ.get('CONCEPTPOWER_ENDPOINT') -CONCEPTPOWER_NAMESPACE = os.environ.get('CONCEPTPOWER_NAMESPACE') - -QUADRIGA_USERID = os.environ.get('QUADRIGA_USERID', '') -QUADRIGA_PASSWORD = os.environ.get('QUADRIGA_PASSWORD', '') -QUADRIGA_ENDPOINT = os.environ.get('QUADRIGA_ENDPOINT', '') -QUADRIGA_CLIENTID = os.environ.get('QUADRIGA_CLIENTID', 'vogonweb') -QUADRIGA_PROJECT = os.environ.get('QUADRIGA_PROJECT', 'vogonweb') - BASE_URI_NAMESPACE = u'http://www.vogonweb.net' GOOGLE_ANALYTICS_ID = os.environ.get('GOOGLE_ANALYTICS_ID', None) @@ -215,11 +204,7 @@ LOGLEVEL = os.environ.get('LOGLEVEL', 'DEBUG') - -# Session Cookie Settings SESSION_COOKIE_NAME = 'vogon' -SESSION_COOKIE_AGE = 1209600 # 2 weeks (default) -SESSION_EXPIRE_AT_BROWSER_CLOSE = True # Concept types PERSONAL_CONCEPT_TYPE = os.environ.get('PERSONAL_CONCEPT_TYPE', @@ -236,6 +221,7 @@ } # Giles Credentials +# Giles and HTTP. GILES_ENDPOINT = os.environ.get('GILES_ENDPOINT') @@ -256,7 +242,17 @@ BASE_URL = os.path.join(os.getenv('BASE_URL', '/'), APP_ROOT) +# Conceptpower Credentials +CONCEPTPOWER_USERID = os.environ.get('CONCEPTPOWER_USERID') +CONCEPTPOWER_PASSWORD = os.environ.get('CONCEPTPOWER_PASSWORD') +CONCEPTPOWER_ENDPOINT = os.environ.get('CONCEPTPOWER_ENDPOINT') +CONCEPTPOWER_NAMESPACE = os.environ.get('CONCEPTPOWER_NAMESPACE') + +QUADRIGA_ENDPOINT = os.environ.get('QUADRIGA_ENDPOINT') + PAGINATION_PAGE_SIZE = 50 CITESPHERE_ITEM_PAGE = 50 + + REPOSITORY_TEXT_PAGINATION_PAGE_SIZE = 20 -PROJECT_TEXT_PAGINATION_PAGE_SIZE = 20 \ No newline at end of file +PROJECT_TEXT_PAGINATION_PAGE_SIZE = 20 diff --git a/vogon/urls.py b/vogon/urls.py index 1611670c..da693b02 100644 --- a/vogon/urls.py +++ b/vogon/urls.py @@ -121,12 +121,7 @@ re_path(r'^concept/(?P[0-9]+)/merge/$', conceptViews.merge_concepts, name='merge_concepts'), # url(r'^concept_autocomplete/', views.search_views.concept_autocomplete, name='concept_autocomplete'), - - re_path(r'^quadruples/appellation/(?P[0-9]+).xml$', views.quadruple_views.appellation_xml, name='appellation_xml'), - re_path(r'^quadruples/relation/(?P[0-9]+).xml$', views.quadruple_views.relation_xml, name='relation_xml'), - re_path(r'^quadruples/relationset/(?P[0-9]+).xml$', views.quadruple_views.relationset_xml, name='relationset_xml'), - re_path(r'^quadruples/text/(?P[0-9]+)/(?P[0-9]+).xml$', views.quadruple_views.text_xml, name='text_xml'), - + re_path(r'^repository/(?P[0-9]+)/collections/$', views.repository_views.repository_collections, name='repository_collections'), re_path(r'^repository/(?P[0-9]+)/browse/$', views.repository_views.repository_browse, name='repository_browse'), re_path(r'^repository/(?P[0-9]+)/search/$', views.repository_views.repository_search, name='repository_search'),