From c8c38640439ea4380650fa2d100f86d42a6fc4f5 Mon Sep 17 00:00:00 2001 From: Ivan Herman Date: Wed, 22 Jan 2020 17:16:12 +0100 Subject: [PATCH 1/6] Got the command line version running with Py3 --- pyRdfa/__init__.py | 17 ++++++++++++++--- pyRdfa/utils.py | 2 +- scripts/localRDFa.py | 10 ++++++---- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pyRdfa/__init__.py b/pyRdfa/__init__.py index 4833afc..635753f 100644 --- a/pyRdfa/__init__.py +++ b/pyRdfa/__init__.py @@ -312,7 +312,7 @@ class pyRdfaError(Exception) : # This comes from wikipedia registered_iana_schemes = [ - "aaa","aaas","acap","cap","cid","crid","data","dav","dict","dns","fax","file", "ftp","geo","go", + "aaa","aaas","acap","cap","cid","crid","data","dav","dict","did","dns","fax","file", "ftp","geo","go", "gopher","h323","http","https","iax","icap","im","imap","info","ipp","iris","ldap", "lsid", "mailto","mid","modem","msrp","msrps", "mtqp", "mupdate","news","nfs","nntp","opaquelocktoken", "pop","pres", "prospero","rstp","rsync", "service","shttp","sieve","sip","sips", "sms", "snmp", "soap", "tag", @@ -614,7 +614,12 @@ def copyErrors(tog, options) : if self.charset : # This means the HTTP header has provided a charset, or the # file is a local file when we suppose it to be a utf-8 - dom = parser.parse(input, override_encoding=self.charset) + # + # 2020-01-20, Ivan Herman + # for some reasons the python3 version ran into a problem with this html5lib call + # the override_encoding argument was not accepted. + # dom = parser.parse(input, override_encoding=self.charset) + dom = parser.parse(input) else : # No charset set. The HTMLLib parser tries to sniff into the # the file to find a meta header for the charset; if that @@ -700,8 +705,14 @@ def rdf_from_sources(self, names, outputFormat = "turtle", rdfOutput = False) : # the value of rdfOutput determines the reaction on exceptions... for name in names : self.graph_from_source(name, graph, rdfOutput) + retval = graph.serialize(format=outputFormat) - return retval + # Stupid difference between python2 and python3... + if PY3 : + return str(graph.serialize(format=outputFormat), encoding='utf-8') + else : + return graph.serialize(format=outputFormat) + def rdf_from_source(self, name, outputFormat = "turtle", rdfOutput = False) : """ diff --git a/pyRdfa/utils.py b/pyRdfa/utils.py index d9f118f..55c667b 100644 --- a/pyRdfa/utils.py +++ b/pyRdfa/utils.py @@ -82,7 +82,7 @@ def __init__(self, name, additional_headers = {}) : import requests # Switching off the verification is not cool. But, at least for now, too many - # sites still go wrong because the cerficates are not o.k. with request... + # sites still go wrong because the certificates are not o.k. with request... r = requests.get(url, headers=additional_headers, verify=False) self.data = r.content self.headers = r.headers diff --git a/scripts/localRDFa.py b/scripts/localRDFa.py index 3be7d4a..f461f45 100755 --- a/scripts/localRDFa.py +++ b/scripts/localRDFa.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Run the pyRdfa package locally, ie, on a local file @@ -49,7 +49,7 @@ """ def usage() : - print usageText % sys.argv[0] + print(usageText % sys.argv[0]) format = "turtle" extras = [] @@ -133,6 +133,8 @@ def usage() : processor = pyRdfa(options, base) if len(value) >= 1 : - print processor.rdf_from_sources(value, outputFormat = format, rdfOutput = rdfOutput) + print(processor.rdf_from_sources(value, outputFormat = format, rdfOutput = rdfOutput)) else : - print processor.rdf_from_source(sys.stdin, outputFormat = format, rdfOutput = rdfOutput) + print(processor.rdf_from_source(sys.stdin, outputFormat = format, rdfOutput = rdfOutput)) + + From 03080caae96d74341ae87fb2bb49a4b232321728 Mon Sep 17 00:00:00 2001 From: Ivan Herman Date: Wed, 22 Jan 2020 17:29:29 +0100 Subject: [PATCH 2/6] Syntactically upgraded the CGI script; version bumped --- pyRdfa/__init__.py | 4 +- scripts/CGI_RDFa.py | 162 ++++++++++++++++++++++++++++++-------------- 2 files changed, 114 insertions(+), 52 deletions(-) diff --git a/pyRdfa/__init__.py b/pyRdfa/__init__.py index 635753f..905df43 100644 --- a/pyRdfa/__init__.py +++ b/pyRdfa/__init__.py @@ -141,7 +141,7 @@ @summary: RDFa parser (distiller) -@requires: Python version 2.5 or up; 2.7 is preferred +@requires: Python version 2.8 or python 3.8 or up @requires: U{RDFLib}; version 3.X is preferred. @requires: U{html5lib} for the HTML5 parsing (note that version 1.0b1 and 1.0b2 should be avoided, it may lead to unicode encoding problems) @requires: U{httpheader}; however, a small modification had to make on the original file, so for this reason and to make distribution easier this module (single file) is added to the package. @@ -156,7 +156,7 @@ @var uri_schemes: List of registered (or widely used) URI schemes; used for warnings... """ -__version__ = "3.5.3" +__version__ = "4.0.0" __author__ = 'Ivan Herman' __contact__ = 'Ivan Herman, ivan@w3.org' __license__ = 'W3C® SOFTWARE NOTICE AND LICENSE, http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231' diff --git a/scripts/CGI_RDFa.py b/scripts/CGI_RDFa.py index 0fbf0bb..10658fb 100755 --- a/scripts/CGI_RDFa.py +++ b/scripts/CGI_RDFa.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.5 +#!/usr/local/bin/python3 # -*- coding: utf-8 -*- # Maintainer: Ivan Herman @@ -15,30 +15,78 @@ """ """ -$Id: RDFa.py,v 1.9 2012/03/12 11:06:47 ivan Exp $ +$Id: RDFa.py,v 1.27 2018/05/23 08:57:19 carcone Exp $ """ -__version__ = "3.0" +__version__ = "4.0.0" import cgi -import cgitb; cgitb.enable() +import cgitb import sys, os -import StringIO +#import StringIO #cgi.print_environ() if sys.platform == "darwin" : # this is my local machine - sys.path.insert(0,"/Users/ivan/W3C/dev/2004/PythonLib-IH") - sys.path.insert(0,"/Users/ivan/Library/Python") - sys.path.insert(0,"/Users/ivan/W3C/dev/2004/PythonLib-IH/rdfa-1.1") + sys.path.insert(0,'/Users/ivan/Library/Python') + sys.path.insert(0,'/Users/ivan/Library/Python/RDFa') os.environ['PyRdfaCacheDir'] = '/Users/ivan/.pyrdfa-cache' + cgitb.enable() + else : - # this is the server on W3C - sys.path.insert(0,"/usr/local/lib/python2.4/site-packages/PythonLib-IH") - sys.path.insert(0,"/usr/local/lib/python2.4/site-packages/PythonLib-IH/rdfa-1.1") + # This will have to be updated for the Python3 installation!!! + # webencodings pip3 should also be done!!! + sys.path.insert(0,"/usr/lib/python2.7/dist-packages") + sys.path.insert(0,'/home/ivan/lib/python') os.environ['PyRdfaCacheDir'] = '/usr/local/apache/cgi/cgi-bin-other/RDFa/data-local' + cgitb.enable(display=0, logdir="/home/nobody/tracebacks/") from pyRdfa import processURI, RDFaError +# Register the RDFa JSON-LD serializer; for some reasons installing via pip did not work +from rdflib.plugin import register, Serializer +register('json', Serializer, 'rdflib_jsonld.serializer', 'JsonLDSerializer') + +def err_message(msg) : + from cleanhtml import clean_print + print('Content-type: text/html; charset=utf-8') + print('Status: 400 Invalid Input') + print() + print("") + print("") + print("Error in RDFa processing") + print("") + print("

Error in distilling RDFa

") + print("

") + clean_print("pyRdfa cannot process this URI: %s", uri) + print("

") + if len(msg) != 0 : + print("

") + clean_print(msg) + print("

") + print("") + print("") + sys.exit(1) + + +def brett_test(uri) : + + if not sys.platform == "darwin" : + from checkremote import check_url_safety, UnsupportedResourceError + from urllib2 import HTTPError, URLError + try: + check_url_safety(uri) + except HTTPError as e: + err_message('HTTP Error with the error code: %s and the error message: "%s"' (e.code, e.reason)) + except URLError as e: + err_message('URL Error with the error message: "%s"' % e.reason) + except UnsupportedResourceError as e: + msg = e.args[0] + ": " + e.args[1] + err_message('Unsupported Resource Error with the error message "%s"' % msg) + except Exception as e: + l = len(e.args) + msg = "" if l == 0 else (e.args[0] if l == 1 else e.args) + err_message('Exception raised: "%s"' % msg) + # # to make this thing exist... uri = "" @@ -50,53 +98,67 @@ uri = "text:" else : if not "uri" in form : - print 'Content-type: text/html; charset=utf-8' - print 'Status: 400 Invalid Input' - print - print "" - print "" - print "Error in RDFa processing" - print "" - print "

Error in distilling RDFa

" - print "No URI has been specified" - print "" - print "" + print('Content-type: text/html; charset=utf-8') + print('Status: 400 Invalid Input') + print() + print("") + print("") + print("Error in RDFa processing") + print("") + print("

Error in distilling RDFa

") + print("

No URI has been specified

") + print("") + print("") sys.exit(1) try : - #uri = form["uri"].value uri = form.getfirst("uri") except : - print 'Content-type: text/html; charset=utf-8' - print 'Status: 400 Invalid Input' - print - print "" - print "" - print "Error in RDFa processing" - print "" - print "

Error in distilling RDFa

" - print "No URI has been specified" - print "" - print "" + print('Content-type: text/html; charset=utf-8') + print('Status: 400 Invalid Input') + print() + print("") + print("") + print("Error in RDFa processing") + print("") + print("

Error in distilling RDFa

") + print("

No URI has been specified

") + print("") + print("") sys.exit(1) if "validate" in form : from rdfavalidator import validateURI - print 'Content-Type: text/html; charset=utf-8' - print - print validateURI(uri, form) + if not (uri == 'text:' or uri == 'uploaded:') : + brett_test(uri) + print('Content-Type: text/html; charset=utf-8') + print() + print(validateURI(uri, form)) else : - # Thanks to Sergio and Diego for the idea and code for the referer branch - if uri == "referer" : - uri = os.getenv('HTTP_REFERER') - newuri = "http://www.w3.org/2012/pyRdfa/extract?uri=" + uri - print "Status: 302 Moved" - print "Location: " + newuri - print - else : - if "format" in form.keys() : - format = form.getfirst("format") + try : + # Thanks to Sergio and Diego for the idea and code for the referer branch + if uri == "referer" : + uri = os.getenv('HTTP_REFERER') + if uri is None: + newuri = "http://www.w3.org/2012/pyRdfa/no_referer.html" + else: + brett_test(uri) + newuri = "http://www.w3.org/2012/pyRdfa/extract?uri=" + uri + print("Status: 307 Moved Temporarily") + print("Location: " + newuri) + print() else : - format = "turtle" - retval = processURI(uri, format, form) - print retval + # last point of check: use Brett's script to check the validity of the URI + if not (uri == 'text:' or uri == 'uploaded:') : + brett_test(uri) + + if "format" in form.keys() : + format = form.getfirst("format") + else : + format = "turtle" + retval = processURI(uri, format, form) + print(retval) + except Exception as e : + l = len(e.args) + msg = "" if l == 0 else (e.args[0] if l == 1 else e.args) + err_message('Exception raised: "%s"' % msg) From f2b11f91dfc34d231f89e15ba6c685b361cbeaa8 Mon Sep 17 00:00:00 2001 From: Ivan Herman Date: Thu, 23 Jan 2020 18:16:07 +0100 Subject: [PATCH 3/6] Forgot to remove a serialization (was done twice, unnecessarily) --- pyRdfa/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyRdfa/__init__.py b/pyRdfa/__init__.py index 905df43..cd0bb71 100644 --- a/pyRdfa/__init__.py +++ b/pyRdfa/__init__.py @@ -706,7 +706,6 @@ def rdf_from_sources(self, names, outputFormat = "turtle", rdfOutput = False) : for name in names : self.graph_from_source(name, graph, rdfOutput) - retval = graph.serialize(format=outputFormat) # Stupid difference between python2 and python3... if PY3 : return str(graph.serialize(format=outputFormat), encoding='utf-8') From 980545454d32a6578047e749fcc66bacd0742918 Mon Sep 17 00:00:00 2001 From: Ivan Herman Date: Fri, 24 Jan 2020 15:52:10 +0100 Subject: [PATCH 4/6] Regenerated the documentation --- Doc-pyRdfa/class-tree.html | 2 +- Doc-pyRdfa/help.html | 2 +- Doc-pyRdfa/identifier-index.html | 2 +- Doc-pyRdfa/module-tree.html | 2 +- Doc-pyRdfa/pyRdfa-module.html | 6 +- Doc-pyRdfa/pyRdfa-pysrc.html | 679 +++++++++--------- Doc-pyRdfa/pyRdfa.FailedSource-class.html | 2 +- Doc-pyRdfa/pyRdfa.HTTPError-class.html | 2 +- Doc-pyRdfa/pyRdfa.ProcessingError-class.html | 2 +- Doc-pyRdfa/pyRdfa.RDFaError-class.html | 2 +- Doc-pyRdfa/pyRdfa.embeddedRDF-module.html | 2 +- Doc-pyRdfa/pyRdfa.embeddedRDF-pysrc.html | 2 +- Doc-pyRdfa/pyRdfa.host-module.html | 15 +- Doc-pyRdfa/pyRdfa.host-pysrc.html | 397 +++++----- .../pyRdfa.host.HostLanguage-class.html | 2 +- Doc-pyRdfa/pyRdfa.host.MediaTypes-class.html | 2 +- Doc-pyRdfa/pyRdfa.host.atom-module.html | 2 +- Doc-pyRdfa/pyRdfa.host.atom-pysrc.html | 2 +- Doc-pyRdfa/pyRdfa.host.html5-module.html | 2 +- Doc-pyRdfa/pyRdfa.host.html5-pysrc.html | 2 +- Doc-pyRdfa/pyRdfa.initialcontext-module.html | 2 +- Doc-pyRdfa/pyRdfa.initialcontext-pysrc.html | 93 +-- .../pyRdfa.initialcontext.Wrapper-class.html | 2 +- Doc-pyRdfa/pyRdfa.options-module.html | 2 +- Doc-pyRdfa/pyRdfa.options-pysrc.html | 2 +- Doc-pyRdfa/pyRdfa.options.Options-class.html | 2 +- .../pyRdfa.options.ProcessorGraph-class.html | 2 +- Doc-pyRdfa/pyRdfa.parse-module.html | 2 +- Doc-pyRdfa/pyRdfa.parse-pysrc.html | 2 +- Doc-pyRdfa/pyRdfa.property-module.html | 2 +- Doc-pyRdfa/pyRdfa.property-pysrc.html | 2 +- ...pyRdfa.property.ProcessProperty-class.html | 2 +- Doc-pyRdfa/pyRdfa.pyRdfa-class.html | 2 +- Doc-pyRdfa/pyRdfa.pyRdfaError-class.html | 2 +- Doc-pyRdfa/pyRdfa.rdflibparsers-module.html | 2 +- Doc-pyRdfa/pyRdfa.rdflibparsers-pysrc.html | 2 +- .../pyRdfa.rdflibparsers.HTurtle-class.html | 2 +- ...dfa.rdflibparsers.HTurtleParser-class.html | 2 +- ...Rdfa.rdflibparsers.RDFa10Parser-class.html | 2 +- ...pyRdfa.rdflibparsers.RDFaParser-class.html | 2 +- ...libparsers.StructuredDataParser-class.html | 2 +- Doc-pyRdfa/pyRdfa.rdfs-module.html | 16 +- Doc-pyRdfa/pyRdfa.rdfs-pysrc.html | 2 +- Doc-pyRdfa/pyRdfa.rdfs.cache-module.html | 2 +- Doc-pyRdfa/pyRdfa.rdfs.cache-pysrc.html | 2 +- .../pyRdfa.rdfs.cache.CachedVocab-class.html | 2 +- ...dfa.rdfs.cache.CachedVocabIndex-class.html | 2 +- Doc-pyRdfa/pyRdfa.rdfs.process-module.html | 2 +- Doc-pyRdfa/pyRdfa.rdfs.process-pysrc.html | 2 +- .../pyRdfa.rdfs.process.MiniOWL-class.html | 2 +- Doc-pyRdfa/pyRdfa.state-module.html | 4 +- Doc-pyRdfa/pyRdfa.state-pysrc.html | 2 +- .../pyRdfa.state.ExecutionContext-class.html | 2 +- .../pyRdfa.state.ListStructure-class.html | 2 +- Doc-pyRdfa/pyRdfa.termorcurie-module.html | 6 +- Doc-pyRdfa/pyRdfa.termorcurie-pysrc.html | 2 +- ...Rdfa.termorcurie.InitialContext-class.html | 2 +- .../pyRdfa.termorcurie.TermOrCurie-class.html | 2 +- Doc-pyRdfa/pyRdfa.transform-module.html | 2 +- Doc-pyRdfa/pyRdfa.transform-pysrc.html | 2 +- .../pyRdfa.transform.DublinCore-module.html | 2 +- .../pyRdfa.transform.DublinCore-pysrc.html | 2 +- .../pyRdfa.transform.OpenID-module.html | 2 +- Doc-pyRdfa/pyRdfa.transform.OpenID-pysrc.html | 2 +- Doc-pyRdfa/pyRdfa.transform.lite-module.html | 2 +- Doc-pyRdfa/pyRdfa.transform.lite-pysrc.html | 2 +- .../pyRdfa.transform.metaname-module.html | 2 +- .../pyRdfa.transform.metaname-pysrc.html | 2 +- .../pyRdfa.transform.prototype-module.html | 2 +- .../pyRdfa.transform.prototype-pysrc.html | 2 +- Doc-pyRdfa/pyRdfa.utils-module.html | 3 +- Doc-pyRdfa/pyRdfa.utils-pysrc.html | 6 +- Doc-pyRdfa/pyRdfa.utils.URIOpener-class.html | 2 +- Doc-pyRdfa/pyRdfaExtras-module.html | 2 +- Doc-pyRdfa/pyRdfaExtras-pysrc.html | 2 +- Doc-pyRdfa/pyRdfaExtras.MyGraph-class.html | 3 +- pyRdfa/host/__init__.py | 3 +- 77 files changed, 691 insertions(+), 670 deletions(-) diff --git a/Doc-pyRdfa/class-tree.html b/Doc-pyRdfa/class-tree.html index 18e006f..94a676d 100644 --- a/Doc-pyRdfa/class-tree.html +++ b/Doc-pyRdfa/class-tree.html @@ -201,7 +201,7 @@

Class Hierarchy

- Generated by Epydoc 3.0.1 on Tue Dec 4 17:36:20 2018 + Generated by Epydoc 3.0.1 on Fri Jan 24 15:50:14 2020 The Navigation Bar
- Generated by Epydoc 3.0.1 on Tue Dec 4 17:36:20 2018 + Generated by Epydoc 3.0.1 on Fri Jan 24 15:50:14 2020 Identifier Index
- Generated by Epydoc 3.0.1 on Tue Dec 4 17:36:20 2018 + Generated by Epydoc 3.0.1 on Fri Jan 24 15:50:14 2020 Module Hierarchy
- Generated by Epydoc 3.0.1 on Tue Dec 4 17:36:20 2018 + Generated by Epydoc 3.0.1 on Fri Jan 24 15:50:14 2020 Transformers
Requires:
  • - Python version 2.5 or up; 2.7 is preferred + Python version 2.8 or python 3.8 or up
  • RDFLib; version 3.X @@ -380,7 +380,7 @@

    Transformers

    target="_top">W3C® SOFTWARE NOTICE AND LICENSE

    Version: - 3.4.3 + 4.0.0

    Contact: Ivan Herman, ivan@w3.org @@ -2067,7 +2067,7 @@

    builtInTransformers

    - Generated by Epydoc 3.0.1 on Tue Dec 4 17:36:20 2018 + Generated by Epydoc 3.0.1 on Fri Jan 24 15:50:14 2020 Source Code for Package pyRdfa141 142 143 @summary: RDFa parser (distiller) -144 @requires: Python version 2.5 or up; 2.7 is preferred +144 @requires: Python version 2.8 or python 3.8 or up 145 @requires: U{RDFLib<http://rdflib.net>}; version 3.X is preferred. 146 @requires: U{html5lib<http://code.google.com/p/html5lib/>} for the HTML5 parsing (note that version 1.0b1 and 1.0b2 should be avoided, it may lead to unicode encoding problems) 147 @requires: U{httpheader<http://deron.meranda.us/python/httpheader/>}; however, a small modification had to make on the original file, so for this reason and to make distribution easier this module (single file) is added to the package. @@ -210,7 +210,7 @@

    Source Code for Package pyRdfa156 @var uri_schemes: List of registered (or widely used) URI schemes; used for warnings... 157 """ 158 -159 __version__ = "3.4.3" +159 __version__ = "4.0.0" 160 __author__ = 'Ivan Herman' 161 __contact__ = 'Ivan Herman, ivan@w3.org' 162 __license__ = 'W3C® SOFTWARE NOTICE AND LICENSE, http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231' @@ -443,7 +443,7 @@

    Source Code for Package pyRdfa312 313 # This comes from wikipedia 314 registered_iana_schemes = [ -315 "aaa","aaas","acap","cap","cid","crid","data","dav","dict","dns","fax","file", "ftp","geo","go", +315 "aaa","aaas","acap","cap","cid","crid","data","dav","dict","did","dns","fax","file", "ftp","geo","go", 316 "gopher","h323","http","https","iax","icap","im","imap","info","ipp","iris","ldap", "lsid", 317 "mailto","mid","modem","msrp","msrps", "mtqp", "mupdate","news","nfs","nntp","opaquelocktoken", 318 "pop","pres", "prospero","rstp","rsync", "service","shttp","sieve","sip","sips", "sms", "snmp", "soap", "tag", @@ -575,7 +575,7 @@

    Source Code for Package pyRdfa444 self.charset = 'utf-8' 445 break 446 self.options.set_host_language(self.media_type) -447 return file(name) +447 return open(name) 448 else : 449 return name 450 except HTTPError : @@ -746,37 +746,42 @@

    Source Code for Package pyRdfa614 if self.charset : 615 # This means the HTTP header has provided a charset, or the 616 # file is a local file when we suppose it to be a utf-8 -617 dom = parser.617 # +618 # 2020-01-20, Ivan Herman +619 # for some reasons the python3 version ran into a problem with this html5lib call +620 # the override_encoding argument was not accepted. +621 # dom = parser.parse(input, override_encoding=self.charset) +622 dom = parser.parse(input, override_encoding=self.charset) -618 else : -619 # No charset set. The HTMLLib parser tries to sniff into the -620 # the file to find a meta header for the charset; if that -621 # works, fine, otherwise it falls back on window-... -622 dom = parser.parse(input) +623 else : +624 # No charset set. The HTMLLib parser tries to sniff into the +625 # the file to find a meta header for the charset; if that +626 # works, fine, otherwise it falls back on window-... +627 dom = parser.parse(input) -623 -624 try : -625 if isstring : -626 input.close() -627 input = self._get_input(name) -628 else : -629 input.seek(0) -630 from .host import adjust_html_version -631 self.rdfa_version = adjust_html_version(input, self.rdfa_version) -632 except : -633 # if anything goes wrong, it is not really important; rdfa version stays what it was... -634 pass -635 -636 else : -637 from .host import adjust_xhtml_and_version -638 if isinstance(input, StringIO) or isinstance(input, file): -639 628 +629 try : +630 if isstring : +631 input.close() +632 input = self._get_input(name) +633 else : +634 input.seek(0) +635 from .host import adjust_html_version +636 self.rdfa_version = adjust_html_version(input, self.rdfa_version) +637 except : +638 # if anything goes wrong, it is not really important; rdfa version stays what it was... +639 pass +640 +641 else : +642 from .host import adjust_xhtml_and_version +643 if isinstance(input, StringIO) or isinstance(input, file): +644 Source Code for Package pyRdfaparse -640 else: -641 645 else: +646 parse = xml.dom.minidom.parseString -642 dom = 647 dom = parse(input) -643 (adjusted_host_language, version) = adjust_xhtml_and_version(dom, self.options.host_language, self.rdfa_version) -644 self.options.host_language = adjusted_host_language -645 self.rdfa_version = version -646 except ImportError : -647 msg = "HTML5 parser not available. Try installing html5lib <http://code.google.com/p/html5lib>" -648 raise ImportError(msg) -649 except Exception : -650 e = sys.exc_info()[1] -651 # These are various parsing exception. Per spec, this is a case when -652 # error triples MUST be returned, ie, the usage of rdfOutput (which switches between an HTML formatted -653 # return page or a graph with error triples) does not apply -654 err = self.options.add_error(str(e), context = name) -655 self.http_status = 400 -656 self.options.processor_graph.add_http_context(err, 400) -657 return copyErrors(graph, self.options) -658 -659 # If we got here, we have a DOM tree to operate on... -660 return self.648 (adjusted_host_language, version) = adjust_xhtml_and_version(dom, self.options.host_language, self.rdfa_version) +649 self.options.host_language = adjusted_host_language +650 self.rdfa_version = version +651 except ImportError : +652 msg = "HTML5 parser not available. Try installing html5lib <http://code.google.com/p/html5lib>" +653 raise ImportError(msg) +654 except Exception : +655 e = sys.exc_info()[1] +656 # These are various parsing exception. Per spec, this is a case when +657 # error triples MUST be returned, ie, the usage of rdfOutput (which switches between an HTML formatted +658 # return page or a graph with error triples) does not apply +659 err = self.options.add_error(str(e), context = name) +660 self.http_status = 400 +661 self.options.processor_graph.add_http_context(err, 400) +662 return copyErrors(graph, self.options) +663 +664 # If we got here, we have a DOM tree to operate on... +665 return self.graph_from_DOM(dom, graph, pgraph) -661 except Exception : -662 # Something nasty happened during the generation of the graph... -663 (a,b,c) = sys.exc_info() -664 sys.excepthook(a,b,c) -665 if isinstance(b, ImportError) : -666 self.http_status = None -667 else : -668 self.http_status = 500 -669 if not rdfOutput : raise b -670 err = self.options.add_error(str(b), context = name) -671 self.options.processor_graph.add_http_context(err, 500) -672 return copyErrors(graph, self.options) -673 -
    674 - def rdf_from_sources(self, names, outputFormat = "turtle", rdfOutput = False) : -
    675 """ -676 Extract and RDF graph from a list of RDFa sources and serialize them in one graph. The sources are parsed, the RDF -677 extracted, and serialization is done in the specified format. -678 @param names: list of sources, each can be a URI, a file name, or a file-like object -679 @keyword outputFormat: serialization format. Can be one of "turtle", "n3", "xml", "pretty-xml", "nt". "xml", "pretty-xml", "json" or "json-ld". "turtle" and "n3", "xml" and "pretty-xml", and "json" and "json-ld" are synonyms, respectively. Note that the JSON-LD serialization works with RDFLib 3.* only. -680 @keyword rdfOutput: controls what happens in case an exception is raised. If the value is False, the caller is responsible handling it; otherwise a graph is returned with an error message included in the processor graph -681 @type rdfOutput: boolean -682 @return: a serialized RDF Graph -683 @rtype: string -684 """ -685 # This is better because it gives access to the various, non-standard serializations -686 # If it does not work because the extra are not installed, fall back to the standard -687 # rdlib distribution... -688 -689 if rdflib.__version__ >= "3.0.0" : -690 graph = Graph() -691 else : -692 # We may need the extra utilities for older rdflib versions... -693 try : -694 from pyRdfaExtras import MyGraph -695 graph = MyGraph() -696 except : -697 graph = Graph() -698 -699 # graph.bind("xsd", Namespace('http://www.w3.org/2001/XMLSchema#')) -700 # the value of rdfOutput determines the reaction on exceptions... -701 for name in names : -702 self.graph_from_source(name, graph, rdfOutput) -703 retval = graph.serialize(format=outputFormat) -704 return retval -
    705 -
    706 - def rdf_from_source(self, name, outputFormat = "turtle", rdfOutput = False) : -
    707 """ -708 Extract and RDF graph from an RDFa source and serialize it in one graph. The source is parsed, the RDF -709 extracted, and serialization is done in the specified format. -710 @param name: a URI, a file name, or a file-like object -711 @keyword outputFormat: serialization format. Can be one of "turtle", "n3", "xml", "pretty-xml", "nt". "xml", "pretty-xml", or "json-ld". "turtle" and "n3", or "xml" and "pretty-xml" are synonyms, respectively. Note that the JSON-LD serialization works with RDFLib 3.* only. -712 @keyword rdfOutput: controls what happens in case an exception is raised. If the value is False, the caller is responsible handling it; otherwise a graph is returned with an error message included in the processor graph -713 @type rdfOutput: boolean -714 @return: a serialized RDF Graph -715 @rtype: string -716 """ -717 return self.rdf_from_sources([name], outputFormat, rdfOutput) -
    718 -719 ################################################# CGI Entry point -
    720 -def processURI(uri, outputFormat, form={}) : -
    721 """The standard processing of an RDFa uri options in a form; used as an entry point from a CGI call. -722 -723 The call accepts extra form options (i.e., HTTP GET options) as follows: -724 -725 - C{graph=[output|processor|output,processor|processor,output]} specifying which graphs are returned. Default: C{output} -726 - C{space_preserve=[true|false]} means that plain literals are normalized in terms of white spaces. Default: C{false} -727 - C{rfa_version} provides the RDFa version that should be used for distilling. The string should be of the form "1.0" or "1.1". Default is the highest version the current package implements, currently "1.1" -728 - C{host_language=[xhtml,html,xml]} : the host language. Used when files are uploaded or text is added verbatim, otherwise the HTTP return header should be used. Default C{xml} -729 - C{embedded_rdf=[true|false]} : whether embedded turtle or RDF/XML content should be added to the output graph. Default: C{false} -730 - C{vocab_expansion=[true|false]} : whether the vocabularies should be expanded through the restricted RDFS entailment. Default: C{false} -731 - C{vocab_cache=[true|false]} : whether vocab caching should be performed or whether it should be ignored and vocabulary files should be picked up every time. Default: C{false} -732 - C{vocab_cache_report=[true|false]} : whether vocab caching details should be reported. Default: C{false} -733 - C{vocab_cache_bypass=[true|false]} : whether vocab caches have to be regenerated every time. Default: C{false} -734 - C{rdfa_lite=[true|false]} : whether warnings should be generated for non RDFa Lite attribute usage. Default: C{false} -735 -736 @param uri: URI to access. Note that the C{text:} and C{uploaded:} fake URI values are treated separately; the former is for textual intput (in which case a StringIO is used to get the data) and the latter is for uploaded file, where the form gives access to the file directly. -737 @param outputFormat: serialization format, as defined by the package. Currently "xml", "turtle", "nt", or "json". Default is "turtle", also used if any other string is given. -738 @param form: extra call options (from the CGI call) to set up the local options -739 @type form: cgi FieldStorage instance -740 @return: serialized graph -741 @rtype: string -742 """ -743 def _get_option(param, compare_value, default) : -744 param_old = param.replace('_','-') -745 if param in list(form.keys()) : -746 val = form.getfirst(param).lower() -747 return val == compare_value -748 elif param_old in list(form.keys()) : -749 # this is to ensure the old style parameters are still valid... -750 # in the old days I used '-' in the parameters, the standard favours '_' -751 val = form.getfirst(param_old).lower() -752 return val == compare_value -753 else : -754 return default -
    755 -756 if uri == "uploaded:" : -757 input = form["uploaded"].file -758 base = "" -759 elif uri == "text:" : -760 input = StringIO(form.getfirst("text")) -761 base = "" -762 else : -763 input = uri -764 base = uri -765 -766 if "rdfa_version" in list(form.keys()) : -767 rdfa_version = form.getfirst("rdfa_version") -768 else : -769 rdfa_version = None -770 -771 # working through the possible options -772 # Host language: HTML, XHTML, or XML -773 # Note that these options should be used for the upload and inline version only in case of a form -774 # for real uris the returned content type should be used -775 if "host_language" in list(form.keys()) : -776 if form.getfirst("host_language").lower() == "xhtml" : -777 media_type = MediaTypes.xhtml -778 elif form.getfirst("host_language").lower() == "html" : -779 media_type = MediaTypes.html -780 elif form.getfirst("host_language").lower() == "svg" : -781 media_type = MediaTypes.svg -782 elif form.getfirst("host_language").lower() == "atom" : -783 media_type = MediaTypes.666 except Exception : +667 # Something nasty happened during the generation of the graph... +668 (a,b,c) = sys.exc_info() +669 sys.excepthook(a,b,c) +670 if isinstance(b, ImportError) : +671 self.http_status = None +672 else : +673 self.http_status = 500 +674 if not rdfOutput : raise b +675 err = self.options.add_error(str(b), context = name) +676 self.options.processor_graph.add_http_context(err, 500) +677 return copyErrors(graph, self.options) +678 +
    679 - def rdf_from_sources(self, names, outputFormat = "turtle", rdfOutput = False) : +
    680 """ +681 Extract and RDF graph from a list of RDFa sources and serialize them in one graph. The sources are parsed, the RDF +682 extracted, and serialization is done in the specified format. +683 @param names: list of sources, each can be a URI, a file name, or a file-like object +684 @keyword outputFormat: serialization format. Can be one of "turtle", "n3", "xml", "pretty-xml", "nt". "xml", "pretty-xml", "json" or "json-ld". "turtle" and "n3", "xml" and "pretty-xml", and "json" and "json-ld" are synonyms, respectively. Note that the JSON-LD serialization works with RDFLib 3.* only. +685 @keyword rdfOutput: controls what happens in case an exception is raised. If the value is False, the caller is responsible handling it; otherwise a graph is returned with an error message included in the processor graph +686 @type rdfOutput: boolean +687 @return: a serialized RDF Graph +688 @rtype: string +689 """ +690 # This is better because it gives access to the various, non-standard serializations +691 # If it does not work because the extra are not installed, fall back to the standard +692 # rdlib distribution... +693 +694 if rdflib.__version__ >= "3.0.0" : +695 graph = Graph() +696 else : +697 # We may need the extra utilities for older rdflib versions... +698 try : +699 from pyRdfaExtras import MyGraph +700 graph = MyGraph() +701 except : +702 graph = Graph() +703 +704 # graph.bind("xsd", Namespace('http://www.w3.org/2001/XMLSchema#')) +705 # the value of rdfOutput determines the reaction on exceptions... +706 for name in names : +707 self.graph_from_source(name, graph, rdfOutput) +708 +709 # Stupid difference between python2 and python3... +710 if PY3 : +711 return str(graph.serialize(format=outputFormat), encoding='utf-8') +712 else : +713 return graph.serialize(format=outputFormat) +
    714 +715 +
    716 - def rdf_from_source(self, name, outputFormat = "turtle", rdfOutput = False) : +
    717 """ +718 Extract and RDF graph from an RDFa source and serialize it in one graph. The source is parsed, the RDF +719 extracted, and serialization is done in the specified format. +720 @param name: a URI, a file name, or a file-like object +721 @keyword outputFormat: serialization format. Can be one of "turtle", "n3", "xml", "pretty-xml", "nt". "xml", "pretty-xml", or "json-ld". "turtle" and "n3", or "xml" and "pretty-xml" are synonyms, respectively. Note that the JSON-LD serialization works with RDFLib 3.* only. +722 @keyword rdfOutput: controls what happens in case an exception is raised. If the value is False, the caller is responsible handling it; otherwise a graph is returned with an error message included in the processor graph +723 @type rdfOutput: boolean +724 @return: a serialized RDF Graph +725 @rtype: string +726 """ +727 return self.rdf_from_sources([name], outputFormat, rdfOutput) +
    728 +729 ################################################# CGI Entry point +
    730 -def processURI(uri, outputFormat, form={}) : +
    731 """The standard processing of an RDFa uri options in a form; used as an entry point from a CGI call. +732 +733 The call accepts extra form options (i.e., HTTP GET options) as follows: +734 +735 - C{graph=[output|processor|output,processor|processor,output]} specifying which graphs are returned. Default: C{output} +736 - C{space_preserve=[true|false]} means that plain literals are normalized in terms of white spaces. Default: C{false} +737 - C{rfa_version} provides the RDFa version that should be used for distilling. The string should be of the form "1.0" or "1.1". Default is the highest version the current package implements, currently "1.1" +738 - C{host_language=[xhtml,html,xml]} : the host language. Used when files are uploaded or text is added verbatim, otherwise the HTTP return header should be used. Default C{xml} +739 - C{embedded_rdf=[true|false]} : whether embedded turtle or RDF/XML content should be added to the output graph. Default: C{false} +740 - C{vocab_expansion=[true|false]} : whether the vocabularies should be expanded through the restricted RDFS entailment. Default: C{false} +741 - C{vocab_cache=[true|false]} : whether vocab caching should be performed or whether it should be ignored and vocabulary files should be picked up every time. Default: C{false} +742 - C{vocab_cache_report=[true|false]} : whether vocab caching details should be reported. Default: C{false} +743 - C{vocab_cache_bypass=[true|false]} : whether vocab caches have to be regenerated every time. Default: C{false} +744 - C{rdfa_lite=[true|false]} : whether warnings should be generated for non RDFa Lite attribute usage. Default: C{false} +745 +746 @param uri: URI to access. Note that the C{text:} and C{uploaded:} fake URI values are treated separately; the former is for textual intput (in which case a StringIO is used to get the data) and the latter is for uploaded file, where the form gives access to the file directly. +747 @param outputFormat: serialization format, as defined by the package. Currently "xml", "turtle", "nt", or "json". Default is "turtle", also used if any other string is given. +748 @param form: extra call options (from the CGI call) to set up the local options +749 @type form: cgi FieldStorage instance +750 @return: serialized graph +751 @rtype: string +752 """ +753 def _get_option(param, compare_value, default) : +754 param_old = param.replace('_','-') +755 if param in list(form.keys()) : +756 val = form.getfirst(param).lower() +757 return val == compare_value +758 elif param_old in list(form.keys()) : +759 # this is to ensure the old style parameters are still valid... +760 # in the old days I used '-' in the parameters, the standard favours '_' +761 val = form.getfirst(param_old).lower() +762 return val == compare_value +763 else : +764 return default +
    765 +766 if uri == "uploaded:" : +767 input = form["uploaded"].file +768 base = "" +769 elif uri == "text:" : +770 input = StringIO(form.getfirst("text")) +771 base = "" +772 else : +773 input = uri +774 base = uri +775 +776 if "rdfa_version" in list(form.keys()) : +777 rdfa_version = form.getfirst("rdfa_version") +778 else : +779 rdfa_version = None +780 +781 # working through the possible options +782 # Host language: HTML, XHTML, or XML +783 # Note that these options should be used for the upload and inline version only in case of a form +784 # for real uris the returned content type should be used +785 if "host_language" in list(form.keys()) : +786 if form.getfirst("host_language").lower() == "xhtml" : +787 media_type = MediaTypes.xhtml +788 elif form.getfirst("host_language").lower() == "html" : +789 media_type = MediaTypes.html +790 elif form.getfirst("host_language").lower() == "svg" : +791 media_type = MediaTypes.svg +792 elif form.getfirst("host_language").lower() == "atom" : +793 media_type = MediaTypes.atom -784 else : -785 media_type = MediaTypes.xml -786 else : -787 media_type = "" -788 -789 transformers = [] -790 -791 check_lite = "rdfa_lite" in list(form.keys()) and form.getfirst("rdfa_lite").lower() == "true" -792 -793 # The code below is left for backward compatibility only. In fact, these options are not exposed any more, -794 # they are not really in use -795 if "extras" in list(form.keys()) and form.getfirst("extras").lower() == "true" : -796 from .transform.metaname import meta_transform -797 from .transform.OpenID import OpenID_transform -798 from .transform.DublinCore import DC_transform -799 for t in [OpenID_transform, DC_transform, meta_transform] : -800 transformers.append(t) -801 else : -802 if "extra-meta" in list(form.keys()) and form.getfirst("extra-meta").lower() == "true" : -803 from .transform.metaname import meta_transform -804 transformers.append(meta_transform) -805 if "extra-openid" in list(form.keys()) and form.getfirst("extra-openid").lower() == "true" : -806 from .transform.OpenID import OpenID_transform -807 transformers.append(OpenID_transform) -808 if "extra-dc" in list(form.keys()) and form.getfirst("extra-dc").lower() == "true" : -809 from .transform.DublinCore import DC_transform -810 transformers.append(DC_transform) -811 -812 output_default_graph = True -813 output_processor_graph = False -814 # Note that I use the 'graph' and the 'rdfagraph' form keys here. Reason is that -815 # I used 'graph' in the previous versions, including the RDFa 1.0 processor, -816 # so if I removed that altogether that would create backward incompatibilities -817 # On the other hand, the RDFa 1.1 doc clearly refers to 'rdfagraph' as the standard -818 # key. -819 a = None -820 if "graph" in list(form.keys()) : -821 a = form.getfirst("graph").lower() -822 elif "rdfagraph" in list(form.keys()) : -823 a = form.getfirst("rdfagraph").lower() -824 if a != None : -825 if a == "processor" : -826 output_default_graph = False -827 output_processor_graph = True -828 elif a == "processor,output" or a == "output,processor" : -829 output_processor_graph = True -830 -831 embedded_rdf = _get_option( "embedded_rdf", "true", False) -832 space_preserve = _get_option( "space_preserve", "true", True) -833 vocab_cache = _get_option( "vocab_cache", "true", True) -834 vocab_cache_report = _get_option( "vocab_cache_report", "true", False) -835 refresh_vocab_cache = _get_option( "vocab_cache_refresh", "true", False) -836 vocab_expansion = _get_option( "vocab_expansion", "true", False) -837 if vocab_cache_report : output_processor_graph = True -838 -839 options = Options(output_default_graph = output_default_graph, -840 output_processor_graph = output_processor_graph, -841 space_preserve = space_preserve, -842 transformers = transformers, -843 vocab_cache = vocab_cache, -844 vocab_cache_report = vocab_cache_report, -845 refresh_vocab_cache = refresh_vocab_cache, -846 vocab_expansion = vocab_expansion, -847 embedded_rdf = embedded_rdf, -848 check_lite = check_lite -849 ) -850 processor = pyRdfa(options = options, base = base, media_type = media_type, rdfa_version = rdfa_version) -851 -852 # Decide the output format; the issue is what should happen in case of a top level error like an inaccessibility of -853 # the html source: should a graph be returned or an HTML page with an error message? -854 -855 # decide whether HTML or RDF should be sent. -856 htmlOutput = False -857 #if 'HTTP_ACCEPT' in os.environ : -858 # acc = os.environ['HTTP_ACCEPT'] -859 # possibilities = ['text/html', -860 # 'application/rdf+xml', -861 # 'text/turtle; charset=utf-8', -862 # 'application/json', -863 # 'application/ld+json', -864 # 'text/rdf+n3'] -865 # -866 # # this nice module does content negotiation and returns the preferred format -867 # sg = acceptable_content_type(acc, possibilities) -868 # htmlOutput = (sg != None and sg[0] == content_type('text/html')) -869 # os.environ['rdfaerror'] = 'true' -870 -871 # This is really for testing purposes only, it is an unpublished flag to force RDF output no -872 # matter what -873 try : -874 graph = processor.rdf_from_source(input, outputFormat, rdfOutput = ("forceRDFOutput" in list(form.keys())) or not htmlOutput) -875 if outputFormat == "n3" : -876 retval = 'Content-Type: text/rdf+n3; charset=utf-8\n' -877 elif outputFormat == "nt" or outputFormat == "turtle" : -878 retval = 'Content-Type: text/turtle; charset=utf-8\n' -879 elif outputFormat == "json-ld" or outputFormat == "json" : -880 retval = 'Content-Type: application/ld+json; charset=utf-8\n' -881 else : -882 retval = 'Content-Type: application/rdf+xml; charset=utf-8\n' -883 retval += '\n' -884 retval += graph -885 return retval -886 except HTTPError : -887 (type,h,traceback) = sys.exc_info() -888 import cgi -889 -890 retval = 'Content-type: text/html; charset=utf-8\nStatus: %s \n\n' % h.http_code -891 retval += "<html>\n" -892 retval += "<head>\n" -893 retval += "<title>HTTP Error in distilling RDFa content</title>\n" -894 retval += "</head><body>\n" -895 retval += "<h1>HTTP Error in distilling RDFa content</h1>\n" -896 retval += "<p>HTTP Error: %s (%s)</p>\n" % (h.http_code,h.msg) -897 retval += "<p>On URI: <code>'%s'</code></p>\n" % cgi.escape(uri) -898 retval +="</body>\n" -899 retval +="</html>\n" -900 return retval -901 except : -902 # This branch should occur only if an exception is really raised, ie, if it is not turned -903 # into a graph value. -904 (type,value,traceback) = sys.exc_info() -905 -906 import traceback, cgi -907 -908 retval = 'Content-type: text/html; charset=utf-8\nStatus: %s\n\n' % processor.http_status -909 retval += "<html>\n" -910 retval += "<head>\n" -911 retval += "<title>Exception in RDFa processing</title>\n" -912 retval += "</head><body>\n" -913 retval += "<h1>Exception in distilling RDFa</h1>\n" -914 retval += "<pre>\n" -915 strio = StringIO() -916 traceback.print_exc(file=strio) -917 retval += strio.getvalue() -918 retval +="</pre>\n" -919 retval +="<pre>%s</pre>\n" % value -920 retval +="<h1>Distiller request details</h1>\n" -921 retval +="<dl>\n" -922 if uri == "text:" and "text" in form and form["text"].value != None and len(form["text"].value.strip()) != 0 : -923 retval +="<dt>Text input:</dt><dd>%s</dd>\n" % cgi.escape(form["text"].value).replace('\n','<br/>') -924 elif uri == "uploaded:" : -925 retval +="<dt>Uploaded file</dt>\n" -926 else : -927 retval +="<dt>URI received:</dt><dd><code>'%s'</code></dd>\n" % cgi.escape(uri) -928 if "host_language" in list(form.keys()) : -929 retval +="<dt>Media Type:</dt><dd>%s</dd>\n" % media_type -930 if "graph" in list(form.keys()) : -931 retval +="<dt>Requested graphs:</dt><dd>%s</dd>\n" % form.getfirst("graph").lower() -932 else : -933 retval +="<dt>Requested graphs:</dt><dd>default</dd>\n" -934 retval +="<dt>Output serialization format:</dt><dd> %s</dd>\n" % outputFormat -935 if "space_preserve" in form : retval +="<dt>Space preserve:</dt><dd> %s</dd>\n" % form["space_preserve"].value -936 retval +="</dl>\n" -937 retval +="</body>\n" -938 retval +="</html>\n" -939 return retval -940