PKCS11 Support (#3)

* Added changes to enable 3.11 builds
* Fix xmlsec#244 - Fix failing test with libxmlsec-1.2.36, also make libxmlsec version available from Python.
* Fix xmlsec#164 - Add support for loading keys from engine (e.g. pkcs11).
* Fix xmlsec#164 - Add tests for pkcs11 (softhsm) key.
* [pre-commit.ci] auto fixes from pre-commit.com hooks
---------
Co-authored-by: Dan Vella <[email protected]>
Co-authored-by: Tomas Divis <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.appveyor.yml

@@ -16,6 +16,10 @@ environment:
     python_version: 3.10.6
   - python: 310-x64
     python_version: 3.10.6
+  - python: 311
+    python_version: 3.11.2
+  - python: 311-x64
+    python_version: 3.11.2
 
 install:
   - ps: |

.github/workflows/macosx.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"]
+        python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v3
       - name: Setup Python

.github/workflows/manylinux.yml

@@ -5,16 +5,16 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310]
+        python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311]
         image:
           - manylinux2010_x86_64
           - manylinux_2_24_x86_64
           - musllinux_1_1_x86_64
         exclude:
           - image: manylinux2010_x86_64
-            python-abi: cp310-cp310
+            python-abi: cp311-cp311
           - image: manylinux2010_i686
-            python-abi: cp310-cp310
+            python-abi: cp311-cp311
     container: quay.io/pypa/${{ matrix.image }}
     steps:
       - uses: actions/checkout@v1

.github/workflows/opensuse-tumbleweed.yml

@@ -6,7 +6,7 @@ jobs:
     container: opensuse/tumbleweed
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v1
       - name: Install build dependencies

.github/workflows/sdist.yml

@@ -5,20 +5,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install build dependencies
         run: |
           pip install --upgrade pip setuptools wheel
       - name: Package source dist
         run: |
           python setup.py sdist
       - name: Install test dependencies
-        env:
-          PYXMLSEC_STATIC_DEPS: true
         run: |
+          sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl
           pip install --upgrade -r requirements-test.txt
           pip install black  # for stub generation tests
           pip install dist/xmlsec-$(python setup.py --version).tar.gz

.travis.yml

@@ -16,6 +16,9 @@ matrix:
   - python: 3.9
     dist: xenial
     sudo: required
+  - python: 3.11
+    dist: xenial
+    sudo: required
 env:
   global:
   - CFLAGS=-coverage

README.rst

@@ -37,7 +37,7 @@ Check the `examples <https://xmlsec.readthedocs.io/en/latest/examples.html>`_ se
 Requirements
 ************
 - ``libxml2 >= 2.9.1``
-- ``libxmlsec1 >= 1.2.18``
+- ``libxmlsec1 >= 1.2.33``
 
 Install
 *******

setup.py

@@ -533,6 +533,7 @@ def prepare_static_build_linux(self):
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.11',
         'Topic :: Text Processing :: Markup :: XML',
         'Typing :: Typed',
     ],

src/keys.c

@@ -185,6 +185,47 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject*
     return NULL;
 }
 
+static const char PyXmlSec_KeyFromEngine__doc__[] = \
+    "from_engine(engine_and_key_id) -> xmlsec.Key\n"
+    "Loads PKI key from an engine.\n\n"
+    ":param engine_and_key_id: engine and key id, i.e. 'pkcs11;pkcs11:token=XmlsecToken;object=XmlsecKey;pin-value=password'\n"
+    ":type engine_and_key_id: :class:`str`, "
+    ":return: pointer to newly created key\n"
+    ":rtype: :class:`~xmlsec.Key`";
+static PyObject* PyXmlSec_KeyFromEngine(PyObject* self, PyObject* args, PyObject* kwargs) {
+    static char *kwlist[] = {"engine_and_key_id", NULL};
+
+    const char* engine_and_key_id = NULL;
+    PyXmlSec_Key* key = NULL;
+
+    PYXMLSEC_DEBUG("load key from engine - start");
+    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s:from_engine", kwlist, &engine_and_key_id)) {
+        goto ON_FAIL;
+    }
+
+    if ((key = PyXmlSec_NewKey1((PyTypeObject*)self)) == NULL) goto ON_FAIL;
+
+    Py_BEGIN_ALLOW_THREADS;
+    key->handle = xmlSecCryptoAppKeyLoad(engine_and_key_id, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(),
+                                         (void*)engine_and_key_id);
+    Py_END_ALLOW_THREADS;
+
+    if (key->handle == NULL) {
+        PyXmlSec_SetLastError("cannot read key");
+        goto ON_FAIL;
+    }
+
+    key->is_own = 1;
+
+    PYXMLSEC_DEBUG("load key from engine - ok");
+    return (PyObject*)key;
+
+ON_FAIL:
+    PYXMLSEC_DEBUG("load key from engine - fail");
+    Py_XDECREF(key);
+    return NULL;
+}
+
 static const char PyXmlSec_KeyGenerate__doc__[] = \
     "generate(klass, size, type) -> xmlsec.Key\n"
     "Generates key of kind ``klass`` with ``size`` and ``type``.\n\n"
@@ -494,6 +535,12 @@ static PyMethodDef PyXmlSec_KeyMethods[] = {
         METH_CLASS|METH_VARARGS|METH_KEYWORDS,
         PyXmlSec_KeyFromFile__doc__
     },
+    {
+        "from_engine",
+        (PyCFunction)PyXmlSec_KeyFromEngine,
+        METH_CLASS|METH_VARARGS|METH_KEYWORDS,
+        PyXmlSec_KeyFromEngine__doc__
+    },
     {
         "generate",
         (PyCFunction)PyXmlSec_KeyGenerate,

src/main.c

@@ -119,6 +119,13 @@ static PyObject* PyXmlSec_PyShutdown(PyObject* self) {
     Py_RETURN_NONE;
 }
 
+static char PyXmlSec_GetLibXmlSecVersion__doc__[] = \
+    "get_libxmlsec_version() -> tuple\n"
+    "Returns Version tuple of wrapped libxml library.";
+static PyObject* PyXmlSec_GetLibXmlSecVersion() {
+    return Py_BuildValue("(iii)", XMLSEC_VERSION_MAJOR, XMLSEC_VERSION_MINOR, XMLSEC_VERSION_SUBMINOR);
+}
+
 static char PyXmlSec_PyEnableDebugOutput__doc__[] = \
     "enable_debug_trace(enabled) -> None\n"
     "Enables or disables calling LibXML2 callback from the default errors callback.\n\n"
@@ -386,6 +393,12 @@ static PyMethodDef PyXmlSec_MainMethods[] = {
         METH_NOARGS,
         PyXmlSec_PyShutdown__doc__
     },
+    {
+        "get_libxmlsec_version",
+        (PyCFunction)PyXmlSec_GetLibXmlSecVersion,
+        METH_NOARGS,
+        PyXmlSec_GetLibXmlSecVersion__doc__
+    },
     {
         "enable_debug_trace",
         (PyCFunction)PyXmlSec_PyEnableDebugOutput,

src/xmlsec/__init__.pyi

@@ -49,6 +49,8 @@ class Key:
     @classmethod
     def from_file(cls: type[Self], file: GenericPath[AnyStr] | IO[AnyStr], format: int, password: str | None = ...) -> Self: ...
     @classmethod
+    def from_engine(cls: type[Self], engine_and_key_id: AnyStr) -> Self: ...
+    @classmethod
     def from_memory(cls: type[Self], data: AnyStr, format: int, password: str | None = ...) -> Self: ...
     @classmethod
     def generate(cls: type[Self], klass: KeyData, size: int, type: int) -> Self: ...

tests/data/sign5-out-xmlsec_1_2_36_to_37.xml

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+XML Security Library example: Signed XML doc file (sign5 example).
+-->
+<Envelope xmlns="urn:envelope">
+  <Data>
+	Hello, World!
+  </Data>
+<Signature xmlns="http://www.w3.org/2000/09/xmldsig#">
+<SignedInfo>
+<CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"/>
+<SignatureMethod Algorithm="http://www.w3.org/2000/09/xmldsig#rsa-sha1"/>
+<Reference>
+<Transforms>
+<Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/>
+</Transforms>
+<DigestMethod Algorithm="http://www.w3.org/2000/09/xmldsig#sha1"/>
+<DigestValue>HjY8ilZAIEM2tBbPn5mYO1ieIX4=</DigestValue>
+</Reference>
+</SignedInfo>
+<SignatureValue>SIaj/6KY3C1SmDXU2++Gm31U1xTadFp04WhBgfsJFbxrL+q7GKSKN9kfQ+UpN9+i
+D5fWmuavXEHe4Gw6RMaMEkq2URQo7F68+d5J/ajq8/l4n+xE6/reGScVwT6L4dEP
+XXVJcAi2ZnQ3O7GTNvNGCPibL9mUcyCWBFZ92Uemtc/vJFCQ7ZyKMdMfACgxOwyN
+T/9971oog241/2doudhonc0I/3mgPYWkZdX6yvr62mEjnG+oUZkhWYJ4ewZJ4hM4
+JjbFqZO+OEzDRSbw3DkmuBA/mtlx+3t13SESfEub5hqoMdVmtth/eTb64dsPdl9r
+3k1ACVX9f8aHfQQdJOmLFQ==</SignatureValue>
+<KeyInfo>
+<X509Data>
+
+
+
+<X509IssuerSerial>
+<X509IssuerName>Test Issuer</X509IssuerName>
+<X509SerialNumber>1</X509SerialNumber>
+</X509IssuerSerial>
+<X509Certificate>MIIE3zCCBEigAwIBAgIBBTANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCVVMx
+EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE
+ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v
+eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl
+a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tMB4X
+DTAzMDMzMTA0MDIyMloXDTEzMDMyODA0MDIyMlowgb8xCzAJBgNVBAYTAlVTMRMw
+EQYDVQQIEwpDYWxpZm9ybmlhMT0wOwYDVQQKEzRYTUwgU2VjdXJpdHkgTGlicmFy
+eSAoaHR0cDovL3d3dy5hbGVrc2V5LmNvbS94bWxzZWMpMSEwHwYDVQQLExhFeGFt
+cGxlcyBSU0EgQ2VydGlmaWNhdGUxFjAUBgNVBAMTDUFsZWtzZXkgU2FuaW4xITAf
+BgkqhkiG9w0BCQEWEnhtbHNlY0BhbGVrc2V5LmNvbTCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAJe4/rQ/gzV4FokE7CthjL/EXwCBSkXm2c3p4jyXO0Wt
+quaNC3dxBwFPfPl94hmq3ZFZ9PHPPbp4RpYRnLZbRjlzVSOq954AXOXpSew7nD+E
+mTqQrd9+ZIbGJnLOMQh5fhMVuOW/1lYCjWAhTCcYZPv7VXD2M70vVXDVXn6ZrqTg
+qkVHE6gw1aCKncwg7OSOUclUxX8+Zi10v6N6+PPslFc5tKwAdWJhVLTQ4FKG+F53
+7FBDnNK6p4xiWryy/vPMYn4jYGvHUUk3eH4lFTCr+rSuJY8i/KNIf/IKim7g/o3w
+Ae3GM8xrof2mgO8GjK/2QDqOQhQgYRIf4/wFsQXVZcMCAwEAAaOCAVcwggFTMAkG
+A1UdEwQCMAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRp
+ZmljYXRlMB0GA1UdDgQWBBQkhCzy1FkgYosuXIaQo6owuicanDCB+AYDVR0jBIHw
+MIHtgBS0ue+a5pcOaGUemM76VQ2JBttMfKGB0aSBzjCByzELMAkGA1UEBhMCVVMx
+EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE
+ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v
+eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl
+a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tggEA
+MA0GCSqGSIb3DQEBBAUAA4GBALU/mzIxSv8vhDuomxFcplzwdlLZbvSQrfoNkMGY
+1UoS3YJrN+jZLWKSyWE3mIaPpElqXiXQGGkwD5iPQ1iJMbI7BeLvx6ZxX/f+c8Wn
+ss0uc1NxfahMaBoyG15IL4+beqO182fosaKJTrJNG3mc//ANGU9OsQM9mfBEt4oL
+NJ2D</X509Certificate>
+<X509SubjectName/>
+<X509SKI/>
+</X509Data>
+</KeyInfo>
+</Signature></Envelope>