From 52392a2445ed99abcf993b759c448d8a47285433 Mon Sep 17 00:00:00 2001 From: William Murphy Date: Thu, 25 May 2023 15:35:31 -0400 Subject: [PATCH] Add mariner data source (#181) Add functionality to vunnel to ingest published vulnerabilities with CBL Mariner Linux. Signed-off-by: Will Murphy --- poetry.lock | 263 +++++++++- pyproject.toml | 4 + src/vunnel/cli/config.py | 1 + src/vunnel/providers/__init__.py | 2 + src/vunnel/providers/mariner/DEVELOPING.md | 102 ++++ src/vunnel/providers/mariner/__init__.py | 55 ++ .../providers/mariner/generate_models.py | 47 ++ .../providers/mariner/model/__init__.py | 41 ++ .../providers/mariner/model/generated.py | 480 ++++++++++++++++++ src/vunnel/providers/mariner/parser.py | 198 ++++++++ src/vunnel/utils/vulnerability.py | 6 + tests/quality/config.yaml | 4 + tests/quality/vulnerability-match-labels | 2 +- tests/unit/cli/test_cli.py | 15 + tests/unit/providers/mariner/__init__.py | 0 .../mariner-truncated-2.0-oval.xml | 97 ++++ tests/unit/providers/mariner/test_mariner.py | 90 ++++ 17 files changed, 1402 insertions(+), 5 deletions(-) create mode 100644 src/vunnel/providers/mariner/DEVELOPING.md create mode 100644 src/vunnel/providers/mariner/__init__.py create mode 100644 src/vunnel/providers/mariner/generate_models.py create mode 100644 src/vunnel/providers/mariner/model/__init__.py create mode 100644 src/vunnel/providers/mariner/model/generated.py create mode 100644 src/vunnel/providers/mariner/parser.py create mode 100644 tests/unit/providers/mariner/__init__.py create mode 100644 tests/unit/providers/mariner/test-fixtures/mariner-truncated-2.0-oval.xml create mode 100644 tests/unit/providers/mariner/test_mariner.py diff --git a/poetry.lock b/poetry.lock index 5dc5864a..0a4ec724 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "attrs" @@ -193,6 +193,20 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "click-default-group" +version = "1.2.2" +description = "Extends click.Group to invoke a command without explicit subcommand name" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "click-default-group-1.2.2.tar.gz", hash = "sha256:d9560e8e8dfa44b3562fbc9425042a0fd6d21956fcc2db0077f63f34253ab904"}, +] + +[package.dependencies] +click = "*" + [[package]] name = "colorama" version = "0.4.6" @@ -376,6 +390,25 @@ files = [ {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] +[[package]] +name = "docformatter" +version = "1.5.0" +description = "Formats docstrings to follow PEP 257" +category = "main" +optional = false +python-versions = ">=3.6,<4.0" +files = [ + {file = "docformatter-1.5.0-py3-none-any.whl", hash = "sha256:ae56c64822c3184602ac83ec37650c9785e80dfec17b4eba4f49ad68815d71c0"}, + {file = "docformatter-1.5.0.tar.gz", hash = "sha256:9dc71659d3b853c3018cd7b2ec34d5d054370128e12b79ee655498cb339cc711"}, +] + +[package.dependencies] +tomli = {version = ">=2.0.0,<3.0.0", markers = "python_version >= \"3.7\""} +untokenize = ">=0.1.1,<0.2.0" + +[package.extras] +tomli = ["tomli (<2.0.0)"] + [[package]] name = "dunamai" version = "1.17.0" @@ -633,6 +666,24 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + [[package]] name = "jsonschema" version = "4.17.3" @@ -653,6 +704,99 @@ pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] +[[package]] +name = "lxml" +version = "4.9.2" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" +files = [ + {file = "lxml-4.9.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:76cf573e5a365e790396a5cc2b909812633409306c6531a6877c59061e42c4f2"}, + {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b1f42b6921d0e81b1bcb5e395bc091a70f41c4d4e55ba99c6da2b31626c44892"}, + {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9f102706d0ca011de571de32c3247c6476b55bb6bc65a20f682f000b07a4852a"}, + {file = "lxml-4.9.2-cp27-cp27m-win32.whl", hash = "sha256:8d0b4612b66ff5d62d03bcaa043bb018f74dfea51184e53f067e6fdcba4bd8de"}, + {file = "lxml-4.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:4c8f293f14abc8fd3e8e01c5bd86e6ed0b6ef71936ded5bf10fe7a5efefbaca3"}, + {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2899456259589aa38bfb018c364d6ae7b53c5c22d8e27d0ec7609c2a1ff78b50"}, + {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6749649eecd6a9871cae297bffa4ee76f90b4504a2a2ab528d9ebe912b101975"}, + {file = "lxml-4.9.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a08cff61517ee26cb56f1e949cca38caabe9ea9fbb4b1e10a805dc39844b7d5c"}, + {file = "lxml-4.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:85cabf64adec449132e55616e7ca3e1000ab449d1d0f9d7f83146ed5bdcb6d8a"}, + {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8340225bd5e7a701c0fa98284c849c9b9fc9238abf53a0ebd90900f25d39a4e4"}, + {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:1ab8f1f932e8f82355e75dda5413a57612c6ea448069d4fb2e217e9a4bed13d4"}, + {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:699a9af7dffaf67deeae27b2112aa06b41c370d5e7633e0ee0aea2e0b6c211f7"}, + {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9cc34af337a97d470040f99ba4282f6e6bac88407d021688a5d585e44a23184"}, + {file = "lxml-4.9.2-cp310-cp310-win32.whl", hash = "sha256:d02a5399126a53492415d4906ab0ad0375a5456cc05c3fc0fc4ca11771745cda"}, + {file = "lxml-4.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:a38486985ca49cfa574a507e7a2215c0c780fd1778bb6290c21193b7211702ab"}, + {file = "lxml-4.9.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c83203addf554215463b59f6399835201999b5e48019dc17f182ed5ad87205c9"}, + {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2a87fa548561d2f4643c99cd13131acb607ddabb70682dcf1dff5f71f781a4bf"}, + {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:d6b430a9938a5a5d85fc107d852262ddcd48602c120e3dbb02137c83d212b380"}, + {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3efea981d956a6f7173b4659849f55081867cf897e719f57383698af6f618a92"}, + {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:df0623dcf9668ad0445e0558a21211d4e9a149ea8f5666917c8eeec515f0a6d1"}, + {file = "lxml-4.9.2-cp311-cp311-win32.whl", hash = "sha256:da248f93f0418a9e9d94b0080d7ebc407a9a5e6d0b57bb30db9b5cc28de1ad33"}, + {file = "lxml-4.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:3818b8e2c4b5148567e1b09ce739006acfaa44ce3156f8cbbc11062994b8e8dd"}, + {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca989b91cf3a3ba28930a9fc1e9aeafc2a395448641df1f387a2d394638943b0"}, + {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:822068f85e12a6e292803e112ab876bc03ed1f03dddb80154c395f891ca6b31e"}, + {file = "lxml-4.9.2-cp35-cp35m-win32.whl", hash = "sha256:be7292c55101e22f2a3d4d8913944cbea71eea90792bf914add27454a13905df"}, + {file = "lxml-4.9.2-cp35-cp35m-win_amd64.whl", hash = "sha256:998c7c41910666d2976928c38ea96a70d1aa43be6fe502f21a651e17483a43c5"}, + {file = "lxml-4.9.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:b26a29f0b7fc6f0897f043ca366142d2b609dc60756ee6e4e90b5f762c6adc53"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:ab323679b8b3030000f2be63e22cdeea5b47ee0abd2d6a1dc0c8103ddaa56cd7"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:689bb688a1db722485e4610a503e3e9210dcc20c520b45ac8f7533c837be76fe"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f49e52d174375a7def9915c9f06ec4e569d235ad428f70751765f48d5926678c"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:36c3c175d34652a35475a73762b545f4527aec044910a651d2bf50de9c3352b1"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a35f8b7fa99f90dd2f5dc5a9fa12332642f087a7641289ca6c40d6e1a2637d8e"}, + {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:58bfa3aa19ca4c0f28c5dde0ff56c520fbac6f0daf4fac66ed4c8d2fb7f22e74"}, + {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc718cd47b765e790eecb74d044cc8d37d58562f6c314ee9484df26276d36a38"}, + {file = "lxml-4.9.2-cp36-cp36m-win32.whl", hash = "sha256:d5bf6545cd27aaa8a13033ce56354ed9e25ab0e4ac3b5392b763d8d04b08e0c5"}, + {file = "lxml-4.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:3ab9fa9d6dc2a7f29d7affdf3edebf6ece6fb28a6d80b14c3b2fb9d39b9322c3"}, + {file = "lxml-4.9.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:05ca3f6abf5cf78fe053da9b1166e062ade3fa5d4f92b4ed688127ea7d7b1d03"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:a5da296eb617d18e497bcf0a5c528f5d3b18dadb3619fbdadf4ed2356ef8d941"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:04876580c050a8c5341d706dd464ff04fd597095cc8c023252566a8826505726"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c9ec3eaf616d67db0764b3bb983962b4f385a1f08304fd30c7283954e6a7869b"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2a29ba94d065945944016b6b74e538bdb1751a1db6ffb80c9d3c2e40d6fa9894"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a82d05da00a58b8e4c0008edbc8a4b6ec5a4bc1e2ee0fb6ed157cf634ed7fa45"}, + {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:223f4232855ade399bd409331e6ca70fb5578efef22cf4069a6090acc0f53c0e"}, + {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d17bc7c2ccf49c478c5bdd447594e82692c74222698cfc9b5daae7ae7e90743b"}, + {file = "lxml-4.9.2-cp37-cp37m-win32.whl", hash = "sha256:b64d891da92e232c36976c80ed7ebb383e3f148489796d8d31a5b6a677825efe"}, + {file = "lxml-4.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a0a336d6d3e8b234a3aae3c674873d8f0e720b76bc1d9416866c41cd9500ffb9"}, + {file = "lxml-4.9.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:da4dd7c9c50c059aba52b3524f84d7de956f7fef88f0bafcf4ad7dde94a064e8"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:821b7f59b99551c69c85a6039c65b75f5683bdc63270fec660f75da67469ca24"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:e5168986b90a8d1f2f9dc1b841467c74221bd752537b99761a93d2d981e04889"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8e20cb5a47247e383cf4ff523205060991021233ebd6f924bca927fcf25cf86f"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:13598ecfbd2e86ea7ae45ec28a2a54fb87ee9b9fdb0f6d343297d8e548392c03"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:880bbbcbe2fca64e2f4d8e04db47bcdf504936fa2b33933efd945e1b429bea8c"}, + {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7d2278d59425777cfcb19735018d897ca8303abe67cc735f9f97177ceff8027f"}, + {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5344a43228767f53a9df6e5b253f8cdca7dfc7b7aeae52551958192f56d98457"}, + {file = "lxml-4.9.2-cp38-cp38-win32.whl", hash = "sha256:925073b2fe14ab9b87e73f9a5fde6ce6392da430f3004d8b72cc86f746f5163b"}, + {file = "lxml-4.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:9b22c5c66f67ae00c0199f6055705bc3eb3fcb08d03d2ec4059a2b1b25ed48d7"}, + {file = "lxml-4.9.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5f50a1c177e2fa3ee0667a5ab79fdc6b23086bc8b589d90b93b4bd17eb0e64d1"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:090c6543d3696cbe15b4ac6e175e576bcc3f1ccfbba970061b7300b0c15a2140"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:63da2ccc0857c311d764e7d3d90f429c252e83b52d1f8f1d1fe55be26827d1f4"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:5b4545b8a40478183ac06c073e81a5ce4cf01bf1734962577cf2bb569a5b3bbf"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2e430cd2824f05f2d4f687701144556646bae8f249fd60aa1e4c768ba7018947"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6804daeb7ef69e7b36f76caddb85cccd63d0c56dedb47555d2fc969e2af6a1a5"}, + {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a6e441a86553c310258aca15d1c05903aaf4965b23f3bc2d55f200804e005ee5"}, + {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ca34efc80a29351897e18888c71c6aca4a359247c87e0b1c7ada14f0ab0c0fb2"}, + {file = "lxml-4.9.2-cp39-cp39-win32.whl", hash = "sha256:6b418afe5df18233fc6b6093deb82a32895b6bb0b1155c2cdb05203f583053f1"}, + {file = "lxml-4.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1496ea22ca2c830cbcbd473de8f114a320da308438ae65abad6bab7867fe38f"}, + {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b264171e3143d842ded311b7dccd46ff9ef34247129ff5bf5066123c55c2431c"}, + {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0dc313ef231edf866912e9d8f5a042ddab56c752619e92dfd3a2c277e6a7299a"}, + {file = "lxml-4.9.2-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:16efd54337136e8cd72fb9485c368d91d77a47ee2d42b057564aae201257d419"}, + {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0f2b1e0d79180f344ff9f321327b005ca043a50ece8713de61d1cb383fb8ac05"}, + {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:7b770ed79542ed52c519119473898198761d78beb24b107acf3ad65deae61f1f"}, + {file = "lxml-4.9.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efa29c2fe6b4fdd32e8ef81c1528506895eca86e1d8c4657fda04c9b3786ddf9"}, + {file = "lxml-4.9.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7e91ee82f4199af8c43d8158024cbdff3d931df350252288f0d4ce656df7f3b5"}, + {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b23e19989c355ca854276178a0463951a653309fb8e57ce674497f2d9f208746"}, + {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:01d36c05f4afb8f7c20fd9ed5badca32a2029b93b1750f571ccc0b142531caf7"}, + {file = "lxml-4.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7b515674acfdcadb0eb5d00d8a709868173acece5cb0be3dd165950cbfdf5409"}, + {file = "lxml-4.9.2.tar.gz", hash = "sha256:2455cfaeb7ac70338b3257f41e21f0724f4b5b0c0e7702da67ee6c3640835b67"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=0.29.7)"] + [[package]] name = "mando" version = "0.6.4" @@ -696,6 +840,66 @@ profiling = ["gprof2dot"] rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] +[[package]] +name = "markupsafe" +version = "2.1.2" +description = "Safely add untrusted strings to HTML/XML markup." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, + {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, +] + [[package]] name = "marshmallow" version = "3.19.0" @@ -1444,7 +1648,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] @@ -1501,7 +1705,7 @@ tests = ["pytest", "pytest-cov"] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1509,6 +1713,18 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "toposort" +version = "1.10" +description = "Implements a topological sort algorithm." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "toposort-1.10-py3-none-any.whl", hash = "sha256:cbdbc0d0bee4d2695ab2ceec97fe0679e9c10eab4b2a87a9372b929e70563a87"}, + {file = "toposort-1.10.tar.gz", hash = "sha256:bfbb479c53d0a696ea7402601f4e693c97b0367837c8898bc6471adfca37a6bd"}, +] + [[package]] name = "types-pyyaml" version = "6.0.12.10" @@ -1576,6 +1792,17 @@ files = [ mypy-extensions = ">=0.3.0" typing-extensions = ">=3.7.4" +[[package]] +name = "untokenize" +version = "0.1.1" +description = "Transforms tokens into original source code (while preserving whitespace)." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "untokenize-0.1.1.tar.gz", hash = "sha256:3865dbbbb8efb4bb5eaa72f1be7f3e0be00ea8b7f125c69cbd1f5fda926f37a2"}, +] + [[package]] name = "urllib3" version = "1.26.14" @@ -1626,6 +1853,34 @@ files = [ {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, ] +[[package]] +name = "xsdata" +version = "22.12" +description = "Python XML Binding" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "xsdata-22.12-py3-none-any.whl", hash = "sha256:981755b69148fe954c46f4f6eb12f441e915df403ba86b21165e444667970cc1"}, + {file = "xsdata-22.12.tar.gz", hash = "sha256:a3d5f1b7b6fff8c916f7825c836ea285a4e7d3f3a94dcbbed0e63ba15dc94466"}, +] + +[package.dependencies] +click = {version = ">=5.0", optional = true, markers = "extra == \"cli\""} +click-default-group = {version = ">=1.2", optional = true, markers = "extra == \"cli\""} +docformatter = {version = "1.5.0", optional = true, markers = "extra == \"cli\""} +jinja2 = {version = ">=2.10", optional = true, markers = "extra == \"cli\""} +lxml = {version = ">=4.4.1", optional = true, markers = "extra == \"lxml\""} +requests = {version = "*", optional = true, markers = "extra == \"soap\""} +toposort = {version = ">=1.5", optional = true, markers = "extra == \"cli\""} + +[package.extras] +cli = ["click (>=5.0)", "click-default-group (>=1.2)", "docformatter (==1.5.0)", "jinja2 (>=2.10)", "toposort (>=1.5)"] +docs = ["furo", "sphinx", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx-copybutton", "sphinx-inline-tabs"] +lxml = ["lxml (>=4.4.1)"] +soap = ["requests"] +test = ["codecov", "pre-commit", "pytest", "pytest-benchmark", "pytest-cov", "tox"] + [[package]] name = "xxhash" version = "3.2.0" @@ -1785,4 +2040,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "135e6f4dddec3310aca5effd4e84fc886de4c2d9cc11c4da49a5aa02244a2e7c" +content-hash = "9cfda699eda29b0341d75965a91d356d3794958c5ed469914238ae21598f6cd7" diff --git a/pyproject.toml b/pyproject.toml index 12a9b724..7bc497b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ SQLAlchemy = ">= 1.4.46, < 3.0" # note: 1.4.x currently required for enterprise mergedeep = "^1.3.4" future = "^0.18.3" importlib-metadata = "^6.1.0" +xsdata = {extras = ["cli", "lxml", "soap"], version = "^22.12"} [tool.poetry.group.dev.dependencies] pytest = "^7.2.2" @@ -103,6 +104,7 @@ exclude = '''(?x)( | ^src/vunnel/providers/amazon/parser\.py$ # ported from enterprise, never had type hints | ^src/vunnel/providers/debian/parser\.py$ # ported from enterprise, never had type hints | ^src/vunnel/providers/github/parser\.py$ # ported from enterprise, never had type hints + | ^src/vunnel/providers/mariner/model/ # generated code | ^src/vunnel/providers/nvd/parser\.py$ # ported from enterprise, never had type hints | ^src/vunnel/providers/oracle/parser\.py$ # ported from enterprise, never had type hints | ^src/vunnel/providers/rhel/parser\.py$ # ported from enterprise, never had type hints @@ -135,6 +137,7 @@ exclude = ''' | dist | data | backup + | src/vunnel/providers/mariner/model # files in here are generated | tests/quality/vulnerability-match-labels | tests/quality/.yardstick | tests/quality/data @@ -221,4 +224,5 @@ ignore = [ extend-exclude = [ "**/tests/**", + "src/vunnel/providers/mariner/model/**" # these are generated ] diff --git a/src/vunnel/cli/config.py b/src/vunnel/cli/config.py index dfc49901..1cc1105b 100644 --- a/src/vunnel/cli/config.py +++ b/src/vunnel/cli/config.py @@ -18,6 +18,7 @@ class Providers: chainguard: providers.chainguard.Config = field(default_factory=providers.chainguard.Config) debian: providers.debian.Config = field(default_factory=providers.debian.Config) github: providers.github.Config = field(default_factory=providers.github.Config) + mariner: providers.mariner.Config = field(default_factory=providers.mariner.Config) nvd: providers.nvd.Config = field(default_factory=providers.nvd.Config) oracle: providers.oracle.Config = field(default_factory=providers.oracle.Config) rhel: providers.rhel.Config = field(default_factory=providers.rhel.Config) diff --git a/src/vunnel/providers/__init__.py b/src/vunnel/providers/__init__.py index 02d2b090..afcbb1fc 100644 --- a/src/vunnel/providers/__init__.py +++ b/src/vunnel/providers/__init__.py @@ -10,6 +10,7 @@ chainguard, debian, github, + mariner, nvd, oracle, rhel, @@ -31,6 +32,7 @@ amazon.Provider.name(): amazon.Provider, debian.Provider.name(): debian.Provider, github.Provider.name(): github.Provider, + mariner.Provider.name(): mariner.Provider, nvd.Provider.name(): nvd.Provider, oracle.Provider.name(): oracle.Provider, rhel.Provider.name(): rhel.Provider, diff --git a/src/vunnel/providers/mariner/DEVELOPING.md b/src/vunnel/providers/mariner/DEVELOPING.md new file mode 100644 index 00000000..71e4d3f4 --- /dev/null +++ b/src/vunnel/providers/mariner/DEVELOPING.md @@ -0,0 +1,102 @@ +# Developing for the Mariner Provider + +This provider gets its own DEVELOPING.md because it introduces a new pattern for how OVAL XML is parsed in Vunnel. + +## How this provider works + +1. Vulnerabilities in Mariner Linux are published to https://github.com/microsoft/CBL-MarinerVulnerabilityData +2. Each major version of Mariner Linux, currently 1.0 and 2.0, gets a file named `cbl-mariner-${MAJOR_VERSION}-oval.xml` in the root of that repo. +3. Vunnel downloads this XML from GitHub's raw user content +4. Vunnel normalizes and transforms that XML for output + +The transformation is handled differently by this provider: + +1. The file at './model/generated.py' is generated by `./generate_models.py`, using the [`xsdata`](https://pypi.org/project/xsdata/) package. +2. The XML files are loaded as a `etree` from `lxml` +3. XPath expressions are used to enumerate nodes of different types from the document +4. `xsdata` is then used to parse these nodes into generated data classes +5. The transform logic is written in plain python, converting these data classes into Vunnel's `Vulnerability` type +6. Base classes common to all vunnel providers handle the output after that + +## FAQ + +### What to do if the schema of the OVAL XML changes? + +1. Re-run `./generate_models.py` +2. Fix the logic in `./parser.py` to account for any changes in data classes. + +### `xsdata` can parse the whole document, so why use `etree`? + +For error handling - `xsdata` can parse the entire document, but the entire +parse operation succeeds or fails. By looping over parts of the document and +attempting to deserialize each part, we gain the flexibility to ignore parts of +the document that can't be parsed, and still get some information from the +overall document. + +### `xsdata` can generate dataclasses from an XML schema, or from an example document. Why use the example document approach? + +We initially tried generating the dataclasses from the xsd files named as the +schemas in the OVAL XML files, but `xsdata` can't parse these files into the +resulting dataclasses. + +### The dataclasses represent a subset of the schema. What if something changes? + +If something changes we will need to fix the parser. However, we validated the existing parser via the following process: + +1. Clone the [Mariner Linux Vulnerability Repo](https://github.com/microsoft/CBL-MarinerVulnerabilityData). +2. Check out every tag in the repo, and copy the available OVAL XML files into a different directory +3. Run a script against every file in that directory to validate our assumptions about the data shape. + +Also, we specifically asked whether the schema was stable and [were told it was](https://github.com/anchore/grype/issues/1220#issuecomment-1548447284). + +Below is the script used to validate our assumptions about the XML. The only difference we found was that the +"version" field of test objects is called "evr" and used to be called "version". + +``` python +import os +from xml.etree import ElementTree as ET +from lxml import etree + +EXAMPLE_DIR="/Users/willmurphy/work/scratch/xsdata-experiments/examples/" + +def main(): + for file in os.listdir(EXAMPLE_DIR): + validate_file(os.path.join(EXAMPLE_DIR, file)) + + print('yay') + +def validate_file(file): + root = etree.parse(file) + nsmap = etree.XPath("/*")(root)[0].nsmap + default = nsmap[None] + nsmap["default"] = default + del nsmap[None] + selection = etree.XPath("//default:definition", namespaces=nsmap) + definitions = selection(root) + + tests = etree.XPath('//linux-def:rpminfo_test', namespaces=nsmap)(root) + objects = etree.XPath('//linux-def:rpminfo_object', namespaces=nsmap)(root) + states = etree.XPath('//linux-def:rpminfo_state', namespaces=nsmap)(root) + evrs = etree.XPath('//linux-def:evr', namespaces=nsmap)(root) + # at 59187, used version not evr + versions = etree.XPath('//linux-def:version', namespaces=nsmap)(root) + all_versions = evrs + versions + expected_evr_operations = { "less than", "less than or equal" } + unique_evr_operations = {k.attrib['operation'] for k in all_versions} + if len(unique_evr_operations) == 0 or not unique_evr_operations.issubset(expected_evr_operations): + raise Exception("surprise evr operation!") + unique_evr_datatypes = {k.attrib['datatype'] for k in all_versions} + expected_evr_datatypes={ "version", "evr_string" } + if len(unique_evr_datatypes) != 1 or not unique_evr_datatypes.issubset(expected_evr_datatypes): + raise Exception(f"surprise evr datatype! {expected_evr_datatypes.difference(unique_evr_datatypes)}") + all_criteria_trees = etree.XPath('//default:definition/default:criteria', namespaces=nsmap)(root) + for c_node in all_criteria_trees: + children = c_node.getchildren() + if len(children) != 1: + raise Exception("different criterion tree!") + pass + + +if __name__ == "__main__": + main() +``` diff --git a/src/vunnel/providers/mariner/__init__.py b/src/vunnel/providers/mariner/__init__.py new file mode 100644 index 00000000..22f9c8b8 --- /dev/null +++ b/src/vunnel/providers/mariner/__init__.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from vunnel import provider, result, schema +from vunnel.providers.mariner.parser import Parser + +if TYPE_CHECKING: + import datetime + + +@dataclass +class Config: + runtime: provider.RuntimeConfig = field( + default_factory=lambda: provider.RuntimeConfig( + result_store=result.StoreStrategy.SQLITE, + existing_results=provider.ResultStatePolicy.KEEP, + ), + ) + request_timeout: int = 125 + allow_versions: list[str] = field(default_factory=lambda: ["1.0", "2.0"]) + + +class Provider(provider.Provider): + def __init__(self, root: str, config: Config | None = None): + if not config: + config = Config() + super().__init__(root, runtime_cfg=config.runtime) + self.config = config + + self.logger.debug(f"config: {config}") + self.schema = schema.OSSchema() + self.parser = Parser( + workspace=self.workspace, + allow_versions=self.config.allow_versions, + download_timeout=self.config.request_timeout, + logger=self.logger, + ) + + @classmethod + def name(cls) -> str: + return "mariner" + + def update(self, last_updated: datetime.datetime | None) -> tuple[list[str], int]: + with self.results_writer() as writer: + for namespace, vuln_id, record in self.parser.get(): + writer.write( + identifier=os.path.join(namespace, vuln_id), + schema=self.schema, + payload=record, + ) + pass + return self.parser.urls, len(writer) diff --git a/src/vunnel/providers/mariner/generate_models.py b/src/vunnel/providers/mariner/generate_models.py new file mode 100644 index 00000000..8fdf0d85 --- /dev/null +++ b/src/vunnel/providers/mariner/generate_models.py @@ -0,0 +1,47 @@ +import os +import tempfile +from subprocess import PIPE, Popen + +import requests + +MARINER_URL_BASE = "https://raw.githubusercontent.com/microsoft/CBL-MarinerVulnerabilityData/main/{}" +MARINER_URL_FILENAME = "cbl-mariner-{}-oval.xml" + + +def download_version(version: str, dest_dir: str) -> None: + filename = MARINER_URL_FILENAME.format(version) + url = MARINER_URL_BASE.format(filename) + r = requests.get(url, timeout=125) + destination = os.path.join(dest_dir, filename) + with open(destination, "wb") as w: + w.write(r.content) + + +def main() -> None: + versions = ["2.0"] + dest_path = tempfile.TemporaryDirectory() + for v in versions: + download_version(v, dest_path.name) + + script_dir = os.path.realpath(os.path.dirname(__file__)) + args = [ + "xsdata", + "generate", + dest_path.name, + "-r", + "--relative-imports", + "--compound-fields", + "--package", + "model.generated", + "--structure-style", + "single-package", + ] + process = Popen(args=args, stderr=PIPE, stdout=PIPE, cwd=script_dir) + stdout, stderr = process.communicate() + print(stdout) + print(stderr) + pass + + +if __name__ == "__main__": + main() diff --git a/src/vunnel/providers/mariner/model/__init__.py b/src/vunnel/providers/mariner/model/__init__.py new file mode 100644 index 00000000..fbbb6299 --- /dev/null +++ b/src/vunnel/providers/mariner/model/__init__.py @@ -0,0 +1,41 @@ +from .generated import ( + Affected, + Criteria, + Criterion, + Definition, + Definitions, + Evr, + Generator, + Metadata, + Object, + Objects, + OvalDefinitions, + Reference, + RpminfoObject, + RpminfoState, + RpminfoTest, + State, + States, + Tests, +) + +__all__ = [ + "Affected", + "Criteria", + "Criterion", + "Definition", + "Definitions", + "Evr", + "Generator", + "Metadata", + "Object", + "Objects", + "OvalDefinitions", + "Reference", + "RpminfoObject", + "RpminfoState", + "RpminfoTest", + "State", + "States", + "Tests", +] diff --git a/src/vunnel/providers/mariner/model/generated.py b/src/vunnel/providers/mariner/model/generated.py new file mode 100644 index 00000000..e9baad13 --- /dev/null +++ b/src/vunnel/providers/mariner/model/generated.py @@ -0,0 +1,480 @@ +from dataclasses import dataclass, field +from typing import List, Optional, Union +from xsdata.models.datatype import XmlDateTime + + +@dataclass +class Evr: + class Meta: + name = "evr" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux" + + datatype: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + operation: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + value: str = field( + default="" + ) + + +@dataclass +class Object: + class Meta: + name = "object" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux" + + object_ref: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + + +@dataclass +class RpminfoObject: + class Meta: + name = "rpminfo_object" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux" + + id: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + version: Optional[int] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + name: Optional[str] = field( + default=None, + metadata={ + "type": "Element", + } + ) + + +@dataclass +class State: + class Meta: + name = "state" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux" + + state_ref: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + + +@dataclass +class Affected: + class Meta: + name = "affected" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + family: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + platform: Optional[str] = field( + default=None, + metadata={ + "type": "Element", + } + ) + + +@dataclass +class Criterion: + class Meta: + name = "criterion" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + comment: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + test_ref: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + + +@dataclass +class Generator: + class Meta: + name = "generator" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + product_name: Optional[str] = field( + default=None, + metadata={ + "type": "Element", + "namespace": "http://oval.mitre.org/XMLSchema/oval-common-5", + } + ) + product_version: Optional[int] = field( + default=None, + metadata={ + "type": "Element", + "namespace": "http://oval.mitre.org/XMLSchema/oval-common-5", + } + ) + schema_version: Optional[float] = field( + default=None, + metadata={ + "type": "Element", + "namespace": "http://oval.mitre.org/XMLSchema/oval-common-5", + } + ) + timestamp: Optional[XmlDateTime] = field( + default=None, + metadata={ + "type": "Element", + "namespace": "http://oval.mitre.org/XMLSchema/oval-common-5", + } + ) + content_version: Optional[int] = field( + default=None, + metadata={ + "type": "Element", + "namespace": "http://oval.mitre.org/XMLSchema/oval-common-5", + } + ) + + +@dataclass +class Reference: + class Meta: + name = "reference" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + ref_id: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + ref_url: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + source: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + + +@dataclass +class RpminfoState: + class Meta: + name = "rpminfo_state" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux" + + id: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + version: Optional[int] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + evr: Optional[Evr] = field( + default=None, + metadata={ + "type": "Element", + } + ) + + +@dataclass +class RpminfoTest: + class Meta: + name = "rpminfo_test" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux" + + check: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + comment: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + id: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + version: Optional[int] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + object_value: Optional[Object] = field( + default=None, + metadata={ + "name": "object", + "type": "Element", + } + ) + state: Optional[State] = field( + default=None, + metadata={ + "type": "Element", + } + ) + + +@dataclass +class Criteria: + class Meta: + name = "criteria" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + operator: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + criterion: Optional[Criterion] = field( + default=None, + metadata={ + "type": "Element", + } + ) + + +@dataclass +class Metadata: + class Meta: + name = "metadata" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + title: Optional[str] = field( + default=None, + metadata={ + "type": "Element", + } + ) + affected: Optional[Affected] = field( + default=None, + metadata={ + "type": "Element", + } + ) + reference: Optional[Reference] = field( + default=None, + metadata={ + "type": "Element", + } + ) + patchable: Optional[Union[bool, str]] = field( + default=None, + metadata={ + "type": "Element", + } + ) + advisory_date: Optional[XmlDateTime] = field( + default=None, + metadata={ + "type": "Element", + } + ) + advisory_id: Optional[Union[str, int]] = field( + default=None, + metadata={ + "type": "Element", + } + ) + severity: Optional[str] = field( + default=None, + metadata={ + "type": "Element", + } + ) + description: Optional[str] = field( + default=None, + metadata={ + "type": "Element", + } + ) + + +@dataclass +class Objects: + class Meta: + name = "objects" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + rpminfo_object: List[RpminfoObject] = field( + default_factory=list, + metadata={ + "type": "Element", + "namespace": "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux", + } + ) + + +@dataclass +class Definition: + class Meta: + name = "definition" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + class_value: Optional[str] = field( + default=None, + metadata={ + "name": "class", + "type": "Attribute", + } + ) + id: Optional[str] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + version: Optional[int] = field( + default=None, + metadata={ + "type": "Attribute", + } + ) + metadata: Optional[Metadata] = field( + default=None, + metadata={ + "type": "Element", + } + ) + criteria: Optional[Criteria] = field( + default=None, + metadata={ + "type": "Element", + } + ) + + +@dataclass +class States: + class Meta: + name = "states" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + rpminfo_state: List[RpminfoState] = field( + default_factory=list, + metadata={ + "type": "Element", + "namespace": "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux", + } + ) + + +@dataclass +class Tests: + class Meta: + name = "tests" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + rpminfo_test: List[RpminfoTest] = field( + default_factory=list, + metadata={ + "type": "Element", + "namespace": "http://oval.mitre.org/XMLSchema/oval-definitions-5#linux", + } + ) + + +@dataclass +class Definitions: + class Meta: + name = "definitions" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + definition: List[Definition] = field( + default_factory=list, + metadata={ + "type": "Element", + } + ) + + +@dataclass +class OvalDefinitions: + class Meta: + name = "oval_definitions" + namespace = "http://oval.mitre.org/XMLSchema/oval-definitions-5" + + schema_location: Optional[str] = field( + default=None, + metadata={ + "name": "schemaLocation", + "type": "Attribute", + "namespace": "http://www.w3.org/2001/XMLSchema-instance", + } + ) + generator: Optional[Generator] = field( + default=None, + metadata={ + "type": "Element", + } + ) + definitions: Optional[Definitions] = field( + default=None, + metadata={ + "type": "Element", + } + ) + tests: Optional[Tests] = field( + default=None, + metadata={ + "type": "Element", + } + ) + objects: Optional[Objects] = field( + default=None, + metadata={ + "type": "Element", + } + ) + states: Optional[States] = field( + default=None, + metadata={ + "type": "Element", + } + ) diff --git a/src/vunnel/providers/mariner/parser.py b/src/vunnel/providers/mariner/parser.py new file mode 100644 index 00000000..0a8b557d --- /dev/null +++ b/src/vunnel/providers/mariner/parser.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Any + +import requests +from lxml import etree +from xsdata.formats.dataclass.parsers import XmlParser +from xsdata.formats.dataclass.parsers.config import ParserConfig + +from vunnel.providers.mariner.model import Definition, RpminfoObject, RpminfoState, RpminfoTest +from vunnel.utils.vulnerability import FixedIn, Vulnerability + +if TYPE_CHECKING: + import logging + from collections.abc import Generator + + from vunnel.workspace import Workspace + +LESS_THAN_OR_EQUAL_TO = "less than or equal" + + +class MarinerXmlFile: + def __init__(self, oval_file_path: str, logger: logging.Logger): + parser_config = ParserConfig( + fail_on_converter_warnings=False, + fail_on_unknown_attributes=False, + fail_on_unknown_properties=False, + ) + xml_parser = XmlParser(config=parser_config) + root = etree.parse(oval_file_path) + nsmap = etree.XPath("/*")(root)[0].nsmap + default = nsmap[None] + nsmap["default"] = default + del nsmap[None] + self.logger = logger + + # Go element by element to have flexible error handling + self.definitions = [] + for def_element in etree.XPath("//default:definition", namespaces=nsmap)(root): + try: + element_bytes = etree.tostring(def_element) + definition = xml_parser.from_string(element_bytes.decode(), Definition) + self.definitions.append(definition) + except Exception as ex: + self.logger.warning(f"skipping definition element in {oval_file_path} due to {ex}") + pass + + self.tests_by_id = {} + for test_element in etree.XPath("//linux-def:rpminfo_test", namespaces=nsmap)(root): + try: + element_bytes = etree.tostring(test_element) + test_obj = xml_parser.from_string(element_bytes.decode(), RpminfoTest) + self.tests_by_id[test_obj.id] = test_obj + except Exception as ex: + self.logger.warning(f"skipping rpminfo_test element in {oval_file_path} due to {ex}") + self.objects_by_id = {} + for obj_element in etree.XPath("//linux-def:rpminfo_object", namespaces=nsmap)(root): + try: + element_bytes = etree.tostring(obj_element) + obj = xml_parser.from_string(element_bytes.decode(), RpminfoObject) + self.objects_by_id[obj.id] = obj + except Exception as ex: + self.logger.warning(f"skipping rpminfo_object element in {oval_file_path} due to {ex}") + + self.states_by_id = {} + for state_element in etree.XPath("//linux-def:rpminfo_state", namespaces=nsmap)(root): + try: + element_bytes = etree.tostring(state_element) + state_obj = xml_parser.from_string(element_bytes.decode(), RpminfoState) + self.states_by_id[state_obj.id] = state_obj + except Exception as ex: + self.logger.warning(f"skipping rpminfo_object element in {oval_file_path} due to {ex}") + self.mariner_version = oval_file_path.split("-")[-2] + + def name_and_version(self, test_id: str) -> tuple[str | None, str | None]: + test = self.tests_by_id.get(test_id, None) + if test is None or test.object_value is None or test.state is None: + return (None, None) + obj = self.objects_by_id.get(test.object_value.object_ref, None) + if obj is None: + return (None, None) + state = self.states_by_id.get(test.state.state_ref, None) + if state is None or state.evr is None: + return (None, None) + return (obj.name, state.evr.value) + + def namespace_name(self) -> str: + return f"mariner:{self.mariner_version}" + + def get_test(self, definition: Definition) -> RpminfoTest | None: + if definition is None or definition.criteria is None or definition.criteria.criterion is None: + return None + return self.tests_by_id.get(definition.criteria.criterion.test_ref, None) + + def get_state(self, definition: Definition) -> RpminfoState | None: + test = self.get_test(definition) + if test is None or test.state is None or test.state.state_ref is None: + return None + return self.states_by_id.get(test.state.state_ref, None) + + def get_object(self, definition: Definition) -> RpminfoObject | None: + test = self.get_test(definition) + if test is None or test.object_value is None or test.object_value.object_ref is None: + return None + return self.objects_by_id.get(test.object_value.object_ref, None) + + def make_fixed_in(self, definition: Definition) -> FixedIn | None: + state = self.get_state(definition) + obj = self.get_object(definition) + if state is None or state.evr is None: + return None + if obj is None or obj.name is None: + return None + version = state.evr.value + # There are 2 choices for state.ever.operation: "less than" or "less than or equal to". + # So for this vulnerability, either the statement, "versions < 3.2.1 are vulernable" + # or the statement "versions <= 3.2.1 are vulnerable". In the second statement, + # the data has no information about any fixed version, so we report "None" + # as the fixed version, meaning we consider all version vulnerable. + # For example, if version 3.2.1 of a library is vulnerable, and is the latest version + # mariner data might have "versions <= 3.2.1" is vulnerable. + if state.evr.operation == LESS_THAN_OR_EQUAL_TO: + version = "None" # legacy API needs the string "None" instead of None + return FixedIn(Name=obj.name, NamespaceName=self.namespace_name(), VersionFormat="rpm", Version=version) + + def vulnerability_id(self, definition: Definition) -> str | None: + if definition.metadata is None or definition.metadata.reference is None or definition.metadata.reference.ref_id is None: + return None + return definition.metadata.reference.ref_id + + def description(self, definition: Definition) -> str | None: + if not definition.metadata: + return None + return definition.metadata.description + + def vulnerabilities(self) -> Generator[Vulnerability, None, None]: + for d in self.definitions: + if d.metadata is None or d.metadata.severity is None: + self.logger.warning("skipping definition because severity could not be found") + continue + if d.metadata.description: + pass + link = "" + if d.metadata.reference and d.metadata.reference.ref_url: + link = d.metadata.reference.ref_url + fixed_in = self.make_fixed_in(d) + if not fixed_in: + continue + vulnerability_id = self.vulnerability_id(d) + if not vulnerability_id: + continue + yield Vulnerability( + Name=vulnerability_id, # intentional; legacy API uses Name as field for vulnerability ID. + NamespaceName=self.namespace_name(), + Description=self.description(d) or "", + Severity=d.metadata.severity, + Link=link, + CVSS=[], + FixedIn=[fixed_in], + Metadata={}, # Ideally, there's no metadata here. + ) + + +MARINER_URL_BASE = "https://raw.githubusercontent.com/microsoft/CBL-MarinerVulnerabilityData/main/{}" +MARINER_URL_FILENAME = "cbl-mariner-{}-oval.xml" + + +class Parser: + def __init__(self, workspace: Workspace, download_timeout: int, allow_versions: list[Any], logger: logging.Logger): + self.workspace = workspace + self.download_timeout = download_timeout + self.allow_versions = allow_versions + self._urls: set[str] = set() + self.logger = logger + + def _download(self) -> list[str]: + return [self._download_version(v) for v in self.allow_versions] + + def _download_version(self, version: str) -> str: + filename = MARINER_URL_FILENAME.format(version) + url = MARINER_URL_BASE.format(filename) + r = requests.get(url, timeout=self.download_timeout) + destination = os.path.join(self.workspace.input_path, filename) + with open(destination, "wb") as writer: + writer.write(r.content) + self._urls.add(url) + return destination + + @property + def urls(self) -> list[str]: + return list(self._urls) + + def get(self) -> Generator[tuple[str, str, dict[str, dict[str, Any]]], None, None]: + for oval_file_path in self._download(): + parsed_file = MarinerXmlFile(oval_file_path, self.logger) + for v in parsed_file.vulnerabilities(): + yield v.NamespaceName, v.Name, v.to_payload() diff --git a/src/vunnel/utils/vulnerability.py b/src/vunnel/utils/vulnerability.py index b957c620..b7b44b95 100644 --- a/src/vunnel/utils/vulnerability.py +++ b/src/vunnel/utils/vulnerability.py @@ -87,12 +87,18 @@ class Vulnerability: """ Name: str + """Name is a str that contains the vulnerability ID""" NamespaceName: str Description: str Severity: str Link: str CVSS: list[CVSS] FixedIn: list[FixedIn] + """ + For a package with no fix, a single FixedIn should be reported with + "None" as the Version. The name field on a FixedIn is the name of the + vulnerable package. + """ Metadata: dict[str, Any] = field(default_factory=dict) def to_payload(self): diff --git a/tests/quality/config.yaml b/tests/quality/config.yaml index 3b15a67e..55f134dd 100644 --- a/tests/quality/config.yaml +++ b/tests/quality/config.yaml @@ -98,6 +98,10 @@ tests: - docker.io/anchore/test_images:grype-quality-golang-d89207b@sha256:7536ee345532f674ec9e448e3768db4e546c48220ba2b6ec9bc9cfbfb3b7b74a - docker.io/anchore/test_images:grype-quality-ruby-d89207b@sha256:1a5a5f870924e88a6f0f2b8089cf276ef0a79b5244a052cdfe4a47bb9e5a2c10 + - provider: mariner + images: + - mcr.microsoft.com/cbl-mariner/base/core:2.0.20220731-amd64@sha256:3c0f7e103ff3c39e81e7c9c042d2b321d833fb6d26d8636567f7d88a6bdde74a + - provider: nvd images: - docker.io/busybox:1.28.1@sha256:2107a35b58593c58ec5f4e8f2c4a70d195321078aebfadfbfb223a2ff4a4ed21 diff --git a/tests/quality/vulnerability-match-labels b/tests/quality/vulnerability-match-labels index bbd9bf4c..dce0048c 160000 --- a/tests/quality/vulnerability-match-labels +++ b/tests/quality/vulnerability-match-labels @@ -1 +1 @@ -Subproject commit bbd9bf4ca5054accbc3bce828f90844bab07759f +Subproject commit dce0048c4f94d59e0e60c8167216ab337e516caf diff --git a/tests/unit/cli/test_cli.py b/tests/unit/cli/test_cli.py index 28c6d443..ed5a6116 100644 --- a/tests/unit/cli/test_cli.py +++ b/tests/unit/cli/test_cli.py @@ -195,6 +195,21 @@ def test_config(monkeypatch) -> None: retry_delay: 5 result_store: sqlite token: secret + mariner: + allow_versions: + - '1.0' + - '2.0' + request_timeout: 125 + runtime: + existing_input: keep + existing_results: keep + on_error: + action: fail + input: keep + results: keep + retry_count: 3 + retry_delay: 5 + result_store: sqlite nvd: api_key: secret request_timeout: 125 diff --git a/tests/unit/providers/mariner/__init__.py b/tests/unit/providers/mariner/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/providers/mariner/test-fixtures/mariner-truncated-2.0-oval.xml b/tests/unit/providers/mariner/test-fixtures/mariner-truncated-2.0-oval.xml new file mode 100644 index 00000000..e1270c15 --- /dev/null +++ b/tests/unit/providers/mariner/test-fixtures/mariner-truncated-2.0-oval.xml @@ -0,0 +1,97 @@ + + + CBL-Mariner OVAL Definition Generator + 15 + 5.11 + 2023-05-11T12:02:01.195098043Z + 1683806521 + + + + + CVE-2023-21980 affecting package mysql 8.0.32-1 + + CBL-Mariner + + + true + 2023-05-03T16:24:32Z + 26220-1 + High + CVE-2023-21980 affecting package mysql 8.0.32-1. An upgraded version of the package is available that resolves this issue. + + + + + + + + CVE-2023-21977 affecting package mysql 8.0.32-1 + + CBL-Mariner + + + true + 2023-05-03T16:24:32Z + 26178-1 + Medium + CVE-2023-21977 affecting package mysql 8.0.32-1. An upgraded version of the package is available that resolves this issue. + + + + + + + + CVE-2022-3736 affecting package bind 9.16.33-1 + + CBL-Mariner + + + false + 13203 + High + CVE-2022-3736 affecting package bind 9.16.33-1. No patch is available currently. + + + + + + + + + + + + + + + + + + + + + + + mysql + + + mysql + + + bind + + + + + 0:8.0.33-1.cm2 + + + 0:8.0.33-1.cm2 + + + 0:9.16.33-1.cm2 + + + diff --git a/tests/unit/providers/mariner/test_mariner.py b/tests/unit/providers/mariner/test_mariner.py new file mode 100644 index 00000000..18732290 --- /dev/null +++ b/tests/unit/providers/mariner/test_mariner.py @@ -0,0 +1,90 @@ +from __future__ import annotations +import logging + +import shutil + +import pytest +from pytest_unordered import unordered +from vunnel import result, workspace +from vunnel.providers.mariner import Config, Provider, parser +from vunnel.providers.mariner.parser import MarinerXmlFile +from vunnel.utils.vulnerability import Vulnerability, FixedIn + + +@pytest.mark.parametrize( + ("input_file", "expected"), + [ + ( + "test-fixtures/mariner-truncated-2.0-oval.xml", + [ + Vulnerability( + Name="CVE-2023-21980", + NamespaceName="mariner:2.0", + Description="CVE-2023-21980 affecting package mysql 8.0.32-1. An upgraded version of the package is available that resolves this issue.", + Severity="High", + Link="https://nvd.nist.gov/vuln/detail/CVE-2023-21980", + CVSS=[], + FixedIn=[FixedIn(Name="mysql", NamespaceName="mariner:2.0", VersionFormat="rpm", Version="0:8.0.33-1.cm2")], + Metadata={}, + ), + Vulnerability( + Name="CVE-2023-21977", + NamespaceName="mariner:2.0", + Description="CVE-2023-21977 affecting package mysql 8.0.32-1. An upgraded version of the package is available that resolves this issue.", + Severity="Medium", + Link="https://nvd.nist.gov/vuln/detail/CVE-2023-21977", + CVSS=[], + FixedIn=[FixedIn(Name="mysql", NamespaceName="mariner:2.0", VersionFormat="rpm", Version="0:8.0.33-1.cm2")], + Metadata={}, + ), + Vulnerability( + Name="CVE-2022-3736", + NamespaceName="mariner:2.0", + Description="CVE-2022-3736 affecting package bind 9.16.33-1. No patch is available currently.", + Severity="High", + Link="https://nvd.nist.gov/vuln/detail/CVE-2022-3736", + CVSS=[], + FixedIn=[ + FixedIn(Name="bind", NamespaceName="mariner:2.0", VersionFormat="rpm", Version="None"), + ], + ), + ], + ) + ], +) +def test_parse(tmpdir, helpers, input_file, expected): + mock_data_path = helpers.local_dir(input_file) + subject = MarinerXmlFile(mock_data_path, logger=logging.getLogger("test_pariner")) + + vulnerabilities = [v for v in subject.vulnerabilities()] + assert len(vulnerabilities) == len(expected) + assert vulnerabilities == expected + + +@pytest.fixture() +def disable_get_requests(monkeypatch): + def disabled(*args, **kwargs): + raise RuntimeError("requests disabled but HTTP GET attempted") + + monkeypatch.setattr(parser.requests, "get", disabled) + + +def test_provider_schema(helpers, disable_get_requests, monkeypatch): + workspace = helpers.provider_workspace_helper(name=Provider.name()) + + c = Config(allow_versions=["2.0"]) + c.runtime.result_store = result.StoreStrategy.FLAT_FILE + p = Provider(root=workspace.root, config=c) + + mock_data_path = helpers.local_dir("test-fixtures/mariner-truncated-2.0-oval.xml") + shutil.copy(mock_data_path, workspace.input_dir / "mariner-truncated-2.0-oval.xml") + + def mock_download(*args, **kwargs): + return [mock_data_path] + + monkeypatch.setattr(p.parser, "_download", mock_download) + + p.update(None) + + assert 3 == workspace.num_result_entries() + assert workspace.result_schemas_valid(require_entries=True)