From 97a4f5dcc7fd16dbc4df9a5549d2ad3da56e177e Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Thu, 2 Jul 2020 07:29:24 +0530 Subject: [PATCH 01/16] implementation to handle go.mod files Signed-off-by: rpotter12 --- src/packagedcode/__init__.py | 2 + src/packagedcode/go_mod.py | 199 +++++++++++++++++++++++++++++++++++ src/packagedcode/golang.py | 132 +++++++++++++++++++++++ 3 files changed, 333 insertions(+) create mode 100644 src/packagedcode/go_mod.py create mode 100644 src/packagedcode/golang.py diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index 57737a406c7..62bc5b50c26 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -33,6 +33,7 @@ from packagedcode import conda from packagedcode import cargo from packagedcode import freebsd +from packagedcode import golang from packagedcode import haxe from packagedcode import maven from packagedcode import npm @@ -73,6 +74,7 @@ models.ChromeExtension, models.IOSApp, pypi.PythonPackage, + golang.GolangPackage, models.CabPackage, models.MsiInstallerPackage, models.InstallShieldPackage, diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py new file mode 100644 index 00000000000..495145c7a47 --- /dev/null +++ b/src/packagedcode/go_mod.py @@ -0,0 +1,199 @@ + +# Copyright (c) 2019 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import io +import logging +import re + +import attr + + +""" +https://golang.org/ref/mod#go.mod-files + +For example: + +module example.com/my/thing + +go 1.12 + +require example.com/other/thing v1.0.2 +require example.com/new/thing v2.3.4 +exclude example.com/old/thing v1.2.3 +require ( + example.com/new/thing v2.3.4 + example.com/old/thing v1.2.3 +) +require ( + example.com/new/thing v2.3.4 + example.com/old/thing v1.2.3 +) + +""" + + +TRACE = False + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + +@attr.s() +class GoMod(object): + # Regex expressions to parse different types of dependency + parse_module = re.compile( + r'^module\s' + r'(?P[a-z].*)' + ).match + + parse_module_name = re.compile( + r'^module\s.*/' + r'(?P[^\s]*)' + ).match + + parse_require = re.compile( + r'^require\s' + r'.*/(?P[^\s]*)' + r'\s' + r'(?P[a-z].*)' + ).match + + parse_exclude = re.compile( + r'^exclude\s' + r'.*/(?P[^\s]*)' + r'\s' + r'(?P[a-z].*)' + ).match + + parse_dep_link = re.compile( + r'^.*/(?P[^\s]*)' + r'\s' + r'(?P[a-z].*)' + ).match + + def preprocess(self, line): + """ + Return line string after removing commented portion and excess spaces. + """ + if "//" in line: + line = line[:line.index('//')] + line = line.strip() + return line + + def parse_gomod(self, location): + """ + Return a dictionary contains all the important go.mod file data. + """ + with io.open(location, encoding='utf-8', closefd=True) as data: + lines = data.readlines() + + gomod_data = {} + require = [] + exclude = [] + + for i, line in enumerate(lines): + line = self.preprocess(line) + parsed_module = self.parse_module(line) + if parsed_module: + gomod_data['module'] = parsed_module.group('module') + + parsed_module_name = self.parse_module_name(line) + if parsed_module_name: + gomod_data['name'] = parsed_module_name.group('name') + + parsed_require = self.parse_require(line) + if parsed_require: + line_req = [parsed_require.group('name'), parsed_require.group('version')] + require.append(line_req) + + parsed_exclude = self.parse_exclude(line) + if parsed_exclude: + line_req = [parsed_exclude.group('name'), parsed_exclude.group('version')] + exclude.append(line_req) + + if 'require' in line and '(' in line: + for req in lines[i+1:]: + req = self.preprocess(req) + if ')' in req: + break + parsed_dep_link = self.parse_dep_link(req) + if parsed_dep_link: + line_req = [parsed_dep_link.group('name'), parsed_dep_link.group('version')] + require.append(line_req) + + if 'exclude' in line and '(' in line: + for exc in lines[i+1:]: + exc = self.preprocess(exc) + if ')' in exc: + break + parsed_dep_link = self.parse_dep_link(exc) + if parsed_dep_link: + line_exclude = [parsed_dep_link.group('name'), parsed_dep_link.group('version')] + exclude.append(line_exclude) + + gomod_data['require'] = require + gomod_data['exclude'] = exclude + + return gomod_data + + +""" +module is in the form +module github.com/alecthomas/participle + +For example: +>>> ob = GoMod() +>>> p = ob.parse_module('module github.com/alecthomas/participle') +>>> assert p.group('module') == ('github.com/alecthomas/participle') + +require or exclude can be in the form +require github.com/davecgh/go-spew v1.1.1 +or +exclude github.com/davecgh/go-spew v1.1.1 +or +github.com/davecgh/go-spew v1.1.1 + +For example: +>>> ob = GoMod() + +>>> p = ob.parse_require('require github.com/davecgh/go-spew v1.1.1') +>>> assert p.group('name') == ('go-spew') +>>> assert p.group('version') == ('v1.1.1') + +>>> p = ob.parse_exclude('exclude github.com/davecgh/go-spew v1.1.1') +>>> assert p.group('name') == ('go-spew') +>>> assert p.group('version') == ('v1.1.1') + +>>> p = ob.parse_dep_link('github.com/davecgh/go-spew v1.1.1') +>>> assert p.group('name') == ('go-spew') +>>> assert p.group('version') == ('v1.1.1') +""" diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py new file mode 100644 index 00000000000..bf4f79ff8e2 --- /dev/null +++ b/src/packagedcode/golang.py @@ -0,0 +1,132 @@ + +# Copyright (c) 2019 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from collections import OrderedDict +import io +import logging +import re + +import attr +from packageurl import PackageURL + +from commoncode import filetype +from commoncode import fileutils +from packagedcode.go_mod import GoMod +from packagedcode import models + + +""" +Handle Go packages including go.mod and go.sum files. +""" + +TRACE = False + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + +@attr.s() +class GolangPackage(models.Package): + metafiles = ('go.mod',) + default_type = 'gopkg' + default_primary_language = 'Go' + default_web_baseurl = None + default_download_baseurl = None + default_api_baseurl = None + + @classmethod + def recognize(cls, location): + if fileutils.file_name(location).lower() == 'go.mod': + gomod_obj = GoMod() + gomod_data = gomod_obj.parse_gomod(location) + yield build_gomod_package(gomod_data) + + @classmethod + def get_package_root(cls, manifest_resource, codebase): + return manifest_resource.parent(codebase) + + def repository_homepage_url(self): + return self.homepage_url + + def repository_download_url(self): + return self.download_url + + +def build_gomod_package(gomod_data): + """ + Return a Package object from a go.mod file or None. + """ + name = gomod_data.get('name') + homepage_url = "https://{}".format(gomod_data.get('module')) + download_url = "https://{}/archive/master.zip".format(gomod_data.get('module')) + + package_dependencies = [] + require = gomod_data.get('require') + if require: + for name, version in require: + package_dependencies.append( + models.DependentPackage( + purl=PackageURL( + type='gopkg', + name=name + ).to_string(), + requirement=version, + scope='require', + is_runtime=True, + is_optional=False, + is_resolved=False, + ) + ) + exclude = gomod_data.get('exclude') + if exclude: + for name, version in exclude: + package_dependencies.append( + models.DependentPackage( + purl=PackageURL( + type='gopkg', + name=name + ).to_string(), + requirement=version, + scope='exclude', + is_runtime=True, + is_optional=False, + is_resolved=False, + ) + ) + + return GolangPackage( + name=name, + homepage_url=homepage_url, + download_url=download_url, + code_view_url=homepage_url, + dependencies=package_dependencies + ) \ No newline at end of file From c0ec7e193650c749cfa5fe7da8f1679fa899b4d7 Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Thu, 2 Jul 2020 07:33:14 +0530 Subject: [PATCH 02/16] sample go.mod test files Signed-off-by: rpotter12 --- tests/packagedcode/data/go/kingpin/go.mod | 11 ++ .../data/go/kingpin/output.expected.json | 78 ++++++++++++++ tests/packagedcode/data/go/participle/go.mod | 9 ++ .../data/go/participle/output.expected.json | 62 +++++++++++ tests/packagedcode/data/go/sample/go.mod | 12 +++ .../data/go/sample/output.expected.json | 101 ++++++++++++++++++ tests/packagedcode/data/go/uap-go/go.mod | 3 + .../data/go/uap-go/output.expected.json | 77 +++++++++++++ tests/packagedcode/data/go/user_agent/go.mod | 3 + .../data/go/user_agent/output.expected.json | 68 ++++++++++++ tests/packagedcode/data/plugin/help.txt | 5 + 11 files changed, 429 insertions(+) create mode 100644 tests/packagedcode/data/go/kingpin/go.mod create mode 100644 tests/packagedcode/data/go/kingpin/output.expected.json create mode 100644 tests/packagedcode/data/go/participle/go.mod create mode 100644 tests/packagedcode/data/go/participle/output.expected.json create mode 100644 tests/packagedcode/data/go/sample/go.mod create mode 100644 tests/packagedcode/data/go/sample/output.expected.json create mode 100644 tests/packagedcode/data/go/uap-go/go.mod create mode 100644 tests/packagedcode/data/go/uap-go/output.expected.json create mode 100644 tests/packagedcode/data/go/user_agent/go.mod create mode 100644 tests/packagedcode/data/go/user_agent/output.expected.json diff --git a/tests/packagedcode/data/go/kingpin/go.mod b/tests/packagedcode/data/go/kingpin/go.mod new file mode 100644 index 00000000000..2451d0f3bba --- /dev/null +++ b/tests/packagedcode/data/go/kingpin/go.mod @@ -0,0 +1,11 @@ +module github.com/alecthomas/kingpin + +require ( + github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc + github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.2.2 +) + +go 1.13 diff --git a/tests/packagedcode/data/go/kingpin/output.expected.json b/tests/packagedcode/data/go/kingpin/output.expected.json new file mode 100644 index 00000000000..eadcd9e694d --- /dev/null +++ b/tests/packagedcode/data/go/kingpin/output.expected.json @@ -0,0 +1,78 @@ +[ + { + "type": "gopkg", + "namespace": null, + "name": "testify", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/alecthomas/kingpin", + "download_url": "https://github.com/alecthomas/kingpin/archive/master.zip", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/alecthomas/kingpin", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gopkg/template@v0.0.0-20160405071501-a0175ee3bccc", + "requirement": "v0.0.0-20160405071501-a0175ee3bccc", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:gopkg/units@v0.0.0-20151022065526-2efee857e7cf", + "requirement": "v0.0.0-20151022065526-2efee857e7cf", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:gopkg/go-spew@v1.1.1", + "requirement": "v1.1.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:gopkg/go-difflib@v1.0.0", + "requirement": "v1.0.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:gopkg/testify@v1.2.2", + "requirement": "v1.2.2", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gopkg/testify", + "repository_homepage_url": "https://github.com/alecthomas/kingpin", + "repository_download_url": "https://github.com/alecthomas/kingpin/archive/master.zip", + "api_data_url": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/go/participle/go.mod b/tests/packagedcode/data/go/participle/go.mod new file mode 100644 index 00000000000..8c03b07a1f4 --- /dev/null +++ b/tests/packagedcode/data/go/participle/go.mod @@ -0,0 +1,9 @@ +module github.com/alecthomas/participle + +require ( + github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/stretchr/testify v1.4.0 +) + +go 1.13 diff --git a/tests/packagedcode/data/go/participle/output.expected.json b/tests/packagedcode/data/go/participle/output.expected.json new file mode 100644 index 00000000000..bc4ebc86752 --- /dev/null +++ b/tests/packagedcode/data/go/participle/output.expected.json @@ -0,0 +1,62 @@ +[ + { + "type": "gopkg", + "namespace": null, + "name": "testify", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/alecthomas/participle", + "download_url": "https://github.com/alecthomas/participle/archive/master.zip", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/alecthomas/participle", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gopkg/repr@v0.0.0-20181024024818-d37bc2a10ba1", + "requirement": "v0.0.0-20181024024818-d37bc2a10ba1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:gopkg/go-spew@v1.1.1", + "requirement": "v1.1.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:gopkg/testify@v1.4.0", + "requirement": "v1.4.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gopkg/testify", + "repository_homepage_url": "https://github.com/alecthomas/participle", + "repository_download_url": "https://github.com/alecthomas/participle/archive/master.zip", + "api_data_url": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/go/sample/go.mod b/tests/packagedcode/data/go/sample/go.mod new file mode 100644 index 00000000000..260eedff2ee --- /dev/null +++ b/tests/packagedcode/data/go/sample/go.mod @@ -0,0 +1,12 @@ +module github.com/alecthomas/sample + +require github.com/davecgh/go-spew v1.1.1 // indirect +exclude github.com/stretchr/testify v1.4.0 +require ( + github.com/stretchr/testify v1.4.0 +) +exclude ( + github.com/alecthomas/repr v0.0.0 +) + +go 1.13 diff --git a/tests/packagedcode/data/go/sample/output.expected.json b/tests/packagedcode/data/go/sample/output.expected.json new file mode 100644 index 00000000000..7a78d0c5982 --- /dev/null +++ b/tests/packagedcode/data/go/sample/output.expected.json @@ -0,0 +1,101 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "3.1.2", + "options": { + "input": [ + "go.mod" + ], + "--json-pp": "output.expected.json", + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2020-06-29T015258.034475", + "end_timestamp": "2020-06-29T015259.003571", + "duration": 0.9691345691680908, + "message": null, + "errors": [], + "extra_data": { + "files_count": 1 + } + } + ], + "files": [ + { + "path": "go.mod", + "type": "file", + "packages": [ + { + "type": "gopkg", + "namespace": null, + "name": "repr", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/alecthomas/sample", + "download_url": "https://github.com/alecthomas/sample/archive/master.zip", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/alecthomas/sample", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gopkg/go-spew@v1.1.1", + "requirement": "v1.1.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:gopkg/testify@v1.4.0", + "requirement": "v1.4.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:gopkg/testify@v1.4.0", + "requirement": "v1.4.0", + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:gopkg/repr@v0.0.0", + "requirement": "v0.0.0", + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gopkg/repr", + "repository_homepage_url": "https://github.com/alecthomas/sample", + "repository_download_url": "https://github.com/alecthomas/sample/archive/master.zip", + "api_data_url": null + } + ], + "scan_errors": [] + } + ] +} diff --git a/tests/packagedcode/data/go/uap-go/go.mod b/tests/packagedcode/data/go/uap-go/go.mod new file mode 100644 index 00000000000..94dbdeea260 --- /dev/null +++ b/tests/packagedcode/data/go/uap-go/go.mod @@ -0,0 +1,3 @@ +module github.com/ua-parser/uap-go + +require gopkg.in/yaml.v2 v2.2.1 diff --git a/tests/packagedcode/data/go/uap-go/output.expected.json b/tests/packagedcode/data/go/uap-go/output.expected.json new file mode 100644 index 00000000000..dcaea74f2b0 --- /dev/null +++ b/tests/packagedcode/data/go/uap-go/output.expected.json @@ -0,0 +1,77 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "3.1.2", + "options": { + "input": [ + "go.mod" + ], + "--json-pp": "output.expected.json", + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2020-06-29T015311.409941", + "end_timestamp": "2020-06-29T015312.376565", + "duration": 0.9666707515716553, + "message": null, + "errors": [], + "extra_data": { + "files_count": 1 + } + } + ], + "files": [ + { + "path": "go.mod", + "type": "file", + "packages": [ + { + "type": "gopkg", + "namespace": null, + "name": "yaml.v2", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/ua-parser/uap-go", + "download_url": "https://github.com/ua-parser/uap-go/archive/master.zip", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/ua-parser/uap-go", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gopkg/yaml.v2@v2.2.1", + "requirement": "v2.2.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gopkg/yaml.v2", + "repository_homepage_url": "https://github.com/ua-parser/uap-go", + "repository_download_url": "https://github.com/ua-parser/uap-go/archive/master.zip", + "api_data_url": null + } + ], + "scan_errors": [] + } + ] +} diff --git a/tests/packagedcode/data/go/user_agent/go.mod b/tests/packagedcode/data/go/user_agent/go.mod new file mode 100644 index 00000000000..a17c7ad0c35 --- /dev/null +++ b/tests/packagedcode/data/go/user_agent/go.mod @@ -0,0 +1,3 @@ +module github.com/mssola/user_agent + +go 1.13 diff --git a/tests/packagedcode/data/go/user_agent/output.expected.json b/tests/packagedcode/data/go/user_agent/output.expected.json new file mode 100644 index 00000000000..12490e15806 --- /dev/null +++ b/tests/packagedcode/data/go/user_agent/output.expected.json @@ -0,0 +1,68 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "3.1.2", + "options": { + "input": [ + "go.mod" + ], + "--json-pp": "output.expected.json", + "--package": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2020-06-29T015336.174980", + "end_timestamp": "2020-06-29T015337.144426", + "duration": 0.9694929122924805, + "message": null, + "errors": [], + "extra_data": { + "files_count": 1 + } + } + ], + "files": [ + { + "path": "go.mod", + "type": "file", + "packages": [ + { + "type": "gopkg", + "namespace": null, + "name": "user_agent", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/mssola/user_agent", + "download_url": "https://github.com/mssola/user_agent/archive/master.zip", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/mssola/user_agent", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gopkg/user_agent", + "repository_homepage_url": "https://github.com/mssola/user_agent", + "repository_download_url": "https://github.com/mssola/user_agent/archive/master.zip", + "api_data_url": null + } + ], + "scan_errors": [] + } + ] +} diff --git a/tests/packagedcode/data/plugin/help.txt b/tests/packagedcode/data/plugin/help.txt index ae70e508d32..f617264e4e0 100644 --- a/tests/packagedcode/data/plugin/help.txt +++ b/tests/packagedcode/data/plugin/help.txt @@ -114,6 +114,11 @@ Package: gem extensions: .gem filetypes: .tar, tar archive +-------------------------------------------- +Package: gopkg + class: packagedcode.golang:GolangPackage + metafiles: go.mod + -------------------------------------------- Package: haxe class: packagedcode.haxe:HaxePackage From 9fc4f00923cd5a6f19620cfa199d35b9b4317ba1 Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Mon, 6 Jul 2020 08:39:02 +0530 Subject: [PATCH 03/16] tests for sample go.mod file Signed-off-by: rpotter12 --- .../data/go/kingpin/output.expected.json | 20 +-- .../data/go/participle/output.expected.json | 12 +- .../data/go/sample/output.expected.json | 163 +++++++----------- .../data/go/uap-go/output.expected.json | 121 +++++-------- .../data/go/user_agent/output.expected.json | 105 ++++------- tests/packagedcode/test_golang.py | 68 ++++++++ 6 files changed, 232 insertions(+), 257 deletions(-) create mode 100644 tests/packagedcode/test_golang.py diff --git a/tests/packagedcode/data/go/kingpin/output.expected.json b/tests/packagedcode/data/go/kingpin/output.expected.json index eadcd9e694d..0a54e587f7b 100644 --- a/tests/packagedcode/data/go/kingpin/output.expected.json +++ b/tests/packagedcode/data/go/kingpin/output.expected.json @@ -28,44 +28,44 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:gopkg/template@v0.0.0-20160405071501-a0175ee3bccc", + "purl": "pkg:gopkg/template", "requirement": "v0.0.0-20160405071501-a0175ee3bccc", "scope": "require", "is_runtime": true, "is_optional": false, - "is_resolved": true + "is_resolved": false }, { - "purl": "pkg:gopkg/units@v0.0.0-20151022065526-2efee857e7cf", + "purl": "pkg:gopkg/units", "requirement": "v0.0.0-20151022065526-2efee857e7cf", "scope": "require", "is_runtime": true, "is_optional": false, - "is_resolved": true + "is_resolved": false }, { - "purl": "pkg:gopkg/go-spew@v1.1.1", + "purl": "pkg:gopkg/go-spew", "requirement": "v1.1.1", "scope": "require", "is_runtime": true, "is_optional": false, - "is_resolved": true + "is_resolved": false }, { - "purl": "pkg:gopkg/go-difflib@v1.0.0", + "purl": "pkg:gopkg/go-difflib", "requirement": "v1.0.0", "scope": "require", "is_runtime": true, "is_optional": false, - "is_resolved": true + "is_resolved": false }, { - "purl": "pkg:gopkg/testify@v1.2.2", + "purl": "pkg:gopkg/testify", "requirement": "v1.2.2", "scope": "require", "is_runtime": true, "is_optional": false, - "is_resolved": true + "is_resolved": false } ], "contains_source_code": null, diff --git a/tests/packagedcode/data/go/participle/output.expected.json b/tests/packagedcode/data/go/participle/output.expected.json index bc4ebc86752..e5f52da4d93 100644 --- a/tests/packagedcode/data/go/participle/output.expected.json +++ b/tests/packagedcode/data/go/participle/output.expected.json @@ -28,28 +28,28 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:gopkg/repr@v0.0.0-20181024024818-d37bc2a10ba1", + "purl": "pkg:gopkg/repr", "requirement": "v0.0.0-20181024024818-d37bc2a10ba1", "scope": "require", "is_runtime": true, "is_optional": false, - "is_resolved": true + "is_resolved": false }, { - "purl": "pkg:gopkg/go-spew@v1.1.1", + "purl": "pkg:gopkg/go-spew", "requirement": "v1.1.1", "scope": "require", "is_runtime": true, "is_optional": false, - "is_resolved": true + "is_resolved": false }, { - "purl": "pkg:gopkg/testify@v1.4.0", + "purl": "pkg:gopkg/testify", "requirement": "v1.4.0", "scope": "require", "is_runtime": true, "is_optional": false, - "is_resolved": true + "is_resolved": false } ], "contains_source_code": null, diff --git a/tests/packagedcode/data/go/sample/output.expected.json b/tests/packagedcode/data/go/sample/output.expected.json index 7a78d0c5982..d7bc6f7388c 100644 --- a/tests/packagedcode/data/go/sample/output.expected.json +++ b/tests/packagedcode/data/go/sample/output.expected.json @@ -1,101 +1,70 @@ -{ - "headers": [ - { - "tool_name": "scancode-toolkit", - "tool_version": "3.1.2", - "options": { - "input": [ - "go.mod" - ], - "--json-pp": "output.expected.json", - "--package": true - }, - "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2020-06-29T015258.034475", - "end_timestamp": "2020-06-29T015259.003571", - "duration": 0.9691345691680908, - "message": null, - "errors": [], - "extra_data": { - "files_count": 1 - } - } - ], - "files": [ - { - "path": "go.mod", - "type": "file", - "packages": [ - { - "type": "gopkg", - "namespace": null, - "name": "repr", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": "Go", - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/alecthomas/sample", - "download_url": "https://github.com/alecthomas/sample/archive/master.zip", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/alecthomas/sample", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [ - { - "purl": "pkg:gopkg/go-spew@v1.1.1", +[ + { + "type": "gopkg", + "namespace": null, + "name": "repr", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/alecthomas/sample", + "download_url": "https://github.com/alecthomas/sample/archive/master.zip", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/alecthomas/sample", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gopkg/go-spew", "requirement": "v1.1.1", "scope": "require", "is_runtime": true, "is_optional": false, - "is_resolved": true - }, - { - "purl": "pkg:gopkg/testify@v1.4.0", - "requirement": "v1.4.0", - "scope": "require", - "is_runtime": true, - "is_optional": false, - "is_resolved": true - }, - { - "purl": "pkg:gopkg/testify@v1.4.0", - "requirement": "v1.4.0", - "scope": "exclude", - "is_runtime": true, - "is_optional": false, - "is_resolved": true - }, - { - "purl": "pkg:gopkg/repr@v0.0.0", - "requirement": "v0.0.0", - "scope": "exclude", - "is_runtime": true, - "is_optional": false, - "is_resolved": true - } - ], - "contains_source_code": null, - "source_packages": [], - "purl": "pkg:gopkg/repr", - "repository_homepage_url": "https://github.com/alecthomas/sample", - "repository_download_url": "https://github.com/alecthomas/sample/archive/master.zip", - "api_data_url": null - } - ], - "scan_errors": [] - } - ] -} + "is_resolved": false + }, + { + "purl": "pkg:gopkg/testify", + "requirement": "v1.4.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gopkg/testify", + "requirement": "v1.4.0", + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + }, + { + "purl": "pkg:gopkg/repr", + "requirement": "v0.0.0", + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_resolved": false + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gopkg/repr", + "repository_homepage_url": "https://github.com/alecthomas/sample", + "repository_download_url": "https://github.com/alecthomas/sample/archive/master.zip", + "api_data_url": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/go/uap-go/output.expected.json b/tests/packagedcode/data/go/uap-go/output.expected.json index dcaea74f2b0..9d13f2080e9 100644 --- a/tests/packagedcode/data/go/uap-go/output.expected.json +++ b/tests/packagedcode/data/go/uap-go/output.expected.json @@ -1,77 +1,46 @@ -{ - "headers": [ - { - "tool_name": "scancode-toolkit", - "tool_version": "3.1.2", - "options": { - "input": [ - "go.mod" - ], - "--json-pp": "output.expected.json", - "--package": true - }, - "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2020-06-29T015311.409941", - "end_timestamp": "2020-06-29T015312.376565", - "duration": 0.9666707515716553, - "message": null, - "errors": [], - "extra_data": { - "files_count": 1 +[ + { + "type": "gopkg", + "namespace": null, + "name": "yaml.v2", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/ua-parser/uap-go", + "download_url": "https://github.com/ua-parser/uap-go/archive/master.zip", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/ua-parser/uap-go", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:gopkg/yaml.v2", + "requirement": "v2.2.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_resolved": false } - } - ], - "files": [ - { - "path": "go.mod", - "type": "file", - "packages": [ - { - "type": "gopkg", - "namespace": null, - "name": "yaml.v2", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": "Go", - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/ua-parser/uap-go", - "download_url": "https://github.com/ua-parser/uap-go/archive/master.zip", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/ua-parser/uap-go", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [ - { - "purl": "pkg:gopkg/yaml.v2@v2.2.1", - "requirement": "v2.2.1", - "scope": "require", - "is_runtime": true, - "is_optional": false, - "is_resolved": true - } - ], - "contains_source_code": null, - "source_packages": [], - "purl": "pkg:gopkg/yaml.v2", - "repository_homepage_url": "https://github.com/ua-parser/uap-go", - "repository_download_url": "https://github.com/ua-parser/uap-go/archive/master.zip", - "api_data_url": null - } - ], - "scan_errors": [] - } - ] -} + ], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gopkg/yaml.v2", + "repository_homepage_url": "https://github.com/ua-parser/uap-go", + "repository_download_url": "https://github.com/ua-parser/uap-go/archive/master.zip", + "api_data_url": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/go/user_agent/output.expected.json b/tests/packagedcode/data/go/user_agent/output.expected.json index 12490e15806..2b5642c1914 100644 --- a/tests/packagedcode/data/go/user_agent/output.expected.json +++ b/tests/packagedcode/data/go/user_agent/output.expected.json @@ -1,68 +1,37 @@ -{ - "headers": [ - { - "tool_name": "scancode-toolkit", - "tool_version": "3.1.2", - "options": { - "input": [ - "go.mod" - ], - "--json-pp": "output.expected.json", - "--package": true - }, - "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2020-06-29T015336.174980", - "end_timestamp": "2020-06-29T015337.144426", - "duration": 0.9694929122924805, - "message": null, - "errors": [], - "extra_data": { - "files_count": 1 - } - } - ], - "files": [ - { - "path": "go.mod", - "type": "file", - "packages": [ - { - "type": "gopkg", - "namespace": null, - "name": "user_agent", - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": "Go", - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": "https://github.com/mssola/user_agent", - "download_url": "https://github.com/mssola/user_agent/archive/master.zip", - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": "https://github.com/mssola/user_agent", - "vcs_url": null, - "copyright": null, - "license_expression": null, - "declared_license": null, - "notice_text": null, - "root_path": null, - "dependencies": [], - "contains_source_code": null, - "source_packages": [], - "purl": "pkg:gopkg/user_agent", - "repository_homepage_url": "https://github.com/mssola/user_agent", - "repository_download_url": "https://github.com/mssola/user_agent/archive/master.zip", - "api_data_url": null - } - ], - "scan_errors": [] - } - ] -} +[ + { + "type": "gopkg", + "namespace": null, + "name": "user_agent", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://github.com/mssola/user_agent", + "download_url": "https://github.com/mssola/user_agent/archive/master.zip", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/mssola/user_agent", + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:gopkg/user_agent", + "repository_homepage_url": "https://github.com/mssola/user_agent", + "repository_download_url": "https://github.com/mssola/user_agent/archive/master.zip", + "api_data_url": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_golang.py b/tests/packagedcode/test_golang.py new file mode 100644 index 00000000000..ee80a5a4734 --- /dev/null +++ b/tests/packagedcode/test_golang.py @@ -0,0 +1,68 @@ + +# Copyright (c) 2019 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import os +import pytest + +from packagedcode import golang + +from packages_test_utils import PackageTester + + +class TestGolang(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_parse_kingpin(self): + test_file = self.get_test_loc('go/kingpin/go.mod') + expected_loc = self.get_test_loc('go/kingpin/output.expected.json') + package = golang.GolangPackage.recognize(test_file) + self.check_packages(package, expected_loc, regen=False) + + def test_parse_participle(self): + test_file = self.get_test_loc('go/participle/go.mod') + expected_loc = self.get_test_loc('go/participle/output.expected.json') + package = golang.GolangPackage.recognize(test_file) + self.check_packages(package, expected_loc, regen=False) + + def test_parse_sample(self): + test_file = self.get_test_loc('go/sample/go.mod') + expected_loc = self.get_test_loc('go/sample/output.expected.json') + package = golang.GolangPackage.recognize(test_file) + self.check_packages(package, expected_loc, regen=False) + + def test_parse_uap_go(self): + test_file = self.get_test_loc('go/uap-go/go.mod') + expected_loc = self.get_test_loc('go/uap-go/output.expected.json') + package = golang.GolangPackage.recognize(test_file) + self.check_packages(package, expected_loc, regen=False) + + def test_parse_user_agent(self): + test_file = self.get_test_loc('go/user_agent/go.mod') + expected_loc = self.get_test_loc('go/user_agent/output.expected.json') + package = golang.GolangPackage.recognize(test_file) + self.check_packages(package, expected_loc, regen=False) \ No newline at end of file From f2e1f4121d23c8165b0520d7e26c6e32d025091e Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Wed, 8 Jul 2020 03:10:35 +0530 Subject: [PATCH 04/16] improve docstrings Signed-off-by: rpotter12 --- src/packagedcode/go_mod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index 495145c7a47..e746c0a4117 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -111,7 +111,7 @@ def preprocess(self, line): def parse_gomod(self, location): """ - Return a dictionary contains all the important go.mod file data. + Return a dictionary containing all the important go.mod file data. """ with io.open(location, encoding='utf-8', closefd=True) as data: lines = data.readlines() From c22421fbb6c4aa9f00275b0078785b44cd7cd9fc Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Fri, 10 Jul 2020 07:45:03 +0530 Subject: [PATCH 05/16] add package default type Signed-off-by: rpotter12 --- src/packagedcode/golang.py | 89 ++++++++++--------- .../data/go/kingpin/output.expected.json | 26 +++--- .../data/go/participle/output.expected.json | 22 ++--- .../data/go/sample/output.expected.json | 24 ++--- .../data/go/uap-go/output.expected.json | 18 ++-- .../data/go/user_agent/output.expected.json | 14 +-- tests/packagedcode/data/plugin/help.txt | 2 +- 7 files changed, 101 insertions(+), 94 deletions(-) diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index bf4f79ff8e2..83400efc7e5 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -43,6 +43,20 @@ """ Handle Go packages including go.mod and go.sum files. """ +""" +Sample go.mod file: +module github.com/alecthomas/participle +require ( + github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/stretchr/testify v1.4.0 +) +go 1.13 +""" + +# TODO: +# go.mod file does not contain version number. +# valid download url need version number TRACE = False @@ -57,9 +71,9 @@ @attr.s() class GolangPackage(models.Package): metafiles = ('go.mod',) - default_type = 'gopkg' + default_type = 'golang' default_primary_language = 'Go' - default_web_baseurl = None + default_web_baseurl = 'https://pkg.go.dev' default_download_baseurl = None default_api_baseurl = None @@ -77,56 +91,49 @@ def get_package_root(cls, manifest_resource, codebase): def repository_homepage_url(self): return self.homepage_url - def repository_download_url(self): - return self.download_url - def build_gomod_package(gomod_data): """ Return a Package object from a go.mod file or None. """ - name = gomod_data.get('name') - homepage_url = "https://{}".format(gomod_data.get('module')) - download_url = "https://{}/archive/master.zip".format(gomod_data.get('module')) package_dependencies = [] - require = gomod_data.get('require') - if require: - for name, version in require: - package_dependencies.append( - models.DependentPackage( - purl=PackageURL( - type='gopkg', - name=name - ).to_string(), - requirement=version, - scope='require', - is_runtime=True, - is_optional=False, - is_resolved=False, - ) + require = gomod_data.get('require') or [] + for name, version in require: + package_dependencies.append( + models.DependentPackage( + purl=PackageURL( + type='golang', + name=name + ).to_string(), + requirement=version, + scope='require', + is_runtime=True, + is_optional=False, + is_resolved=False, ) - exclude = gomod_data.get('exclude') - if exclude: - for name, version in exclude: - package_dependencies.append( - models.DependentPackage( - purl=PackageURL( - type='gopkg', - name=name - ).to_string(), - requirement=version, - scope='exclude', - is_runtime=True, - is_optional=False, - is_resolved=False, - ) + ) + exclude = gomod_data.get('exclude') or [] + for name, version in exclude: + package_dependencies.append( + models.DependentPackage( + purl=PackageURL( + type='golang', + name=name + ).to_string(), + requirement=version, + scope='exclude', + is_runtime=True, + is_optional=False, + is_resolved=False, ) + ) + + name = gomod_data.get('name') + homepage_url = 'https://pkg.go.dev/{}'.format(gomod_data.get('module')) return GolangPackage( name=name, homepage_url=homepage_url, - download_url=download_url, - code_view_url=homepage_url, dependencies=package_dependencies - ) \ No newline at end of file + ) diff --git a/tests/packagedcode/data/go/kingpin/output.expected.json b/tests/packagedcode/data/go/kingpin/output.expected.json index 0a54e587f7b..e6577465f9d 100644 --- a/tests/packagedcode/data/go/kingpin/output.expected.json +++ b/tests/packagedcode/data/go/kingpin/output.expected.json @@ -1,8 +1,8 @@ [ { - "type": "gopkg", + "type": "golang", "namespace": null, - "name": "testify", + "name": "kingpin", "version": null, "qualifiers": {}, "subpath": null, @@ -11,15 +11,15 @@ "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/alecthomas/kingpin", - "download_url": "https://github.com/alecthomas/kingpin/archive/master.zip", + "homepage_url": "https://pkg.go.dev/github.com/alecthomas/kingpin", + "download_url": null, "size": null, "sha1": null, "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": "https://github.com/alecthomas/kingpin", + "code_view_url": null, "vcs_url": null, "copyright": null, "license_expression": null, @@ -28,7 +28,7 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:gopkg/template", + "purl": "pkg:golang/template", "requirement": "v0.0.0-20160405071501-a0175ee3bccc", "scope": "require", "is_runtime": true, @@ -36,7 +36,7 @@ "is_resolved": false }, { - "purl": "pkg:gopkg/units", + "purl": "pkg:golang/units", "requirement": "v0.0.0-20151022065526-2efee857e7cf", "scope": "require", "is_runtime": true, @@ -44,7 +44,7 @@ "is_resolved": false }, { - "purl": "pkg:gopkg/go-spew", + "purl": "pkg:golang/go-spew", "requirement": "v1.1.1", "scope": "require", "is_runtime": true, @@ -52,7 +52,7 @@ "is_resolved": false }, { - "purl": "pkg:gopkg/go-difflib", + "purl": "pkg:golang/go-difflib", "requirement": "v1.0.0", "scope": "require", "is_runtime": true, @@ -60,7 +60,7 @@ "is_resolved": false }, { - "purl": "pkg:gopkg/testify", + "purl": "pkg:golang/testify", "requirement": "v1.2.2", "scope": "require", "is_runtime": true, @@ -70,9 +70,9 @@ ], "contains_source_code": null, "source_packages": [], - "purl": "pkg:gopkg/testify", - "repository_homepage_url": "https://github.com/alecthomas/kingpin", - "repository_download_url": "https://github.com/alecthomas/kingpin/archive/master.zip", + "purl": "pkg:golang/kingpin", + "repository_homepage_url": "https://pkg.go.dev/github.com/alecthomas/kingpin", + "repository_download_url": null, "api_data_url": null } ] \ No newline at end of file diff --git a/tests/packagedcode/data/go/participle/output.expected.json b/tests/packagedcode/data/go/participle/output.expected.json index e5f52da4d93..2aec8d9ea69 100644 --- a/tests/packagedcode/data/go/participle/output.expected.json +++ b/tests/packagedcode/data/go/participle/output.expected.json @@ -1,8 +1,8 @@ [ { - "type": "gopkg", + "type": "golang", "namespace": null, - "name": "testify", + "name": "participle", "version": null, "qualifiers": {}, "subpath": null, @@ -11,15 +11,15 @@ "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/alecthomas/participle", - "download_url": "https://github.com/alecthomas/participle/archive/master.zip", + "homepage_url": "https://pkg.go.dev/github.com/alecthomas/participle", + "download_url": null, "size": null, "sha1": null, "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": "https://github.com/alecthomas/participle", + "code_view_url": null, "vcs_url": null, "copyright": null, "license_expression": null, @@ -28,7 +28,7 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:gopkg/repr", + "purl": "pkg:golang/repr", "requirement": "v0.0.0-20181024024818-d37bc2a10ba1", "scope": "require", "is_runtime": true, @@ -36,7 +36,7 @@ "is_resolved": false }, { - "purl": "pkg:gopkg/go-spew", + "purl": "pkg:golang/go-spew", "requirement": "v1.1.1", "scope": "require", "is_runtime": true, @@ -44,7 +44,7 @@ "is_resolved": false }, { - "purl": "pkg:gopkg/testify", + "purl": "pkg:golang/testify", "requirement": "v1.4.0", "scope": "require", "is_runtime": true, @@ -54,9 +54,9 @@ ], "contains_source_code": null, "source_packages": [], - "purl": "pkg:gopkg/testify", - "repository_homepage_url": "https://github.com/alecthomas/participle", - "repository_download_url": "https://github.com/alecthomas/participle/archive/master.zip", + "purl": "pkg:golang/participle", + "repository_homepage_url": "https://pkg.go.dev/github.com/alecthomas/participle", + "repository_download_url": null, "api_data_url": null } ] \ No newline at end of file diff --git a/tests/packagedcode/data/go/sample/output.expected.json b/tests/packagedcode/data/go/sample/output.expected.json index d7bc6f7388c..564f67a4b1f 100644 --- a/tests/packagedcode/data/go/sample/output.expected.json +++ b/tests/packagedcode/data/go/sample/output.expected.json @@ -1,8 +1,8 @@ [ { - "type": "gopkg", + "type": "golang", "namespace": null, - "name": "repr", + "name": "sample", "version": null, "qualifiers": {}, "subpath": null, @@ -11,15 +11,15 @@ "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/alecthomas/sample", - "download_url": "https://github.com/alecthomas/sample/archive/master.zip", + "homepage_url": "https://pkg.go.dev/github.com/alecthomas/sample", + "download_url": null, "size": null, "sha1": null, "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": "https://github.com/alecthomas/sample", + "code_view_url": null, "vcs_url": null, "copyright": null, "license_expression": null, @@ -28,7 +28,7 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:gopkg/go-spew", + "purl": "pkg:golang/go-spew", "requirement": "v1.1.1", "scope": "require", "is_runtime": true, @@ -36,7 +36,7 @@ "is_resolved": false }, { - "purl": "pkg:gopkg/testify", + "purl": "pkg:golang/testify", "requirement": "v1.4.0", "scope": "require", "is_runtime": true, @@ -44,7 +44,7 @@ "is_resolved": false }, { - "purl": "pkg:gopkg/testify", + "purl": "pkg:golang/testify", "requirement": "v1.4.0", "scope": "exclude", "is_runtime": true, @@ -52,7 +52,7 @@ "is_resolved": false }, { - "purl": "pkg:gopkg/repr", + "purl": "pkg:golang/repr", "requirement": "v0.0.0", "scope": "exclude", "is_runtime": true, @@ -62,9 +62,9 @@ ], "contains_source_code": null, "source_packages": [], - "purl": "pkg:gopkg/repr", - "repository_homepage_url": "https://github.com/alecthomas/sample", - "repository_download_url": "https://github.com/alecthomas/sample/archive/master.zip", + "purl": "pkg:golang/sample", + "repository_homepage_url": "https://pkg.go.dev/github.com/alecthomas/sample", + "repository_download_url": null, "api_data_url": null } ] \ No newline at end of file diff --git a/tests/packagedcode/data/go/uap-go/output.expected.json b/tests/packagedcode/data/go/uap-go/output.expected.json index 9d13f2080e9..3f69942c269 100644 --- a/tests/packagedcode/data/go/uap-go/output.expected.json +++ b/tests/packagedcode/data/go/uap-go/output.expected.json @@ -1,8 +1,8 @@ [ { - "type": "gopkg", + "type": "golang", "namespace": null, - "name": "yaml.v2", + "name": "uap-go", "version": null, "qualifiers": {}, "subpath": null, @@ -11,15 +11,15 @@ "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/ua-parser/uap-go", - "download_url": "https://github.com/ua-parser/uap-go/archive/master.zip", + "homepage_url": "https://pkg.go.dev/github.com/ua-parser/uap-go", + "download_url": null, "size": null, "sha1": null, "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": "https://github.com/ua-parser/uap-go", + "code_view_url": null, "vcs_url": null, "copyright": null, "license_expression": null, @@ -28,7 +28,7 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:gopkg/yaml.v2", + "purl": "pkg:golang/yaml.v2", "requirement": "v2.2.1", "scope": "require", "is_runtime": true, @@ -38,9 +38,9 @@ ], "contains_source_code": null, "source_packages": [], - "purl": "pkg:gopkg/yaml.v2", - "repository_homepage_url": "https://github.com/ua-parser/uap-go", - "repository_download_url": "https://github.com/ua-parser/uap-go/archive/master.zip", + "purl": "pkg:golang/uap-go", + "repository_homepage_url": "https://pkg.go.dev/github.com/ua-parser/uap-go", + "repository_download_url": null, "api_data_url": null } ] \ No newline at end of file diff --git a/tests/packagedcode/data/go/user_agent/output.expected.json b/tests/packagedcode/data/go/user_agent/output.expected.json index 2b5642c1914..80cb45272f3 100644 --- a/tests/packagedcode/data/go/user_agent/output.expected.json +++ b/tests/packagedcode/data/go/user_agent/output.expected.json @@ -1,6 +1,6 @@ [ { - "type": "gopkg", + "type": "golang", "namespace": null, "name": "user_agent", "version": null, @@ -11,15 +11,15 @@ "release_date": null, "parties": [], "keywords": [], - "homepage_url": "https://github.com/mssola/user_agent", - "download_url": "https://github.com/mssola/user_agent/archive/master.zip", + "homepage_url": "https://pkg.go.dev/github.com/mssola/user_agent", + "download_url": null, "size": null, "sha1": null, "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": "https://github.com/mssola/user_agent", + "code_view_url": null, "vcs_url": null, "copyright": null, "license_expression": null, @@ -29,9 +29,9 @@ "dependencies": [], "contains_source_code": null, "source_packages": [], - "purl": "pkg:gopkg/user_agent", - "repository_homepage_url": "https://github.com/mssola/user_agent", - "repository_download_url": "https://github.com/mssola/user_agent/archive/master.zip", + "purl": "pkg:golang/user_agent", + "repository_homepage_url": "https://pkg.go.dev/github.com/mssola/user_agent", + "repository_download_url": null, "api_data_url": null } ] \ No newline at end of file diff --git a/tests/packagedcode/data/plugin/help.txt b/tests/packagedcode/data/plugin/help.txt index f617264e4e0..36b5e48c174 100644 --- a/tests/packagedcode/data/plugin/help.txt +++ b/tests/packagedcode/data/plugin/help.txt @@ -115,7 +115,7 @@ Package: gem filetypes: .tar, tar archive -------------------------------------------- -Package: gopkg +Package: golang class: packagedcode.golang:GolangPackage metafiles: go.mod From 7bef31bcc6bef5f0424ffe4f9aeba554581c811f Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Tue, 21 Jul 2020 01:45:14 +0530 Subject: [PATCH 06/16] add namespace Signed-off-by: rpotter12 --- src/packagedcode/go_mod.py | 105 ++++++++++++++++++++----------------- src/packagedcode/golang.py | 8 ++- 2 files changed, 64 insertions(+), 49 deletions(-) diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index e746c0a4117..3a69278e38c 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -56,6 +56,41 @@ """ +""" +module is in the form +module github.com/alecthomas/participle + +For example: +>>> ob = GoMod() +>>> p = ob.parse_module('module github.com/alecthomas/participle') +>>> assert p.group('module') == ('github.com/alecthomas/participle') + +require or exclude can be in the form +require github.com/davecgh/go-spew v1.1.1 +or +exclude github.com/davecgh/go-spew v1.1.1 +or +github.com/davecgh/go-spew v1.1.1 + +For example: +>>> ob = GoMod() + +>>> p = ob.parse_require('require github.com/davecgh/go-spew v1.1.1') +>>> assert p.group('namespace') == ('github.com/davecgh') +>>> assert p.group('name') == ('go-spew') +>>> assert p.group('version') == ('v1.1.1') + +>>> p = ob.parse_exclude('exclude github.com/davecgh/go-spew v1.1.1') +>>> assert p.group('namespace') == ('github.com/davecgh') +>>> assert p.group('name') == ('go-spew') +>>> assert p.group('version') == ('v1.1.1') + +>>> p = ob.parse_dep_link('github.com/davecgh/go-spew v1.1.1') +>>> assert p.group('namespace') == ('github.com/davecgh') +>>> assert p.group('name') == ('go-spew') +>>> assert p.group('version') == ('v1.1.1') +""" + TRACE = False @@ -76,28 +111,36 @@ class GoMod(object): ).match parse_module_name = re.compile( - r'^module\s.*/' + r'^module(\s)*' + r'(?P(.*))' + r'/' r'(?P[^\s]*)' ).match parse_require = re.compile( - r'^require\s' - r'.*/(?P[^\s]*)' + r'^require(\s)*' + r'(?P(.*))' + r'/' + r'(?P[^\s]*)' r'\s' - r'(?P[a-z].*)' + r'(?P(.*))' ).match parse_exclude = re.compile( - r'^exclude\s' - r'.*/(?P[^\s]*)' + r'^exclude(\s)*' + r'(?P(.*))' + r'/' + r'(?P[^\s]*)' r'\s' - r'(?P[a-z].*)' + r'(?P(.*))' ).match parse_dep_link = re.compile( - r'^.*/(?P[^\s]*)' + r'(?P(.*))' + r'/' + r'(?P[^\s]*)' r'\s' - r'(?P[a-z].*)' + r'(?P(.*))' ).match def preprocess(self, line): @@ -129,16 +172,17 @@ def parse_gomod(self, location): parsed_module_name = self.parse_module_name(line) if parsed_module_name: gomod_data['name'] = parsed_module_name.group('name') + gomod_data['namespace'] = parsed_module_name.group('namespace') parsed_require = self.parse_require(line) if parsed_require: - line_req = [parsed_require.group('name'), parsed_require.group('version')] + line_req = [parsed_require.group('namespace'), parsed_require.group('name'), parsed_require.group('version')] require.append(line_req) parsed_exclude = self.parse_exclude(line) if parsed_exclude: - line_req = [parsed_exclude.group('name'), parsed_exclude.group('version')] - exclude.append(line_req) + line_exclude = [parsed_exclude.group('namespace'), parsed_exclude.group('name'), parsed_exclude.group('version')] + exclude.append(line_exclude) if 'require' in line and '(' in line: for req in lines[i+1:]: @@ -147,7 +191,7 @@ def parse_gomod(self, location): break parsed_dep_link = self.parse_dep_link(req) if parsed_dep_link: - line_req = [parsed_dep_link.group('name'), parsed_dep_link.group('version')] + line_req = [parsed_dep_link.group('namespace'), parsed_dep_link.group('name'), parsed_dep_link.group('version')] require.append(line_req) if 'exclude' in line and '(' in line: @@ -157,43 +201,10 @@ def parse_gomod(self, location): break parsed_dep_link = self.parse_dep_link(exc) if parsed_dep_link: - line_exclude = [parsed_dep_link.group('name'), parsed_dep_link.group('version')] + line_exclude = [parsed_dep_link.group('namespace'), parsed_dep_link.group('name'), parsed_dep_link.group('version')] exclude.append(line_exclude) gomod_data['require'] = require gomod_data['exclude'] = exclude return gomod_data - - -""" -module is in the form -module github.com/alecthomas/participle - -For example: ->>> ob = GoMod() ->>> p = ob.parse_module('module github.com/alecthomas/participle') ->>> assert p.group('module') == ('github.com/alecthomas/participle') - -require or exclude can be in the form -require github.com/davecgh/go-spew v1.1.1 -or -exclude github.com/davecgh/go-spew v1.1.1 -or -github.com/davecgh/go-spew v1.1.1 - -For example: ->>> ob = GoMod() - ->>> p = ob.parse_require('require github.com/davecgh/go-spew v1.1.1') ->>> assert p.group('name') == ('go-spew') ->>> assert p.group('version') == ('v1.1.1') - ->>> p = ob.parse_exclude('exclude github.com/davecgh/go-spew v1.1.1') ->>> assert p.group('name') == ('go-spew') ->>> assert p.group('version') == ('v1.1.1') - ->>> p = ob.parse_dep_link('github.com/davecgh/go-spew v1.1.1') ->>> assert p.group('name') == ('go-spew') ->>> assert p.group('version') == ('v1.1.1') -""" diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 83400efc7e5..9d4d734aa72 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -99,11 +99,12 @@ def build_gomod_package(gomod_data): package_dependencies = [] require = gomod_data.get('require') or [] - for name, version in require: + for namespace, name, version in require: package_dependencies.append( models.DependentPackage( purl=PackageURL( type='golang', + namespace=namespace, name=name ).to_string(), requirement=version, @@ -114,11 +115,12 @@ def build_gomod_package(gomod_data): ) ) exclude = gomod_data.get('exclude') or [] - for name, version in exclude: + for namespace, name, version in exclude: package_dependencies.append( models.DependentPackage( purl=PackageURL( type='golang', + namespace=namespace, name=name ).to_string(), requirement=version, @@ -130,10 +132,12 @@ def build_gomod_package(gomod_data): ) name = gomod_data.get('name') + namespace = gomod_data.get('namespace') homepage_url = 'https://pkg.go.dev/{}'.format(gomod_data.get('module')) return GolangPackage( name=name, + namespace=namespace, homepage_url=homepage_url, dependencies=package_dependencies ) From 1a1f90b7c8e72fa14f8e4d09fe678768e288eeaf Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Tue, 21 Jul 2020 01:57:40 +0530 Subject: [PATCH 07/16] add namespace tests Signed-off-by: rpotter12 --- .../data/{go => golang}/kingpin/go.mod | 0 .../kingpin/output.expected.json | 14 ++++++------- .../data/{go => golang}/participle/go.mod | 0 .../participle/output.expected.json | 10 +++++----- .../data/{go => golang}/sample/go.mod | 1 - .../sample/output.expected.json | 18 +++++------------ .../data/{go => golang}/uap-go/go.mod | 0 .../uap-go/output.expected.json | 6 +++--- .../data/{go => golang}/user_agent/go.mod | 0 .../user_agent/output.expected.json | 4 ++-- tests/packagedcode/test_golang.py | 20 +++++++++---------- 11 files changed, 32 insertions(+), 41 deletions(-) rename tests/packagedcode/data/{go => golang}/kingpin/go.mod (100%) rename tests/packagedcode/data/{go => golang}/kingpin/output.expected.json (81%) rename tests/packagedcode/data/{go => golang}/participle/go.mod (100%) rename tests/packagedcode/data/{go => golang}/participle/output.expected.json (83%) rename tests/packagedcode/data/{go => golang}/sample/go.mod (81%) rename tests/packagedcode/data/{go => golang}/sample/output.expected.json (79%) rename tests/packagedcode/data/{go => golang}/uap-go/go.mod (100%) rename tests/packagedcode/data/{go => golang}/uap-go/output.expected.json (87%) rename tests/packagedcode/data/{go => golang}/user_agent/go.mod (100%) rename tests/packagedcode/data/{go => golang}/user_agent/output.expected.json (90%) diff --git a/tests/packagedcode/data/go/kingpin/go.mod b/tests/packagedcode/data/golang/kingpin/go.mod similarity index 100% rename from tests/packagedcode/data/go/kingpin/go.mod rename to tests/packagedcode/data/golang/kingpin/go.mod diff --git a/tests/packagedcode/data/go/kingpin/output.expected.json b/tests/packagedcode/data/golang/kingpin/output.expected.json similarity index 81% rename from tests/packagedcode/data/go/kingpin/output.expected.json rename to tests/packagedcode/data/golang/kingpin/output.expected.json index e6577465f9d..8356d730490 100644 --- a/tests/packagedcode/data/go/kingpin/output.expected.json +++ b/tests/packagedcode/data/golang/kingpin/output.expected.json @@ -1,7 +1,7 @@ [ { "type": "golang", - "namespace": null, + "namespace": "github.com/alecthomas", "name": "kingpin", "version": null, "qualifiers": {}, @@ -28,7 +28,7 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:golang/template", + "purl": "pkg:golang/github.com/alecthomas/template", "requirement": "v0.0.0-20160405071501-a0175ee3bccc", "scope": "require", "is_runtime": true, @@ -36,7 +36,7 @@ "is_resolved": false }, { - "purl": "pkg:golang/units", + "purl": "pkg:golang/github.com/alecthomas/units", "requirement": "v0.0.0-20151022065526-2efee857e7cf", "scope": "require", "is_runtime": true, @@ -44,7 +44,7 @@ "is_resolved": false }, { - "purl": "pkg:golang/go-spew", + "purl": "pkg:golang/github.com/davecgh/go-spew", "requirement": "v1.1.1", "scope": "require", "is_runtime": true, @@ -52,7 +52,7 @@ "is_resolved": false }, { - "purl": "pkg:golang/go-difflib", + "purl": "pkg:golang/github.com/pmezard/go-difflib", "requirement": "v1.0.0", "scope": "require", "is_runtime": true, @@ -60,7 +60,7 @@ "is_resolved": false }, { - "purl": "pkg:golang/testify", + "purl": "pkg:golang/github.com/stretchr/testify", "requirement": "v1.2.2", "scope": "require", "is_runtime": true, @@ -70,7 +70,7 @@ ], "contains_source_code": null, "source_packages": [], - "purl": "pkg:golang/kingpin", + "purl": "pkg:golang/github.com/alecthomas/kingpin", "repository_homepage_url": "https://pkg.go.dev/github.com/alecthomas/kingpin", "repository_download_url": null, "api_data_url": null diff --git a/tests/packagedcode/data/go/participle/go.mod b/tests/packagedcode/data/golang/participle/go.mod similarity index 100% rename from tests/packagedcode/data/go/participle/go.mod rename to tests/packagedcode/data/golang/participle/go.mod diff --git a/tests/packagedcode/data/go/participle/output.expected.json b/tests/packagedcode/data/golang/participle/output.expected.json similarity index 83% rename from tests/packagedcode/data/go/participle/output.expected.json rename to tests/packagedcode/data/golang/participle/output.expected.json index 2aec8d9ea69..6998807393d 100644 --- a/tests/packagedcode/data/go/participle/output.expected.json +++ b/tests/packagedcode/data/golang/participle/output.expected.json @@ -1,7 +1,7 @@ [ { "type": "golang", - "namespace": null, + "namespace": "github.com/alecthomas", "name": "participle", "version": null, "qualifiers": {}, @@ -28,7 +28,7 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:golang/repr", + "purl": "pkg:golang/github.com/alecthomas/repr", "requirement": "v0.0.0-20181024024818-d37bc2a10ba1", "scope": "require", "is_runtime": true, @@ -36,7 +36,7 @@ "is_resolved": false }, { - "purl": "pkg:golang/go-spew", + "purl": "pkg:golang/github.com/davecgh/go-spew", "requirement": "v1.1.1", "scope": "require", "is_runtime": true, @@ -44,7 +44,7 @@ "is_resolved": false }, { - "purl": "pkg:golang/testify", + "purl": "pkg:golang/github.com/stretchr/testify", "requirement": "v1.4.0", "scope": "require", "is_runtime": true, @@ -54,7 +54,7 @@ ], "contains_source_code": null, "source_packages": [], - "purl": "pkg:golang/participle", + "purl": "pkg:golang/github.com/alecthomas/participle", "repository_homepage_url": "https://pkg.go.dev/github.com/alecthomas/participle", "repository_download_url": null, "api_data_url": null diff --git a/tests/packagedcode/data/go/sample/go.mod b/tests/packagedcode/data/golang/sample/go.mod similarity index 81% rename from tests/packagedcode/data/go/sample/go.mod rename to tests/packagedcode/data/golang/sample/go.mod index 260eedff2ee..74d07853ba7 100644 --- a/tests/packagedcode/data/go/sample/go.mod +++ b/tests/packagedcode/data/golang/sample/go.mod @@ -1,7 +1,6 @@ module github.com/alecthomas/sample require github.com/davecgh/go-spew v1.1.1 // indirect -exclude github.com/stretchr/testify v1.4.0 require ( github.com/stretchr/testify v1.4.0 ) diff --git a/tests/packagedcode/data/go/sample/output.expected.json b/tests/packagedcode/data/golang/sample/output.expected.json similarity index 79% rename from tests/packagedcode/data/go/sample/output.expected.json rename to tests/packagedcode/data/golang/sample/output.expected.json index 564f67a4b1f..549a36fba15 100644 --- a/tests/packagedcode/data/go/sample/output.expected.json +++ b/tests/packagedcode/data/golang/sample/output.expected.json @@ -1,7 +1,7 @@ [ { "type": "golang", - "namespace": null, + "namespace": "github.com/alecthomas", "name": "sample", "version": null, "qualifiers": {}, @@ -28,7 +28,7 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:golang/go-spew", + "purl": "pkg:golang/github.com/davecgh/go-spew", "requirement": "v1.1.1", "scope": "require", "is_runtime": true, @@ -36,7 +36,7 @@ "is_resolved": false }, { - "purl": "pkg:golang/testify", + "purl": "pkg:golang/github.com/stretchr/testify", "requirement": "v1.4.0", "scope": "require", "is_runtime": true, @@ -44,15 +44,7 @@ "is_resolved": false }, { - "purl": "pkg:golang/testify", - "requirement": "v1.4.0", - "scope": "exclude", - "is_runtime": true, - "is_optional": false, - "is_resolved": false - }, - { - "purl": "pkg:golang/repr", + "purl": "pkg:golang/github.com/alecthomas/repr", "requirement": "v0.0.0", "scope": "exclude", "is_runtime": true, @@ -62,7 +54,7 @@ ], "contains_source_code": null, "source_packages": [], - "purl": "pkg:golang/sample", + "purl": "pkg:golang/github.com/alecthomas/sample", "repository_homepage_url": "https://pkg.go.dev/github.com/alecthomas/sample", "repository_download_url": null, "api_data_url": null diff --git a/tests/packagedcode/data/go/uap-go/go.mod b/tests/packagedcode/data/golang/uap-go/go.mod similarity index 100% rename from tests/packagedcode/data/go/uap-go/go.mod rename to tests/packagedcode/data/golang/uap-go/go.mod diff --git a/tests/packagedcode/data/go/uap-go/output.expected.json b/tests/packagedcode/data/golang/uap-go/output.expected.json similarity index 87% rename from tests/packagedcode/data/go/uap-go/output.expected.json rename to tests/packagedcode/data/golang/uap-go/output.expected.json index 3f69942c269..810a391412e 100644 --- a/tests/packagedcode/data/go/uap-go/output.expected.json +++ b/tests/packagedcode/data/golang/uap-go/output.expected.json @@ -1,7 +1,7 @@ [ { "type": "golang", - "namespace": null, + "namespace": "github.com/ua-parser", "name": "uap-go", "version": null, "qualifiers": {}, @@ -28,7 +28,7 @@ "root_path": null, "dependencies": [ { - "purl": "pkg:golang/yaml.v2", + "purl": "pkg:golang/gopkg.in/yaml.v2", "requirement": "v2.2.1", "scope": "require", "is_runtime": true, @@ -38,7 +38,7 @@ ], "contains_source_code": null, "source_packages": [], - "purl": "pkg:golang/uap-go", + "purl": "pkg:golang/github.com/ua-parser/uap-go", "repository_homepage_url": "https://pkg.go.dev/github.com/ua-parser/uap-go", "repository_download_url": null, "api_data_url": null diff --git a/tests/packagedcode/data/go/user_agent/go.mod b/tests/packagedcode/data/golang/user_agent/go.mod similarity index 100% rename from tests/packagedcode/data/go/user_agent/go.mod rename to tests/packagedcode/data/golang/user_agent/go.mod diff --git a/tests/packagedcode/data/go/user_agent/output.expected.json b/tests/packagedcode/data/golang/user_agent/output.expected.json similarity index 90% rename from tests/packagedcode/data/go/user_agent/output.expected.json rename to tests/packagedcode/data/golang/user_agent/output.expected.json index 80cb45272f3..90f7af76336 100644 --- a/tests/packagedcode/data/go/user_agent/output.expected.json +++ b/tests/packagedcode/data/golang/user_agent/output.expected.json @@ -1,7 +1,7 @@ [ { "type": "golang", - "namespace": null, + "namespace": "github.com/mssola", "name": "user_agent", "version": null, "qualifiers": {}, @@ -29,7 +29,7 @@ "dependencies": [], "contains_source_code": null, "source_packages": [], - "purl": "pkg:golang/user_agent", + "purl": "pkg:golang/github.com/mssola/user_agent", "repository_homepage_url": "https://pkg.go.dev/github.com/mssola/user_agent", "repository_download_url": null, "api_data_url": null diff --git a/tests/packagedcode/test_golang.py b/tests/packagedcode/test_golang.py index ee80a5a4734..33109a77869 100644 --- a/tests/packagedcode/test_golang.py +++ b/tests/packagedcode/test_golang.py @@ -38,31 +38,31 @@ class TestGolang(PackageTester): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') def test_parse_kingpin(self): - test_file = self.get_test_loc('go/kingpin/go.mod') - expected_loc = self.get_test_loc('go/kingpin/output.expected.json') + test_file = self.get_test_loc('golang/kingpin/go.mod') + expected_loc = self.get_test_loc('golang/kingpin/output.expected.json') package = golang.GolangPackage.recognize(test_file) self.check_packages(package, expected_loc, regen=False) def test_parse_participle(self): - test_file = self.get_test_loc('go/participle/go.mod') - expected_loc = self.get_test_loc('go/participle/output.expected.json') + test_file = self.get_test_loc('golang/participle/go.mod') + expected_loc = self.get_test_loc('golang/participle/output.expected.json') package = golang.GolangPackage.recognize(test_file) self.check_packages(package, expected_loc, regen=False) def test_parse_sample(self): - test_file = self.get_test_loc('go/sample/go.mod') - expected_loc = self.get_test_loc('go/sample/output.expected.json') + test_file = self.get_test_loc('golang/sample/go.mod') + expected_loc = self.get_test_loc('golang/sample/output.expected.json') package = golang.GolangPackage.recognize(test_file) self.check_packages(package, expected_loc, regen=False) def test_parse_uap_go(self): - test_file = self.get_test_loc('go/uap-go/go.mod') - expected_loc = self.get_test_loc('go/uap-go/output.expected.json') + test_file = self.get_test_loc('golang/uap-go/go.mod') + expected_loc = self.get_test_loc('golang/uap-go/output.expected.json') package = golang.GolangPackage.recognize(test_file) self.check_packages(package, expected_loc, regen=False) def test_parse_user_agent(self): - test_file = self.get_test_loc('go/user_agent/go.mod') - expected_loc = self.get_test_loc('go/user_agent/output.expected.json') + test_file = self.get_test_loc('golang/user_agent/go.mod') + expected_loc = self.get_test_loc('golang/user_agent/output.expected.json') package = golang.GolangPackage.recognize(test_file) self.check_packages(package, expected_loc, regen=False) \ No newline at end of file From f33660257548ff87d0d347458516201ed2fb85ad Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Tue, 21 Jul 2020 02:01:54 +0530 Subject: [PATCH 08/16] add comments Signed-off-by: rpotter12 --- src/packagedcode/golang.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 9d4d734aa72..2940d88daad 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -57,6 +57,7 @@ # TODO: # go.mod file does not contain version number. # valid download url need version number +# CHECK: https://forum.golangbridge.org/t/url-to-download-package/19811 TRACE = False @@ -96,7 +97,6 @@ def build_gomod_package(gomod_data): """ Return a Package object from a go.mod file or None. """ - package_dependencies = [] require = gomod_data.get('require') or [] for namespace, name, version in require: @@ -114,6 +114,7 @@ def build_gomod_package(gomod_data): is_resolved=False, ) ) + exclude = gomod_data.get('exclude') or [] for namespace, name, version in exclude: package_dependencies.append( From 289a0ee348332a095f57bdf6900c64fc1c07aa08 Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Wed, 22 Jul 2020 19:33:06 +0530 Subject: [PATCH 09/16] fix repository homepage url formation Signed-off-by: rpotter12 --- src/packagedcode/golang.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 2940d88daad..ef3d990b9b7 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -89,9 +89,8 @@ def recognize(cls, location): def get_package_root(cls, manifest_resource, codebase): return manifest_resource.parent(codebase) - def repository_homepage_url(self): - return self.homepage_url - + def repository_homepage_url(self, baseurl=default_web_baseurl): + return '{}/{}/{}'.format(baseurl, self.namespace, self.name) def build_gomod_package(gomod_data): """ From 3e88235a43f4121d6b3d5cd5214952a26a60ef95 Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Tue, 14 Jul 2020 08:34:29 +0530 Subject: [PATCH 10/16] packagedcode to handle Pipfile.lock Signed-off-by: rpotter12 --- src/packagedcode/pypi.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index b22395b9daf..0602ed6c19b 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -90,7 +90,7 @@ def logger_debug(*args): @attr.s() class PythonPackage(models.Package): - metafiles = ('metadata.json', '*setup.py', 'PKG-INFO', '*.whl', '*.egg', '*requirements*.txt', '*requirements*.in') + metafiles = ('metadata.json', '*setup.py', 'PKG-INFO', '*.whl', '*.egg', '*requirements*.txt', '*requirements*.in', '*Pipfile.lock') extensions = ('.egg', '.whl', '.pyz', '.pex',) default_type = 'pypi' default_primary_language = 'Python' @@ -119,8 +119,9 @@ def parse(location): file_name = fileutils.file_name(location) parsers = { 'setup.py': parse_setup_py, - 'requirements.txt': parse_requirements_txt, - 'requirements.in': parse_requirements_txt, + 'requirements.txt': parse_dependency_file, + 'requirements.in': parse_dependency_file, + 'Pipfile.lock': parse_dependency_file, 'metadata.json': parse_metadata, 'PKG-INFO': parse_pkg_info, '.whl': parse_wheel, @@ -198,6 +199,7 @@ def parse_dependencies(location, package): dependency_type_by_extensions = { ('.txt', '.in'): 'requirements.txt', + ('Pipfile.lock'): 'Pipfile.lock', } @@ -272,9 +274,9 @@ def parse_with_dparse(location): return package_dependencies -def parse_requirements_txt(location): +def parse_dependency_file(location): """ - Return a package built from requirements.txt. + Return a package built from Python dependency files. """ package_dependencies = parse_with_dparse(location) return PythonPackage(dependencies=package_dependencies) From 8e365027bc455ae0467c4316b2f44cb01635789f Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Tue, 14 Jul 2020 08:36:45 +0530 Subject: [PATCH 11/16] sample Pipfile.lock test files Signed-off-by: rpotter12 --- .../pypi/pipfile.lock/sample1/Pipfile.lock | 103 ++++++++++ .../pipfile.lock/sample1/output.expected.json | 108 ++++++++++ .../pypi/pipfile.lock/sample2/Pipfile.lock | 184 ++++++++++++++++++ .../pipfile.lock/sample2/output.expected.json | 172 ++++++++++++++++ .../pypi/pipfile.lock/sample3/Pipfile.lock | 28 +++ .../pipfile.lock/sample3/output.expected.json | 44 +++++ .../pypi/pipfile.lock/sample4/Pipfile.lock | 151 ++++++++++++++ .../pipfile.lock/sample4/output.expected.json | 140 +++++++++++++ .../pypi/pipfile.lock/sample5/Pipfile.lock | 103 ++++++++++ .../pipfile.lock/sample5/output.expected.json | 108 ++++++++++ 10 files changed, 1141 insertions(+) create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample1/Pipfile.lock create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample1/output.expected.json create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample2/Pipfile.lock create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample2/output.expected.json create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample3/Pipfile.lock create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample3/output.expected.json create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample4/Pipfile.lock create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample4/output.expected.json create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample5/Pipfile.lock create mode 100644 tests/packagedcode/data/pypi/pipfile.lock/sample5/output.expected.json diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample1/Pipfile.lock b/tests/packagedcode/data/pypi/pipfile.lock/sample1/Pipfile.lock new file mode 100644 index 00000000000..2c3cd426416 --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample1/Pipfile.lock @@ -0,0 +1,103 @@ +{ + "_meta": { + "hash": { + "sha256": "813f8e1b624fd42eee7d681228d7aca1fce209e1d60bf21c3eb33a73f7268d57" + }, + "host-environment-markers": { + "implementation_name": "cpython", + "implementation_version": "0", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_python_implementation": "CPython", + "platform_release": "15.6.0", + "platform_system": "Darwin", + "platform_version": "Darwin Kernel Version 15.6.0: Fri Feb 17 10:21:18 PST 2017; root:xnu-3248.60.11.4.1~1/RELEASE_X86_64", + "python_full_version": "2.7.10", + "python_version": "2.7", + "sys_platform": "darwin" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "django": { + "hashes": [ + "sha256:679fc24b3e85bf5a07ca2f6d5c4cdf3d4477bbb02f43a6548335952cc75b5d23", + "sha256:3de62e71ce2cfbcdecb6e344cad04948506c8410ea5c6eab15c8f3b31b8ac1c0" + ], + "version": "==1.7.1" + }, + "feedparser": { + "hashes": [ + "sha256:ae099763f4538aa08c5021d42ba6ce5b9b6218e71423c96031153f379955481a", + "sha256:52542106f55d044a404f00bf328bd29e81e49c38a130a05be66a8e4dc4f9ff57", + "sha256:172c23932965f91ef58e23e5abf93412333eecdc04d9e015cd09056d6b9301b3" + ], + "version": "==5.1.1" + }, + "pyasn1": { + "hashes": [ + "sha256:f81c96761fca60d64b1c9b79ec2e40cf9495a745cf570613079ef324aeb9672b", + "sha256:7d626683e3d792cccc608da02498aff37ab4f3dafd8905d6bf755d11f9b26b43", + "sha256:e85895087905c65b5b594eb91f7522664c85545b147d5f4d4e7b1b07da8dcbdc", + "sha256:5a0db897b311d265cde49615cf783f1c78613138605cdd0f907ecfa5b2aba3ee", + "sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1", + "sha256:a7efe807c4b83a859e2735c692b92ed7b567cfddc4163763412920041d876c2b", + "sha256:b5a9ca48055b9a20f6d1b3d68e38692e5431c86a0f99ea602e61294e891fee5b", + "sha256:c07d6e587b2f928366b1f67c09bda026a3e6fcc99e80a744dc67f8fca3895626", + "sha256:d84c2aea3cf43780e9e6a19f4e4dddee9f6976519020e64e47c57e5c7a8c3dd2", + "sha256:758cb50abddc03e4563fd9e7f03db56e3e87b58c0bd01247360326e5c0c7ffa5", + "sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89", + "sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15" + ], + "version": "==0.4.2" + }, + "pycrypto": { + "hashes": [ + "sha256:f49d8aea2d7d65db9906c7d3b8b3a07fcae8387cf5cb06a7510383e211902d39" + ], + "version": "==2.4" + }, + "pyjwt": { + "hashes": [ + "sha256:2d30be6375be006a6fec531ba15ec9a7cf3ac88fd6ea3caa2b6f86a84c372acc", + "sha256:a0019f8119cd9a31d9c29e7b47256b24642829c10941ee20f513487f466201d9" + ], + "version": "==0.4.2" + }, + "raven": { + "hashes": [ + "sha256:7f14e651d321aedaf00635ab106b72520e4a6565d766b8fc64856e662c3acc8c" + ], + "version": "==1.9.4" + }, + "requests": { + "hashes": [ + "sha256:b5bd2e1b78d28051108ebaa6248750221f9ccef52b4f054cb727de61b0406de0", + "sha256:1266921f1bed5fbf364cd83cf239b6d7b3ea5c32ccccbc93980d9ba12cdcfd02" + ], + "version": "==2.2.1" + }, + "rsa": { + "hashes": [ + "sha256:0c7fde631f84f89e89ec671a9c58feb01ea25fab177dca08ba08650c548d48d5", + "sha256:9f1b6d4015cdf788273ff329d43004a7abf43971e9b06160765bad5227e4d70a" + ], + "version": "==3.4" + }, + "simplejson": { + "hashes": [ + "sha256:ac0f5122a213ef35c3af6464a2885aef5b56a4954f003eac767dd8e077949885" + ], + "version": "==2.4.0" + } + }, + "develop": {} +} diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample1/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample1/output.expected.json new file mode 100644 index 00000000000..5ead799c9ca --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample1/output.expected.json @@ -0,0 +1,108 @@ +{ + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:pypi/django@1.7.1", + "requirement": "==1.7.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/feedparser@5.1.1", + "requirement": "==5.1.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pyasn1@0.4.2", + "requirement": "==0.4.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pycrypto@2.4", + "requirement": "==2.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pyjwt@0.4.2", + "requirement": "==0.4.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/raven@1.9.4", + "requirement": "==1.9.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/requests@2.2.1", + "requirement": "==2.2.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/rsa@3.4", + "requirement": "==3.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/simplejson@2.4.0", + "requirement": "==2.4.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample2/Pipfile.lock b/tests/packagedcode/data/pypi/pipfile.lock/sample2/Pipfile.lock new file mode 100644 index 00000000000..5f1bc99b2ab --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample2/Pipfile.lock @@ -0,0 +1,184 @@ +{ + "_meta": { + "hash": { + "sha256": "6e45251662433bf51f96fb3d2204b65416fece329d60e6235c0f0edc416cfe24" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "2.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "atomicwrites": { + "hashes": [ + "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585", + "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6" + ], + "version": "==1.1.5" + }, + "attrs": { + "hashes": [ + "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265", + "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b" + ], + "version": "==18.1.0" + }, + "contextlib2": { + "hashes": [ + "sha256:509f9419ee91cdd00ba34443217d5ca51f5a364a404e1dce9e8979cea969ca48", + "sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00" + ], + "markers": "python_version == '2.7'", + "version": "==0.5.5" + }, + "coverage": { + "hashes": [ + "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba", + "sha256:0b136648de27201056c1869a6c0d4e23f464750fd9a9ba9750b8336a244429ed", + "sha256:10a46017fef60e16694a30627319f38a2b9b52e90182dddb6e37dcdab0f4bf95", + "sha256:198626739a79b09fa0a2f06e083ffd12eb55449b5f8bfdbeed1df4910b2ca640", + "sha256:23d341cdd4a0371820eb2b0bd6b88f5003a7438bbedb33688cd33b8eae59affd", + "sha256:28b2191e7283f4f3568962e373b47ef7f0392993bb6660d079c62bd50fe9d162", + "sha256:2a5b73210bad5279ddb558d9a2bfedc7f4bf6ad7f3c988641d83c40293deaec1", + "sha256:2eb564bbf7816a9d68dd3369a510be3327f1c618d2357fa6b1216994c2e3d508", + "sha256:337ded681dd2ef9ca04ef5d93cfc87e52e09db2594c296b4a0a3662cb1b41249", + "sha256:3a2184c6d797a125dca8367878d3b9a178b6fdd05fdc2d35d758c3006a1cd694", + "sha256:3c79a6f7b95751cdebcd9037e4d06f8d5a9b60e4ed0cd231342aa8ad7124882a", + "sha256:3d72c20bd105022d29b14a7d628462ebdc61de2f303322c0212a054352f3b287", + "sha256:3eb42bf89a6be7deb64116dd1cc4b08171734d721e7a7e57ad64cc4ef29ed2f1", + "sha256:4635a184d0bbe537aa185a34193898eee409332a8ccb27eea36f262566585000", + "sha256:56e448f051a201c5ebbaa86a5efd0ca90d327204d8b059ab25ad0f35fbfd79f1", + "sha256:5a13ea7911ff5e1796b6d5e4fbbf6952381a611209b736d48e675c2756f3f74e", + "sha256:69bf008a06b76619d3c3f3b1983f5145c75a305a0fea513aca094cae5c40a8f5", + "sha256:6bc583dc18d5979dc0f6cec26a8603129de0304d5ae1f17e57a12834e7235062", + "sha256:701cd6093d63e6b8ad7009d8a92425428bc4d6e7ab8d75efbb665c806c1d79ba", + "sha256:7608a3dd5d73cb06c531b8925e0ef8d3de31fed2544a7de6c63960a1e73ea4bc", + "sha256:76ecd006d1d8f739430ec50cc872889af1f9c1b6b8f48e29941814b09b0fd3cc", + "sha256:7aa36d2b844a3e4a4b356708d79fd2c260281a7390d678a10b91ca595ddc9e99", + "sha256:7d3f553904b0c5c016d1dad058a7554c7ac4c91a789fca496e7d8347ad040653", + "sha256:7e1fe19bd6dce69d9fd159d8e4a80a8f52101380d5d3a4d374b6d3eae0e5de9c", + "sha256:8c3cb8c35ec4d9506979b4cf90ee9918bc2e49f84189d9bf5c36c0c1119c6558", + "sha256:9d6dd10d49e01571bf6e147d3b505141ffc093a06756c60b053a859cb2128b1f", + "sha256:be6cfcd8053d13f5f5eeb284aa8a814220c3da1b0078fa859011c7fffd86dab9", + "sha256:c1bb572fab8208c400adaf06a8133ac0712179a334c09224fb11393e920abcdd", + "sha256:de4418dadaa1c01d497e539210cb6baa015965526ff5afc078c57ca69160108d", + "sha256:e05cb4d9aad6233d67e0541caa7e511fa4047ed7750ec2510d466e806e0255d6", + "sha256:f3f501f345f24383c0000395b26b726e46758b71393267aeae0bd36f8b3ade80" + ], + "version": "==4.5.1" + }, + "funcsigs": { + "hashes": [ + "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", + "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" + ], + "markers": "python_version < '3.0'", + "version": "==1.0.2" + }, + "mock": { + "hashes": [ + "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1", + "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba" + ], + "markers": "python_version == '2.7'", + "version": "==2.0.0" + }, + "more-itertools": { + "hashes": [ + "sha256:2b6b9893337bfd9166bee6a62c2b0c9fe7735dcf85948b387ec8cba30e85d8e8", + "sha256:6703844a52d3588f951883005efcf555e49566a48afd4db4e965d69b883980d3", + "sha256:a18d870ef2ffca2b8463c0070ad17b5978056f403fb64e3f15fe62a52db21cc0" + ], + "version": "==4.2.0" + }, + "pbr": { + "hashes": [ + "sha256:1b8be50d938c9bb75d0eaf7eda111eec1bf6dc88a62a6412e33bf077457e0f45", + "sha256:b486975c0cafb6beeb50ca0e17ba047647f229087bd74e37f4a7e2cac17d2caa" + ], + "version": "==4.2.0" + }, + "pluggy": { + "hashes": [ + "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff", + "sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c", + "sha256:e160a7fcf25762bb60efc7e171d4497ff1d8d2d75a3d0df7a21b76821ecbf5c5" + ], + "version": "==0.6.0" + }, + "py": { + "hashes": [ + "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7", + "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e" + ], + "version": "==1.5.4" + }, + "pytest": { + "hashes": [ + "sha256:0453c8676c2bee6feb0434748b068d5510273a916295fd61d306c4f22fbfd752", + "sha256:4b208614ae6d98195430ad6bde03641c78553acee7c83cec2e85d613c0cd383d" + ], + "version": "==3.6.3" + }, + "pytest-cov": { + "hashes": [ + "sha256:03aa752cf11db41d281ea1d807d954c4eda35cfa1b21d6971966cc041bbf6e2d", + "sha256:890fe5565400902b0c78b5357004aab1c814115894f4f21370e2433256a3eeec" + ], + "index": "pypi", + "version": "==2.5.1" + }, + "pytest-vcr": { + "hashes": [ + "sha256:13f2e24ab4b8674e012ca23df50e34f0b322a8b8cf8aa80f2b432ae95a92e298", + "sha256:b3d1f0a470e172a5f8ef60b3f1daf3ff1ff60a43280a9cdfb24a8c53e3e9c12c" + ], + "index": "pypi", + "version": "==0.3.0" + }, + "pyyaml": { + "hashes": [ + "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", + "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", + "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", + "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", + "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", + "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", + "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", + "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", + "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", + "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", + "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" + ], + "version": "==3.13" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "version": "==1.11.0" + }, + "vcrpy": { + "hashes": [ + "sha256:7031f9c78a70b9586d2db4a2ec135c4e04194cabff58695ef0cc95e7cd66bc01", + "sha256:dadb4b2798a44bdce5301ec01380351ab798a468fc7ee5a34271bdd8527d8f99" + ], + "version": "==1.13.0" + }, + "wrapt": { + "hashes": [ + "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6" + ], + "version": "==1.10.11" + } + }, + "develop": {} +} diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample2/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample2/output.expected.json new file mode 100644 index 00000000000..eb95b3cb5b0 --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample2/output.expected.json @@ -0,0 +1,172 @@ +{ + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:pypi/atomicwrites@1.1.5", + "requirement": "==1.1.5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/attrs@18.1.0", + "requirement": "==18.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/contextlib2@0.5.5", + "requirement": "==0.5.5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/coverage@4.5.1", + "requirement": "==4.5.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/funcsigs@1.0.2", + "requirement": "==1.0.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/mock@2.0.0", + "requirement": "==2.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/more-itertools@4.2.0", + "requirement": "==4.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pbr@4.2.0", + "requirement": "==4.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pluggy@0.6.0", + "requirement": "==0.6.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/py@1.5.4", + "requirement": "==1.5.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pytest@3.6.3", + "requirement": "==3.6.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pytest-cov@2.5.1", + "requirement": "==2.5.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pytest-vcr@0.3.0", + "requirement": "==0.3.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pyyaml@3.13", + "requirement": "==3.13", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/six@1.11.0", + "requirement": "==1.11.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/vcrpy@1.13.0", + "requirement": "==1.13.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/wrapt@1.10.11", + "requirement": "==1.10.11", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample3/Pipfile.lock b/tests/packagedcode/data/pypi/pipfile.lock/sample3/Pipfile.lock new file mode 100644 index 00000000000..0a7c06cfac9 --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample3/Pipfile.lock @@ -0,0 +1,28 @@ +{ + "_meta": { + "hash": { + "sha256": "98012973e54d083ef515d6438e1fcd8218aefac3da6adde13a26ee32af0b7e6f" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "python-baseconv": { + "hashes": [ + "sha256:0539f8bd0464013b05ad62e0a1673f0ac9086c76b43ebf9f833053527cd9931b" + ], + "index": "pypi", + "version": "==1.2.2" + } + }, + "develop": {} +} diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample3/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample3/output.expected.json new file mode 100644 index 00000000000..0b6be9bb0c5 --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample3/output.expected.json @@ -0,0 +1,44 @@ +{ + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:pypi/python-baseconv@1.2.2", + "requirement": "==1.2.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample4/Pipfile.lock b/tests/packagedcode/data/pypi/pipfile.lock/sample4/Pipfile.lock new file mode 100644 index 00000000000..da35acc25d9 --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample4/Pipfile.lock @@ -0,0 +1,151 @@ +{ + "_meta": { + "hash": { + "sha256": "24931cad8ca14fb20d62933a3be5a0544d1dc47f4c3bce54f17ce74037fc7c23" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "functools32": { + "file": "https://github.com/sarugaku/functools32/releases/download/3.2.3-2/functools32-3.2.3.post2-py2.py3-none-any.whl", + "hashes": [ + "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0", + "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d" + ], + "index": "pypi", + "markers": "python_version < '3.0'", + "version": "==3.2.3.post2" + }, + "passa": { + "editable": true, + "path": "." + }, + "urllib3": { + "hashes": [ + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" + ], + "index": "pypi", + "version": "==1.24.2" + } + }, + "develop": { + "alabaster": { + "hashes": [ + "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359", + "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02" + ], + "version": "==0.7.12" + }, + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "version": "==19.3.0" + }, + "certifi": { + "hashes": [ + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + ], + "version": "==2019.9.11" + }, + "markupsafe": { + "hashes": [ + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" + ], + "version": "==1.1.1" + }, + "packaging": { + "hashes": [ + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + ], + "version": "==19.2" + }, + "parver": { + "hashes": [ + "sha256:b8b2976fd8a73a0515465b2a265fd9b20cc25a6dc88bc1154fd5f60f10dad4db", + "sha256:d9ae08a2629105fdb83e4971ae8a04f1de5a3803d1dd928f6e181aeadb398180" + ], + "index": "pypi", + "version": "==0.2.0" + }, + "passa": { + "editable": true, + "path": "." + }, + "pkginfo": { + "hashes": [ + "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", + "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" + ], + "version": "==1.5.0.1" + }, + "pygments": { + "hashes": [ + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + ], + "version": "==2.4.2" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "toml": { + "hashes": [ + "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + ], + "version": "==0.10.0" + }, + "wheel": { + "hashes": [ + "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", + "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" + ], + "index": "pypi", + "version": "==0.32.3" + } + } +} diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample4/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample4/output.expected.json new file mode 100644 index 00000000000..f6301ab9fec --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample4/output.expected.json @@ -0,0 +1,140 @@ +{ + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:pypi/functools32@3.2.3.post2", + "requirement": "==3.2.3.post2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/urllib3@1.24.2", + "requirement": "==1.24.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/alabaster@0.7.12", + "requirement": "==0.7.12", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/attrs@19.3.0", + "requirement": "==19.3.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/certifi@2019.9.11", + "requirement": "==2019.9.11", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/markupsafe@1.1.1", + "requirement": "==1.1.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/packaging@19.2", + "requirement": "==19.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/parver@0.2.0", + "requirement": "==0.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pkginfo@1.5.0.1", + "requirement": "==1.5.0.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pygments@2.4.2", + "requirement": "==2.4.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/six@1.12.0", + "requirement": "==1.12.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/toml@0.10.0", + "requirement": "==0.10.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/wheel@0.32.3", + "requirement": "==0.32.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample5/Pipfile.lock b/tests/packagedcode/data/pypi/pipfile.lock/sample5/Pipfile.lock new file mode 100644 index 00000000000..2c3cd426416 --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample5/Pipfile.lock @@ -0,0 +1,103 @@ +{ + "_meta": { + "hash": { + "sha256": "813f8e1b624fd42eee7d681228d7aca1fce209e1d60bf21c3eb33a73f7268d57" + }, + "host-environment-markers": { + "implementation_name": "cpython", + "implementation_version": "0", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_python_implementation": "CPython", + "platform_release": "15.6.0", + "platform_system": "Darwin", + "platform_version": "Darwin Kernel Version 15.6.0: Fri Feb 17 10:21:18 PST 2017; root:xnu-3248.60.11.4.1~1/RELEASE_X86_64", + "python_full_version": "2.7.10", + "python_version": "2.7", + "sys_platform": "darwin" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "django": { + "hashes": [ + "sha256:679fc24b3e85bf5a07ca2f6d5c4cdf3d4477bbb02f43a6548335952cc75b5d23", + "sha256:3de62e71ce2cfbcdecb6e344cad04948506c8410ea5c6eab15c8f3b31b8ac1c0" + ], + "version": "==1.7.1" + }, + "feedparser": { + "hashes": [ + "sha256:ae099763f4538aa08c5021d42ba6ce5b9b6218e71423c96031153f379955481a", + "sha256:52542106f55d044a404f00bf328bd29e81e49c38a130a05be66a8e4dc4f9ff57", + "sha256:172c23932965f91ef58e23e5abf93412333eecdc04d9e015cd09056d6b9301b3" + ], + "version": "==5.1.1" + }, + "pyasn1": { + "hashes": [ + "sha256:f81c96761fca60d64b1c9b79ec2e40cf9495a745cf570613079ef324aeb9672b", + "sha256:7d626683e3d792cccc608da02498aff37ab4f3dafd8905d6bf755d11f9b26b43", + "sha256:e85895087905c65b5b594eb91f7522664c85545b147d5f4d4e7b1b07da8dcbdc", + "sha256:5a0db897b311d265cde49615cf783f1c78613138605cdd0f907ecfa5b2aba3ee", + "sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1", + "sha256:a7efe807c4b83a859e2735c692b92ed7b567cfddc4163763412920041d876c2b", + "sha256:b5a9ca48055b9a20f6d1b3d68e38692e5431c86a0f99ea602e61294e891fee5b", + "sha256:c07d6e587b2f928366b1f67c09bda026a3e6fcc99e80a744dc67f8fca3895626", + "sha256:d84c2aea3cf43780e9e6a19f4e4dddee9f6976519020e64e47c57e5c7a8c3dd2", + "sha256:758cb50abddc03e4563fd9e7f03db56e3e87b58c0bd01247360326e5c0c7ffa5", + "sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89", + "sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15" + ], + "version": "==0.4.2" + }, + "pycrypto": { + "hashes": [ + "sha256:f49d8aea2d7d65db9906c7d3b8b3a07fcae8387cf5cb06a7510383e211902d39" + ], + "version": "==2.4" + }, + "pyjwt": { + "hashes": [ + "sha256:2d30be6375be006a6fec531ba15ec9a7cf3ac88fd6ea3caa2b6f86a84c372acc", + "sha256:a0019f8119cd9a31d9c29e7b47256b24642829c10941ee20f513487f466201d9" + ], + "version": "==0.4.2" + }, + "raven": { + "hashes": [ + "sha256:7f14e651d321aedaf00635ab106b72520e4a6565d766b8fc64856e662c3acc8c" + ], + "version": "==1.9.4" + }, + "requests": { + "hashes": [ + "sha256:b5bd2e1b78d28051108ebaa6248750221f9ccef52b4f054cb727de61b0406de0", + "sha256:1266921f1bed5fbf364cd83cf239b6d7b3ea5c32ccccbc93980d9ba12cdcfd02" + ], + "version": "==2.2.1" + }, + "rsa": { + "hashes": [ + "sha256:0c7fde631f84f89e89ec671a9c58feb01ea25fab177dca08ba08650c548d48d5", + "sha256:9f1b6d4015cdf788273ff329d43004a7abf43971e9b06160765bad5227e4d70a" + ], + "version": "==3.4" + }, + "simplejson": { + "hashes": [ + "sha256:ac0f5122a213ef35c3af6464a2885aef5b56a4954f003eac767dd8e077949885" + ], + "version": "==2.4.0" + } + }, + "develop": {} +} diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample5/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample5/output.expected.json new file mode 100644 index 00000000000..f6b152e999d --- /dev/null +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample5/output.expected.json @@ -0,0 +1,108 @@ +{ + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "root_path": null, + "dependencies": [ + { + "purl": "pkg:pypi/django@1.7.1", + "requirement": "==1.7.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/feedparser@5.1.1", + "requirement": "==5.1.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pyasn1@0.4.2", + "requirement": "==0.4.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pycrypto@2.4", + "requirement": "==2.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/pyjwt@0.4.2", + "requirement": "==0.4.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/raven@1.9.4", + "requirement": "==1.9.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/requests@2.2.1", + "requirement": "==2.2.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/rsa@3.4", + "requirement": "==3.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + }, + { + "purl": "pkg:pypi/simplejson@2.4.0", + "requirement": "==2.4.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true + } + ], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file From 206ca1d810e06b36eca2f3d47de493926975f66b Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Tue, 14 Jul 2020 08:56:54 +0530 Subject: [PATCH 12/16] tests code to check sample Pipfile.lock files Signed-off-by: rpotter12 --- tests/packagedcode/data/plugin/help.txt | 2 +- tests/packagedcode/test_pypi.py | 75 +++++++++++++++++-------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/tests/packagedcode/data/plugin/help.txt b/tests/packagedcode/data/plugin/help.txt index b2c3a83b667..13ef7a2bbcf 100644 --- a/tests/packagedcode/data/plugin/help.txt +++ b/tests/packagedcode/data/plugin/help.txt @@ -203,7 +203,7 @@ Package: nuget -------------------------------------------- Package: pypi class: packagedcode.pypi:PythonPackage - metafiles: metadata.json, *setup.py, PKG-INFO, *.whl, *.egg, *requirements*.txt, *requirements*.in + metafiles: metadata.json, *setup.py, PKG-INFO, *.whl, *.egg, *requirements*.txt, *requirements*.in, *Pipfile.lock extensions: .egg, .whl, .pyz, .pex -------------------------------------------- diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index 5c3858cc914..d187d9b58fb 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -271,46 +271,76 @@ def test_pkginfo_parse_with_wheelfile_with_parse_function(self): def test_requirements_txt_sample1(self): test_file = self.get_test_loc('pypi/requirements_txt/sample1/requirements.txt') - package = pypi.parse_requirements_txt(test_file) + package = pypi.parse_dependency_file(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample1/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample2(self): test_file = self.get_test_loc('pypi/requirements_txt/sample2/sample-requirements.txt') - package = pypi.parse_requirements_txt(test_file) + package = pypi.parse_dependency_file(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample2/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample3(self): test_file = self.get_test_loc('pypi/requirements_txt/sample3/requirements-dev.txt') - package = pypi.parse_requirements_txt(test_file) + package = pypi.parse_dependency_file(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample3/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample4(self): test_file = self.get_test_loc('pypi/requirements_txt/sample4/requirements.in') - package = pypi.parse_requirements_txt(test_file) + package = pypi.parse_dependency_file(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample4/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample5(self): test_file = self.get_test_loc('pypi/requirements_txt/sample5/requirements-test.txt') - package = pypi.parse_requirements_txt(test_file) + package = pypi.parse_dependency_file(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample5/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample6(self): test_file = self.get_test_loc('pypi/requirements_txt/sample6/requirements-dev.in') - package = pypi.parse_requirements_txt(test_file) + package = pypi.parse_dependency_file(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample6/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample7(self): test_file = self.get_test_loc('pypi/requirements_txt/sample7/requirements-test.in') - package = pypi.parse_requirements_txt(test_file) + package = pypi.parse_dependency_file(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample7/output.expected.json') self.check_package(package, expected_loc, regen=False) + def test_pipfile_lock_sample1(self): + test_file = self.get_test_loc('pypi/pipfile.lock/sample1/Pipfile.lock') + package = pypi.parse_dependency_file(test_file) + expected_loc = self.get_test_loc('pypi/pipfile.lock/sample1/output.expected.json') + self.check_package(package, expected_loc, regen=False) + + def test_pipfile_lock_sample2(self): + test_file = self.get_test_loc('pypi/pipfile.lock/sample2/Pipfile.lock') + package = pypi.parse_dependency_file(test_file) + expected_loc = self.get_test_loc('pypi/pipfile.lock/sample2/output.expected.json') + self.check_package(package, expected_loc, regen=False) + + def test_pipfile_lock_sample3(self): + test_file = self.get_test_loc('pypi/pipfile.lock/sample3/Pipfile.lock') + package = pypi.parse_dependency_file(test_file) + expected_loc = self.get_test_loc('pypi/pipfile.lock/sample3/output.expected.json') + self.check_package(package, expected_loc, regen=False) + + def test_pipfile_lock_sample4(self): + test_file = self.get_test_loc('pypi/pipfile.lock/sample4/Pipfile.lock') + package = pypi.parse_dependency_file(test_file) + expected_loc = self.get_test_loc('pypi/pipfile.lock/sample4/output.expected.json') + self.check_package(package, expected_loc, regen=False) + + def test_pipfile_lock_sample5(self): + test_file = self.get_test_loc('pypi/pipfile.lock/sample5/Pipfile.lock') + package = pypi.parse_dependency_file(test_file) + expected_loc = self.get_test_loc('pypi/pipfile.lock/sample5/output.expected.json') + self.check_package(package, expected_loc, regen=False) + def test_parse_with_dparse(self): test_file = self.get_test_loc('pypi/dparse/requirements.txt') dependencies = pypi.parse_with_dparse(test_file) @@ -319,22 +349,23 @@ def test_parse_with_dparse(self): FILENAME_LIST = [ - 'requirements.txt', - 'sample-requirements.txt', - 'requirements-test.txt', - 'sample-requirements-test.txt', - 'requirements-dev.txt', - 'sample-requirements-dev.txt', - 'requirements.in', - 'sample-requirements.in', - 'requirements-test.in', - 'sample-requirements-test.in', - 'requirements-dev.in', - 'sample-requirements-dev.in' + ('requirements.txt', 'requirements.txt'), + ('sample-requirements.txt', 'requirements.txt'), + ('requirements-test.txt', 'requirements.txt'), + ('sample-requirements-test.txt', 'requirements.txt'), + ('requirements-dev.txt', 'requirements.txt'), + ('sample-requirements-dev.txt', 'requirements.txt'), + ('requirements.in', 'requirements.txt'), + ('sample-requirements.in', 'requirements.txt'), + ('requirements-test.in', 'requirements.txt'), + ('sample-requirements-test.in', 'requirements.txt'), + ('requirements-dev.in', 'requirements.txt'), + ('sample-requirements-dev.in', 'requirements.txt'), + ('Pipfile.lock', 'Pipfile.lock') ] class TestFiletype(object): - @pytest.mark.parametrize('filename', FILENAME_LIST) - def test_file_type(self, filename): + @pytest.mark.parametrize('filename, expected_filename', FILENAME_LIST) + def test_file_type(self, filename, expected_filename): filename = pypi.get_dependency_type(filename) - assert filename == 'requirements.txt' + assert filename == expected_filename From 837ab1bfc87a505c3e3c2f28a237d3239388845e Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Sat, 18 Jul 2020 07:37:30 +0530 Subject: [PATCH 13/16] detect sha256 from Pipfile.lock Signed-off-by: rpotter12 --- src/packagedcode/pypi.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 0602ed6c19b..53e3c6d7f9d 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -119,9 +119,9 @@ def parse(location): file_name = fileutils.file_name(location) parsers = { 'setup.py': parse_setup_py, - 'requirements.txt': parse_dependency_file, - 'requirements.in': parse_dependency_file, - 'Pipfile.lock': parse_dependency_file, + 'requirements.txt': parse_requirements_txt, + 'requirements.in': parse_requirements_txt, + 'Pipfile.lock': parse_pipfile_lock, 'metadata.json': parse_metadata, 'PKG-INFO': parse_pkg_info, '.whl': parse_wheel, @@ -274,14 +274,39 @@ def parse_with_dparse(location): return package_dependencies -def parse_dependency_file(location): +def parse_requirements_txt(location): """ - Return a package built from Python dependency files. + Return a package built from Python requirements.txt files. """ package_dependencies = parse_with_dparse(location) return PythonPackage(dependencies=package_dependencies) +def parse_pipfile_lock(location): + """ + Return package built from Python Pipfile.lock files. + """ + with open('Pipfile.lock', 'r') as f: + content = f.read() + + try: + data = json.loads(content, object_pairs_hook=OrderedDict) + except json.decoder.JSONDecodeError: + data = {} + + sha256 = None + if '_meta' in data: + for name, meta in data['_meta'].items(): + if name=='hash': + sha256 = meta.get('sha256') + + package_dependencies = parse_with_dparse(location) + return PythonPackage( + sha256=sha256, + dependencies=package_dependencies + ) + + def parse_setup_py(location): """ Return a package built from setup.py data. From d330ab304b1169226580fede0cc41122046edb35 Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Sat, 18 Jul 2020 08:01:08 +0530 Subject: [PATCH 14/16] add test for pipfile.lock Signed-off-by: rpotter12 --- src/packagedcode/pypi.py | 2 +- .../pipfile.lock/sample1/output.expected.json | 2 +- .../pipfile.lock/sample2/output.expected.json | 2 +- .../pipfile.lock/sample3/output.expected.json | 2 +- .../pipfile.lock/sample4/output.expected.json | 2 +- .../pipfile.lock/sample5/output.expected.json | 2 +- tests/packagedcode/test_pypi.py | 24 +++++++++---------- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 53e3c6d7f9d..83380de064b 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -286,7 +286,7 @@ def parse_pipfile_lock(location): """ Return package built from Python Pipfile.lock files. """ - with open('Pipfile.lock', 'r') as f: + with open(location, 'r') as f: content = f.read() try: diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample1/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample1/output.expected.json index 5ead799c9ca..f860483a870 100644 --- a/tests/packagedcode/data/pypi/pipfile.lock/sample1/output.expected.json +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample1/output.expected.json @@ -15,7 +15,7 @@ "size": null, "sha1": null, "md5": null, - "sha256": null, + "sha256": "813f8e1b624fd42eee7d681228d7aca1fce209e1d60bf21c3eb33a73f7268d57", "sha512": null, "bug_tracking_url": null, "code_view_url": null, diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample2/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample2/output.expected.json index eb95b3cb5b0..153a17849e5 100644 --- a/tests/packagedcode/data/pypi/pipfile.lock/sample2/output.expected.json +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample2/output.expected.json @@ -15,7 +15,7 @@ "size": null, "sha1": null, "md5": null, - "sha256": null, + "sha256": "6e45251662433bf51f96fb3d2204b65416fece329d60e6235c0f0edc416cfe24", "sha512": null, "bug_tracking_url": null, "code_view_url": null, diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample3/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample3/output.expected.json index 0b6be9bb0c5..a572e4fb553 100644 --- a/tests/packagedcode/data/pypi/pipfile.lock/sample3/output.expected.json +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample3/output.expected.json @@ -15,7 +15,7 @@ "size": null, "sha1": null, "md5": null, - "sha256": null, + "sha256": "98012973e54d083ef515d6438e1fcd8218aefac3da6adde13a26ee32af0b7e6f", "sha512": null, "bug_tracking_url": null, "code_view_url": null, diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample4/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample4/output.expected.json index f6301ab9fec..474e32a20e2 100644 --- a/tests/packagedcode/data/pypi/pipfile.lock/sample4/output.expected.json +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample4/output.expected.json @@ -15,7 +15,7 @@ "size": null, "sha1": null, "md5": null, - "sha256": null, + "sha256": "24931cad8ca14fb20d62933a3be5a0544d1dc47f4c3bce54f17ce74037fc7c23", "sha512": null, "bug_tracking_url": null, "code_view_url": null, diff --git a/tests/packagedcode/data/pypi/pipfile.lock/sample5/output.expected.json b/tests/packagedcode/data/pypi/pipfile.lock/sample5/output.expected.json index f6b152e999d..46eaaf989d2 100644 --- a/tests/packagedcode/data/pypi/pipfile.lock/sample5/output.expected.json +++ b/tests/packagedcode/data/pypi/pipfile.lock/sample5/output.expected.json @@ -15,7 +15,7 @@ "size": null, "sha1": null, "md5": null, - "sha256": null, + "sha256": "813f8e1b624fd42eee7d681228d7aca1fce209e1d60bf21c3eb33a73f7268d57", "sha512": null, "bug_tracking_url": null, "code_view_url": null, diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index d187d9b58fb..9fb8c9f9d4c 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -271,73 +271,73 @@ def test_pkginfo_parse_with_wheelfile_with_parse_function(self): def test_requirements_txt_sample1(self): test_file = self.get_test_loc('pypi/requirements_txt/sample1/requirements.txt') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_requirements_txt(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample1/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample2(self): test_file = self.get_test_loc('pypi/requirements_txt/sample2/sample-requirements.txt') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_requirements_txt(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample2/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample3(self): test_file = self.get_test_loc('pypi/requirements_txt/sample3/requirements-dev.txt') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_requirements_txt(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample3/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample4(self): test_file = self.get_test_loc('pypi/requirements_txt/sample4/requirements.in') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_requirements_txt(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample4/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample5(self): test_file = self.get_test_loc('pypi/requirements_txt/sample5/requirements-test.txt') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_requirements_txt(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample5/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample6(self): test_file = self.get_test_loc('pypi/requirements_txt/sample6/requirements-dev.in') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_requirements_txt(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample6/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_requirements_txt_sample7(self): test_file = self.get_test_loc('pypi/requirements_txt/sample7/requirements-test.in') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_requirements_txt(test_file) expected_loc = self.get_test_loc('pypi/requirements_txt/sample7/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_pipfile_lock_sample1(self): test_file = self.get_test_loc('pypi/pipfile.lock/sample1/Pipfile.lock') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_pipfile_lock(test_file) expected_loc = self.get_test_loc('pypi/pipfile.lock/sample1/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_pipfile_lock_sample2(self): test_file = self.get_test_loc('pypi/pipfile.lock/sample2/Pipfile.lock') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_pipfile_lock(test_file) expected_loc = self.get_test_loc('pypi/pipfile.lock/sample2/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_pipfile_lock_sample3(self): test_file = self.get_test_loc('pypi/pipfile.lock/sample3/Pipfile.lock') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_pipfile_lock(test_file) expected_loc = self.get_test_loc('pypi/pipfile.lock/sample3/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_pipfile_lock_sample4(self): test_file = self.get_test_loc('pypi/pipfile.lock/sample4/Pipfile.lock') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_pipfile_lock(test_file) expected_loc = self.get_test_loc('pypi/pipfile.lock/sample4/output.expected.json') self.check_package(package, expected_loc, regen=False) def test_pipfile_lock_sample5(self): test_file = self.get_test_loc('pypi/pipfile.lock/sample5/Pipfile.lock') - package = pypi.parse_dependency_file(test_file) + package = pypi.parse_pipfile_lock(test_file) expected_loc = self.get_test_loc('pypi/pipfile.lock/sample5/output.expected.json') self.check_package(package, expected_loc, regen=False) From e01e343b7c73bda90f59db3f29e30f4dfbc603a4 Mon Sep 17 00:00:00 2001 From: rpotter12 Date: Wed, 22 Jul 2020 19:58:26 +0530 Subject: [PATCH 15/16] improvisation in doc strings Signed-off-by: rpotter12 --- src/packagedcode/pypi.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 83380de064b..8af96628688 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -276,7 +276,7 @@ def parse_with_dparse(location): def parse_requirements_txt(location): """ - Return a package built from Python requirements.txt files. + Return a PythonPackage built from a Python requirements.txt files at location. """ package_dependencies = parse_with_dparse(location) return PythonPackage(dependencies=package_dependencies) @@ -284,15 +284,12 @@ def parse_requirements_txt(location): def parse_pipfile_lock(location): """ - Return package built from Python Pipfile.lock files. + Return a PythonPackage built from a Python Pipfile.lock file at location. """ - with open(location, 'r') as f: + with open(location) as f: content = f.read() - try: - data = json.loads(content, object_pairs_hook=OrderedDict) - except json.decoder.JSONDecodeError: - data = {} + data = json.loads(content, object_pairs_hook=OrderedDict) sha256 = None if '_meta' in data: @@ -309,7 +306,7 @@ def parse_pipfile_lock(location): def parse_setup_py(location): """ - Return a package built from setup.py data. + Return a PythonPackage built from setup.py data. """ if not location or not location.endswith('setup.py'): return @@ -457,7 +454,7 @@ def parse_metadata(location): def parse_pkg_info(location): """ - Return a Package from a a 'PKG-INFO' file at 'location' or None. + Return a PythonPackage from a a 'PKG-INFO' file at 'location' or None. """ if not location: return From 9c0fd6cf7e972237db829c3f6c3fdbe2ae617f53 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jul 2020 11:11:22 +0530 Subject: [PATCH 16/16] Adding setuptools to thirdparty directory Adding deps setuptools which is deps of zc.lockfile-2.0 Signed-off-by: Abhishek Kumar --- .../setuptools-41.2.0-py2.py3-none-any.whl | Bin 0 -> 576332 bytes ...etuptools-41.2.0-py2.py3-none-any.whl.ABOUT | 15 +++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 thirdparty/setuptools-41.2.0-py2.py3-none-any.whl create mode 100644 thirdparty/setuptools-41.2.0-py2.py3-none-any.whl.ABOUT diff --git a/thirdparty/setuptools-41.2.0-py2.py3-none-any.whl b/thirdparty/setuptools-41.2.0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..82df6f63f4ee97380af0a29d8825ae775333b86d GIT binary patch literal 576332 zcmV)0K+eBVO9KQH0000807^#{Ph3yBMQH#40Db@f01p5F0A*owd0%O6b97;BY%Xwl zEzB_vgFp;K(SA>1xq&nsAQEMYv|M0hkwsZ)Z6nqcxjhgu)im!j^RyeqH`7bn?`