From 70253680c4443e4becbb4e35633bf5d72e0ca88b Mon Sep 17 00:00:00 2001 From: rakesh balusa Date: Wed, 13 Apr 2016 17:10:10 -0700 Subject: [PATCH] #253 Recognize Python packages using setup.py files Signed-off-by: Philippe Ombredanne --- src/packagedcode/pypi.py | 85 +++++++++++++++++ tests/packagedcode/data/pypi/setup.py | 124 +++++++++++++++++++++++++ tests/packagedcode/data/pypi/setup2.py | 124 +++++++++++++++++++++++++ tests/packagedcode/test_pypi.py | 51 ++++++++++ 4 files changed, 384 insertions(+) create mode 100644 src/packagedcode/pypi.py create mode 100644 tests/packagedcode/data/pypi/setup.py create mode 100644 tests/packagedcode/data/pypi/setup2.py create mode 100644 tests/packagedcode/test_pypi.py diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py new file mode 100644 index 00000000000..a314d7fd912 --- /dev/null +++ b/src/packagedcode/pypi.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2015 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function + +import logging +import re + +from packagedcode.models import AssertedLicense +from packagedcode.models import PythonPackage + +""" +Handle Python PyPi packages +""" + + +logger = logging.getLogger(__name__) +# import sys +# logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) +# logger.setLevel(logging.DEBUG) + + +def get_attribute(setup_location, attribute): + """ + Return the value specified for a given 'attribute' mentioned in a 'setup.py' + file. + Example : + setup( + name='requests', + version='1.0', + ) + 'requests' is returned for the attribute 'name' + """ + setup_text = open(setup_location, 'rb').read() + setup_text = setup_text.replace('\n', '') + # FIXME Use a valid parser for parsing 'setup.py' + values = re.findall('setup\(.*?'+attribute+'=[\"\']{1}.*?\',', setup_text) + if len(values) > 1: + return + else: + values = ''.join(values) + output = re.sub('setup\(.*?'+attribute+'=[\"\']{1}', '', values) + if output.endswith('\','): + return output.replace('\',', '') + else: + return output + + +def parse(location): + """ + Parse a 'setup.py' and return a PythonPackage object. + """ + if not location.endswith('setup.py'): + return + package = PythonPackage( + name=get_attribute(location, 'name'), + homepage_url=get_attribute(location, 'url'), + description=get_attribute(location, 'description'), + version=get_attribute(location, 'version'), + authors=[get_attribute(location, 'author')], + asserted_licenses=[AssertedLicense(license=get_attribute(location, 'license'))], + ) + return package diff --git a/tests/packagedcode/data/pypi/setup.py b/tests/packagedcode/data/pypi/setup.py new file mode 100644 index 00000000000..b66230b99dd --- /dev/null +++ b/tests/packagedcode/data/pypi/setup.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from __future__ import absolute_import, print_function + +import io +import os +import re +from glob import glob +from os.path import basename +from os.path import dirname +from os.path import join +from os.path import splitext + +from setuptools import find_packages +from setuptools import setup + + +def read(*names, **kwargs): + return io.open( + join(dirname(__file__), *names), + encoding=kwargs.get('encoding', 'utf8') + ).read() + + +long_description = '%s\n%s' % ( + read('README.rst'), + re.sub(':obj:`~?(.*?)`', r'``\1``', read('CHANGELOG.rst')) +) + +setup( + name='scancode-toolkit', + version='1.5.0', + license='Apache-2.0 with ScanCode acknowledgment and CC0-1.0 and others', + description='ScanCode is a tool to scan code for license, copyright and other interesting facts.', + long_description=long_description, + author='ScanCode', + author_email='info@scancode.io', + url='https://github.com/nexB/scancode-toolkit', + packages=find_packages('src'), + package_dir={'': 'src'}, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + zip_safe=False, + classifiers=[ + # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'License :: OSI Approved :: CC0', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Topic :: Utilities', + ], + keywords=[ + 'license', 'filetype', 'urn', 'date', 'codec', + ], + install_requires=[ + # cluecode + 'py2-ipaddress >= 2.0, <3.0', + 'url >= 0.1.4', + 'publicsuffix2', + # TODO: upgrade to nltk==3.0.1 + 'nltk >= 2.0b4, <3.0.0', + + # extractcode + 'patch >= 1.14.2, < 1.15 ', + # to work around bug http://bugs.python.org/issue19839 + # on multistream bzip2 files + 'bz2file >= 0.98', + + # licensedcode + 'PyYAML >= 3.0, <4.0', + + # textcode + 'Beautifulsoup >= 3.2.0, <4.0.0', + 'Beautifulsoup4 >= 4.3.0, <5.0.0', + 'html5lib', + 'six', + + # typecode and textcode + 'pygments >= 2.0.0, <3.0.0', + 'pdfminer >= 20140328', + + # typecode + 'chardet >= 2.1.1, <3.0.0', + 'binaryornot >= 0.4.0', + + # scancode and AboutCode + 'click >= 4.0.0, < 5.0.0', + 'jinja2 >= 2.7.0, < 3.0.0', + 'MarkupSafe >= 0.23', + 'colorama', + + # AboutCode + 'about-code-tool >= 0.9.0', + + # packagedcode + 'requests >= 2.7.0, < 3.0.0', + ], + + extras_require={ + 'base': [ + 'certifi', + 'setuptools', + 'wheel', + 'pip', + 'wincertstore', + ], + 'dev': [ + 'pytest', + 'execnet', + 'py', + 'pytest-xdist', + 'bumpversion', + ], + + }, + entry_points={ + 'console_scripts': [ + 'scancode = scancode.cli:scancode', + 'extractcode = scancode.extract_cli:extractcode', + ], + }, +) diff --git a/tests/packagedcode/data/pypi/setup2.py b/tests/packagedcode/data/pypi/setup2.py new file mode 100644 index 00000000000..b66230b99dd --- /dev/null +++ b/tests/packagedcode/data/pypi/setup2.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from __future__ import absolute_import, print_function + +import io +import os +import re +from glob import glob +from os.path import basename +from os.path import dirname +from os.path import join +from os.path import splitext + +from setuptools import find_packages +from setuptools import setup + + +def read(*names, **kwargs): + return io.open( + join(dirname(__file__), *names), + encoding=kwargs.get('encoding', 'utf8') + ).read() + + +long_description = '%s\n%s' % ( + read('README.rst'), + re.sub(':obj:`~?(.*?)`', r'``\1``', read('CHANGELOG.rst')) +) + +setup( + name='scancode-toolkit', + version='1.5.0', + license='Apache-2.0 with ScanCode acknowledgment and CC0-1.0 and others', + description='ScanCode is a tool to scan code for license, copyright and other interesting facts.', + long_description=long_description, + author='ScanCode', + author_email='info@scancode.io', + url='https://github.com/nexB/scancode-toolkit', + packages=find_packages('src'), + package_dir={'': 'src'}, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + zip_safe=False, + classifiers=[ + # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'License :: OSI Approved :: CC0', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Topic :: Utilities', + ], + keywords=[ + 'license', 'filetype', 'urn', 'date', 'codec', + ], + install_requires=[ + # cluecode + 'py2-ipaddress >= 2.0, <3.0', + 'url >= 0.1.4', + 'publicsuffix2', + # TODO: upgrade to nltk==3.0.1 + 'nltk >= 2.0b4, <3.0.0', + + # extractcode + 'patch >= 1.14.2, < 1.15 ', + # to work around bug http://bugs.python.org/issue19839 + # on multistream bzip2 files + 'bz2file >= 0.98', + + # licensedcode + 'PyYAML >= 3.0, <4.0', + + # textcode + 'Beautifulsoup >= 3.2.0, <4.0.0', + 'Beautifulsoup4 >= 4.3.0, <5.0.0', + 'html5lib', + 'six', + + # typecode and textcode + 'pygments >= 2.0.0, <3.0.0', + 'pdfminer >= 20140328', + + # typecode + 'chardet >= 2.1.1, <3.0.0', + 'binaryornot >= 0.4.0', + + # scancode and AboutCode + 'click >= 4.0.0, < 5.0.0', + 'jinja2 >= 2.7.0, < 3.0.0', + 'MarkupSafe >= 0.23', + 'colorama', + + # AboutCode + 'about-code-tool >= 0.9.0', + + # packagedcode + 'requests >= 2.7.0, < 3.0.0', + ], + + extras_require={ + 'base': [ + 'certifi', + 'setuptools', + 'wheel', + 'pip', + 'wincertstore', + ], + 'dev': [ + 'pytest', + 'execnet', + 'py', + 'pytest-xdist', + 'bumpversion', + ], + + }, + entry_points={ + 'console_scripts': [ + 'scancode = scancode.cli:scancode', + 'extractcode = scancode.extract_cli:extractcode', + ], + }, +) diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py new file mode 100644 index 00000000000..3ef7592d85f --- /dev/null +++ b/tests/packagedcode/test_pypi.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2015 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function + +import os.path + +from commoncode.testcase import FileBasedTesting + +from packagedcode import pypi + + +class TestPyPi(FileBasedTesting): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_parse(self): + test_file = self.get_test_loc('pypi/setup.py') + package = pypi.parse(test_file) + assert 'scancode-toolkit' == package.name + assert '1.5.0' == package.version + assert 'ScanCode' == package.authors[0] + assert 'ScanCode is a tool to scan code for license, copyright and other interesting facts.' == package.description + assert 'https://github.com/nexB/scancode-toolkit' == package.homepage_url + + def test_get_attribute(self): + test_file = self.get_test_loc('pypi/setup2.py') + assert 'scancode-toolkit' == pypi.get_attribute(test_file, 'name') + assert '1.5.0' == pypi.get_attribute(test_file, 'version') + assert 'ScanCode' == pypi.get_attribute(test_file, 'author')