From 1d1690149a4a66625580088de4520ca4d9ce02e2 Mon Sep 17 00:00:00 2001 From: satwikkansal Date: Sun, 25 Jun 2017 12:31:52 +0530 Subject: [PATCH 1/5] InfoExtractor: Add spec_references field Adds a class attribute named spec_references which contains links to the issues/documentations for relevant specs of supported files. Related to https://github.com/coala/coala-quickstart/issues/126 --- coala_quickstart/info_extraction/InfoExtractor.py | 3 +++ tests/info_extraction/InfoExtractorTest.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/coala_quickstart/info_extraction/InfoExtractor.py b/coala_quickstart/info_extraction/InfoExtractor.py index 81a360d..f8c9565 100644 --- a/coala_quickstart/info_extraction/InfoExtractor.py +++ b/coala_quickstart/info_extraction/InfoExtractor.py @@ -8,6 +8,9 @@ class InfoExtractor: # tuple of file globs supported by the extractor. supported_file_globs = tuple() + # Links to the issues/documentations for relevant specs of supported files. + spec_references = [] + # tuple of ``Info`` classes that can be extracted. supported_info_kinds = (Info,) diff --git a/tests/info_extraction/InfoExtractorTest.py b/tests/info_extraction/InfoExtractorTest.py index 442064d..c59213c 100644 --- a/tests/info_extraction/InfoExtractorTest.py +++ b/tests/info_extraction/InfoExtractorTest.py @@ -18,6 +18,7 @@ class AnotherDummyInfo(Info): description = 'Another such information.' class DummyInfoExtractor(InfoExtractor): + spec_references = ['some/dummy/link', 'another/dummy/link'] def parse_file(self, fname, file_content): return file_content @@ -134,6 +135,7 @@ def test_multiple_target_globs(self): extracted_info[tf]['DummyInfo'][0].extractor, InfoExtractor) + def test_multiple_information(self): target_filenames = ['target_file_1', ] @@ -266,3 +268,13 @@ def test_supported_info_kinds(self): "information kinds of WrongSupportedInfoExtractor")): uut.extract_information() + + def test_spec_references_filed(self): + uut = self.DummyInfoExtractor + self.assertEqual(len(uut.spec_references), 2) + self.assertEqual( + uut.spec_references, + ['some/dummy/link', 'another/dummy/link']) + + uut = self.DummyMultiInfoExtractor + self.assertEqual(uut.spec_references, []) From a3481136ee62c0728d40f647ef418520185e05b5 Mon Sep 17 00:00:00 2001 From: satwikkansal Date: Sun, 25 Jun 2017 12:42:42 +0530 Subject: [PATCH 2/5] Info: Add example_values field. Adds a class attribute named example_values illustrating sample information values that are allowed to be stored inside the Info class. Closes https://github.com/coala/coala-quickstart/issues/125 --- coala_quickstart/info_extraction/Info.py | 5 ++++- tests/info_extraction/InfoTest.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/coala_quickstart/info_extraction/Info.py b/coala_quickstart/info_extraction/Info.py index 4fbc4b6..8a15f97 100644 --- a/coala_quickstart/info_extraction/Info.py +++ b/coala_quickstart/info_extraction/Info.py @@ -2,11 +2,14 @@ class Info: - description = 'Some information' + description = 'Some Description.' # type signature for the information value. value_type = (object,) + # Some example values for reference that also match the value_type. + example_values = [] + def __init__(self, source, value, diff --git a/tests/info_extraction/InfoTest.py b/tests/info_extraction/InfoTest.py index 05d196b..f269e83 100644 --- a/tests/info_extraction/InfoTest.py +++ b/tests/info_extraction/InfoTest.py @@ -27,9 +27,11 @@ def find_information(self, fname, parsed_file): class InfoA(Info): description = 'Information A' value_type = (str, int) + example_values = ['coala', 420] class InfoB(Info): description = "Info class without value_type" + example_values = [["literally", "anything"]] self.info_a = InfoA( 'source_file', @@ -43,7 +45,8 @@ def test_main(self): self.assertEqual(self.base_info.name, 'Info') self.assertEqual(self.base_info.value, 'base_info_value') self.assertEqual(self.base_info.source, 'source_file') - self.assertEqual(self.base_info.description, 'Some information') + self.assertEqual(self.base_info.description, 'Some Description.') + self.assertEqual(len(self.base_info.example_values), 0) self.assertIsInstance(self.base_info.extractor, InfoExtractor) def test_derived_instances(self): @@ -52,6 +55,7 @@ def test_derived_instances(self): self.assertEqual(self.info_a.source, 'source_file') self.assertEqual(self.info_a.extra_param, 'extra_param_value') self.assertEqual(self.info_a.description, 'Information A') + self.assertEqual(self.info_a.example_values, ['coala', 420]) def test_value_type(self): with self.assertRaisesRegexp( From 5ffd9b8f24059f2630b6a65d08e88605e46e50c0 Mon Sep 17 00:00:00 2001 From: satwikkansal Date: Sat, 17 Jun 2017 21:14:02 +0530 Subject: [PATCH 3/5] info_extraction: Add Info classes Adds Info classes relevant to extractors of package.json and other meta-files. Related to https://github.com/coala/coala-quickstart/issues/100 --- .../info_extraction/Information.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 coala_quickstart/info_extraction/Information.py diff --git a/coala_quickstart/info_extraction/Information.py b/coala_quickstart/info_extraction/Information.py new file mode 100644 index 0000000..933ef68 --- /dev/null +++ b/coala_quickstart/info_extraction/Information.py @@ -0,0 +1,58 @@ +from coala_quickstart.info_extraction.Info import Info + + +class LicenseUsedInfo(Info): + description = "License of the project." + value_type = (str,) + example_values = ["MIT", "GPL-3", "Apache-2.0"] + + +class VersionInfo(Info): + description = "Version information, see http://semver.org/" + value_type = (str,) + example_values = [">=1.2.7", "~1.2.3", "^0.2"] + + +class ProjectDependencyInfo(Info): + description = "Dependency of the project." + value_type = (str,) + example_values = ['some_npm_package_name', 'some_gem_name', 'other_dep'] + + def __init__(self, + source, + value, + extractor=None, + version=None, + url=''): + super().__init__(source, value, extractor, version=version, url=url) + + +class PathsInfo(Info): + description = "File path globs mentioned in the file." + value_type = ([str],) + example_values = ["**.py", "dev/tests/**"] + + +class IncludePathsInfo(PathsInfo): + description = "Target files to perform analysis." + + +class IgnorePathsInfo(PathsInfo): + description = "Files to ignore during analysis." + + +class ManFilesInfo(Info): + description = "Filenames to put in place for the man program to find." + value_type = (str, [str]) + example_values = ["./man/doc.1", ["./man/foo.1", "./man/bar.1"]] + + def __init__(self, + source, + value, + extractor=None, + keyword=""): + """ + :param keyword: Primary keyword for ``man`` command that would display + the man pages provided in value argument. + """ + super().__init__(source, value, extractor, keyword=keyword) From 299c5e23917790da3c84911c0f22ca11da4b3603 Mon Sep 17 00:00:00 2001 From: satwikkansal Date: Sat, 17 Jun 2017 21:16:38 +0530 Subject: [PATCH 4/5] info_extractors: Add package.json extractor Adds ``PackageJSONInfoExtractor`` class to extract relevant information from package.json file. Closes https://github.com/coala/coala-quickstart/issues/100 --- .../PackageJSONInfoExtractor.py | 61 ++++++++++++++ coala_quickstart/info_extractors/__init__.py | 0 .../PackageJSONInfoExtractorTest.py | 84 +++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 coala_quickstart/info_extractors/PackageJSONInfoExtractor.py create mode 100644 coala_quickstart/info_extractors/__init__.py create mode 100644 tests/info_extractors/PackageJSONInfoExtractorTest.py diff --git a/coala_quickstart/info_extractors/PackageJSONInfoExtractor.py b/coala_quickstart/info_extractors/PackageJSONInfoExtractor.py new file mode 100644 index 0000000..84829a2 --- /dev/null +++ b/coala_quickstart/info_extractors/PackageJSONInfoExtractor.py @@ -0,0 +1,61 @@ +import json +import logging + +from coala_quickstart.info_extraction.InfoExtractor import InfoExtractor +from coala_quickstart.info_extraction.Information import ( + LicenseUsedInfo, ProjectDependencyInfo, IncludePathsInfo, ManFilesInfo, + VersionInfo) + + +class PackageJSONInfoExtractor(InfoExtractor): + supported_file_globs = ("package.json",) + + spec_references = [ + "https://docs.npmjs.com/files/package.json", + "https://gitlab.com/coala/GSoC-2017/issues/167"] + + supported_info_kinds = ( + LicenseUsedInfo, + ProjectDependencyInfo, + IncludePathsInfo, + ManFilesInfo) + + def parse_file(self, fname, file_content): + parsed_file = {} + + try: + parsed_file = json.loads(file_content) + except Exception: + logging.warning("Error while parsing the file {}".format(fname)) + + return parsed_file + + def find_information(self, fname, parsed_file): + results = [] + + if parsed_file.get("license"): + results.append( + LicenseUsedInfo(fname, parsed_file["license"])) + + if parsed_file.get("dependencies"): + for package_name, version_range in ( + parsed_file["dependencies"].items()): + results.append( + ProjectDependencyInfo( + fname, + package_name, + self.__class__.__name__, + VersionInfo(fname, version_range))) + + if parsed_file.get("files"): + results.append( + IncludePathsInfo(fname, parsed_file["files"])) + + if parsed_file.get("man"): + results.append( + ManFilesInfo( + fname, + parsed_file["man"], + keyword=parsed_file.get("name"))) + + return results diff --git a/coala_quickstart/info_extractors/__init__.py b/coala_quickstart/info_extractors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/info_extractors/PackageJSONInfoExtractorTest.py b/tests/info_extractors/PackageJSONInfoExtractorTest.py new file mode 100644 index 0000000..e0956a5 --- /dev/null +++ b/tests/info_extractors/PackageJSONInfoExtractorTest.py @@ -0,0 +1,84 @@ +import os +import unittest + +from coala_quickstart.info_extractors.PackageJSONInfoExtractor import ( + PackageJSONInfoExtractor) +from coala_quickstart.info_extraction.Information import ( + LicenseUsedInfo, ProjectDependencyInfo, IncludePathsInfo, ManFilesInfo, + VersionInfo) +from tests.TestUtilities import generate_files + + +test_file = """ { + "name": "awesome-packages", + "version": "0.8.0", + "license": "MIT", + "dependencies": { + "coffeelint": "~1", + "ramllint": ">=1.2.2 <1.2.4" + }, + "files": ["dist"], + "man" : ["./man/foo.1", "./man/bar.1"] +} +""" + +invalid_test_file = """ + Some content that is not JSON! +""" + +class PackageJSONInfoExtractorTest(unittest.TestCase): + + def setUp(self): + self.current_dir = os.getcwd() + + def test_extracted_information(self): + + with generate_files( + ["package.json"], + [test_file], + self.current_dir) as gen_file: + + self.uut = PackageJSONInfoExtractor( + ["package.json"], + self.current_dir) + + extracted_information = self.uut.extract_information() + extracted_information = extracted_information["package.json"] + + information_types = extracted_information.keys() + + self.assertIn("LicenseUsedInfo", information_types) + license_info = extracted_information["LicenseUsedInfo"] + self.assertEqual(len(license_info), 1) + self.assertEqual(license_info[0].value, "MIT") + + self.assertIn("ProjectDependencyInfo", information_types) + dep_info = extracted_information["ProjectDependencyInfo"] + self.assertEqual(len(dep_info), 2) + self.assertIn(dep_info[0].value, ["coffeelint", "ramllint"]) + self.assertIsInstance(dep_info[0].version, VersionInfo) + self.assertIn( + dep_info[0].version.value, ["~1", ">=1.2.2 <1.2.4"]) + + self.assertIn("ManFilesInfo", information_types) + man_paths_info = extracted_information["ManFilesInfo"] + self.assertEqual(len(man_paths_info), 1) + self.assertEqual(man_paths_info[0].value, ["./man/foo.1", "./man/bar.1"]) + + self.assertIn("IncludePathsInfo", information_types) + include_paths_info = extracted_information["IncludePathsInfo"] + self.assertEqual(len(include_paths_info), 1) + self.assertEqual(include_paths_info[0].value, ["dist"]) + + def test_invalid_files(self): + + with generate_files( + ["package.json"], + [invalid_test_file], + self.current_dir) as gen_file: + + self.uut = PackageJSONInfoExtractor( + ["package.json"], + self.current_dir) + extracted_information = self.uut.extract_information() + self.assertEqual(extracted_information, {}) From d23329a4737706721ff728c7027fb493487b4a36 Mon Sep 17 00:00:00 2001 From: satwikkansal Date: Sun, 18 Jun 2017 00:05:53 +0530 Subject: [PATCH 5/5] info_extractors: Extract info from Gemfile Adds extractor class ``GemfileInfoExtractor`` to extract relevant information from Gemfile. Related to https://github.com/coala/coala-quickstart/issues/119 --- .../info_extractors/GemfileInfoExtractor.py | 30 ++++++++ requirements.txt | 1 + .../GemfileInfoExtractorTest.py | 72 +++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 coala_quickstart/info_extractors/GemfileInfoExtractor.py create mode 100644 tests/info_extractors/GemfileInfoExtractorTest.py diff --git a/coala_quickstart/info_extractors/GemfileInfoExtractor.py b/coala_quickstart/info_extractors/GemfileInfoExtractor.py new file mode 100644 index 0000000..68700b6 --- /dev/null +++ b/coala_quickstart/info_extractors/GemfileInfoExtractor.py @@ -0,0 +1,30 @@ +from gemfileparser import GemfileParser + +from coala_quickstart.info_extraction.InfoExtractor import InfoExtractor +from coala_quickstart.info_extraction.Information import ( + ProjectDependencyInfo, VersionInfo) + + +class GemfileInfoExtractor(InfoExtractor): + supported_files = ("Gemfile",) + + spec_references = ["https://gitlab.com/coala/GSoC-2017/issues/167", ] + supported_information_kinds = ( + ProjectDependencyInfo, VersionInfo) + + def parse_file(self, fname, file_content): + parser = GemfileParser(fname) + return parser.parse() + + def find_information(self, fname, parsed_file): + results = [] + for grp, deps in parsed_file.items(): + for dep in deps: + results.append( + ProjectDependencyInfo( + fname, + dep.name, + version=VersionInfo(fname, dep.requirement), + url=dep.source)) + + return results diff --git a/requirements.txt b/requirements.txt index 2202379..106ef3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ coala_bears~=0.11 coala_utils~=0.4 +gemfileparser~=0.6.2 \ No newline at end of file diff --git a/tests/info_extractors/GemfileInfoExtractorTest.py b/tests/info_extractors/GemfileInfoExtractorTest.py new file mode 100644 index 0000000..9a983ce --- /dev/null +++ b/tests/info_extractors/GemfileInfoExtractorTest.py @@ -0,0 +1,72 @@ +import os +import unittest + +from coala_quickstart.info_extractors.GemfileInfoExtractor import ( + GemfileInfoExtractor) +from coala_quickstart.info_extraction.Information import ( + ProjectDependencyInfo, VersionInfo) +from tests.TestUtilities import generate_files + + +test_file = """ +source "https://rubygems.org" + +gem "puppet-lint", "2.1.1" +gem "rubocop", "0.47.1" +gem "scss_lint", require: false +gem "RedCloth", :require => "redcloth" +gem "omniauth", ">= 0.2.6", :git => "git://github.com/intridea/omniauth.git" + +group :assets do + gem 'some-gem', source: "https://gems.example.com" +end +gem "rspec-rails", ">= 2.6.1", :group => [:development, :test] + +# Some comment +# gem "not_to_consider" + +group :development, :test, :cucumber do + gem "rspec-rails", "~> 2.0.0" + gem "ruby-debug19", :platforms => :mri_19 +end +""" + + +class GemfileInfoExtractorTest(unittest.TestCase): + + def setUp(self): + self.current_dir = os.getcwd() + + def test_extracted_information(self): + + with generate_files( + ["Gemfile"], + [test_file], + self.current_dir) as gen_file: + + self.uut = GemfileInfoExtractor( + ["Gemfile"], + self.current_dir) + + extracted_info = self.uut.extract_information() + extracted_info = extracted_info[ + os.path.normcase("Gemfile")] + + information_types = extracted_info.keys() + + self.assertIn("ProjectDependencyInfo", information_types) + dep_info = extracted_info["ProjectDependencyInfo"] + self.assertEqual(len(dep_info), 9) + + gems = [('some-gem', ''), ('puppet-lint', '2.1.1'), + ('rubocop', '0.47.1'), ('scss_lint', ''), ('RedCloth', ''), + ('rspec-rails', '>= 2.6.1'), ('rspec-rails', '~> 2.0.0'), + ('ruby-debug19', ''), ('omniauth', '>= 0.2.6')] + + deps = [(d.value, d.version.value) for d in dep_info] + self.assertNotIn(("not_to_consider", ""), deps) + for gem in gems: + self.assertIn(gem, deps) + + source_urls = [d.url for d in dep_info] + self.assertIn("https://gems.example.com", source_urls)