gsoc: Gemfile info extraction

coala_quickstart/info_extraction/Info.py

@@ -2,11 +2,14 @@
 
 
 class Info:
-    description = 'Some information'
+    description = 'Some Description.'
 
     # type signature for the information value.
     value_type = (object,)
 
+    # Some example values for reference that also match the value_type.
+    example_values = []
+
     def __init__(self,
                  source,
                  value,

coala_quickstart/info_extraction/InfoExtractor.py

@@ -8,6 +8,9 @@ class InfoExtractor:
     # tuple of file globs supported by the extractor.
     supported_file_globs = tuple()
 
+    # Links to the issues/documentations for relevant specs of supported files.
+    spec_references = []
+
     # tuple of ``Info`` classes that can be extracted.
     supported_info_kinds = (Info,)
 

coala_quickstart/info_extraction/Information.py

@@ -0,0 +1,58 @@
+from coala_quickstart.info_extraction.Info import Info
+
+
+class LicenseUsedInfo(Info):
+    description = "License of the project."
+    value_type = (str,)
+    example_values = ["MIT", "GPL-3", "Apache-2.0"]
+
+
+class VersionInfo(Info):
+    description = "Version information, see http://semver.org/"
+    value_type = (str,)
+    example_values = [">=1.2.7", "~1.2.3", "^0.2"]
+
+
+class ProjectDependencyInfo(Info):
+    description = "Dependency of the project."
+    value_type = (str,)
+    example_values = ['some_npm_package_name', 'some_gem_name', 'other_dep']
+
+    def __init__(self,
+                 source,
+                 value,
+                 extractor=None,
+                 version=None,
+                 url=''):
+        super().__init__(source, value, extractor, version=version, url=url)
+
+
+class PathsInfo(Info):
+    description = "File path globs mentioned in the file."
+    value_type = ([str],)
+    example_values = ["**.py", "dev/tests/**"]
+
+
+class IncludePathsInfo(PathsInfo):
+    description = "Target files to perform analysis."
+
+
+class IgnorePathsInfo(PathsInfo):
+    description = "Files to ignore during analysis."
+
+
+class ManFilesInfo(Info):
+    description = "Filenames to put in place for the man program to find."
+    value_type = (str, [str])
+    example_values = ["./man/doc.1", ["./man/foo.1", "./man/bar.1"]]
+
+    def __init__(self,
+                 source,
+                 value,
+                 extractor=None,
+                 keyword=""):
+        """
+        :param keyword: Primary keyword for ``man`` command that would display
+                        the man pages provided in value argument.
+        """
+        super().__init__(source, value, extractor, keyword=keyword)

coala_quickstart/info_extractors/GemfileInfoExtractor.py

@@ -0,0 +1,30 @@
+from gemfileparser import GemfileParser
+
+from coala_quickstart.info_extraction.InfoExtractor import InfoExtractor
+from coala_quickstart.info_extraction.Information import (
+    ProjectDependencyInfo, VersionInfo)
+
+
+class GemfileInfoExtractor(InfoExtractor):
+    supported_files = ("Gemfile",)
+
+    spec_references = ["https://gitlab.com/coala/GSoC-2017/issues/167", ]
+    supported_information_kinds = (
+        ProjectDependencyInfo, VersionInfo)
+
+    def parse_file(self, fname, file_content):
+        parser = GemfileParser(fname)
+        return parser.parse()
+
+    def find_information(self, fname, parsed_file):
+        results = []
+        for grp, deps in parsed_file.items():
+            for dep in deps:
+                results.append(
+                    ProjectDependencyInfo(
+                        fname,
+                        dep.name,
+                        version=VersionInfo(fname, dep.requirement),
+                        url=dep.source))
+
+        return results

coala_quickstart/info_extractors/PackageJSONInfoExtractor.py

@@ -0,0 +1,61 @@
+import json
+import logging
+
+from coala_quickstart.info_extraction.InfoExtractor import InfoExtractor
+from coala_quickstart.info_extraction.Information import (
+    LicenseUsedInfo, ProjectDependencyInfo, IncludePathsInfo, ManFilesInfo,
+    VersionInfo)
+
+
+class PackageJSONInfoExtractor(InfoExtractor):
+    supported_file_globs = ("package.json",)
+
+    spec_references = [
+        "https://docs.npmjs.com/files/package.json",
+        "https://gitlab.com/coala/GSoC-2017/issues/167"]
+
+    supported_info_kinds = (
+        LicenseUsedInfo,
+        ProjectDependencyInfo,
+        IncludePathsInfo,
+        ManFilesInfo)
+
+    def parse_file(self, fname, file_content):
+        parsed_file = {}
+
+        try:
+            parsed_file = json.loads(file_content)
+        except Exception:
+            logging.warning("Error while parsing the file {}".format(fname))
+
+        return parsed_file
+
+    def find_information(self, fname, parsed_file):
+        results = []
+
+        if parsed_file.get("license"):
+            results.append(
+                LicenseUsedInfo(fname, parsed_file["license"]))
+
+        if parsed_file.get("dependencies"):
+            for package_name, version_range in (
+                    parsed_file["dependencies"].items()):
+                results.append(
+                    ProjectDependencyInfo(
+                        fname,
+                        package_name,
+                        self.__class__.__name__,
+                        VersionInfo(fname, version_range)))
+
+        if parsed_file.get("files"):
+            results.append(
+                IncludePathsInfo(fname, parsed_file["files"]))
+
+        if parsed_file.get("man"):
+            results.append(
+                ManFilesInfo(
+                    fname,
+                    parsed_file["man"],
+                    keyword=parsed_file.get("name")))
+
+        return results

requirements.txt

@@ -1,2 +1,3 @@
 coala_bears~=0.11
 coala_utils~=0.4
+gemfileparser~=0.6.2

tests/info_extraction/InfoExtractorTest.py

@@ -18,6 +18,7 @@ class AnotherDummyInfo(Info):
             description = 'Another such information.'
 
         class DummyInfoExtractor(InfoExtractor):
+            spec_references = ['some/dummy/link', 'another/dummy/link']
 
             def parse_file(self, fname, file_content):
                 return file_content
@@ -134,6 +135,7 @@ def test_multiple_target_globs(self):
                     extracted_info[tf]['DummyInfo'][0].extractor,
                     InfoExtractor)
 
+
     def test_multiple_information(self):
 
         target_filenames = ['target_file_1', ]
@@ -266,3 +268,13 @@ def test_supported_info_kinds(self):
                      "information kinds of WrongSupportedInfoExtractor")):
 
                 uut.extract_information()
+
+    def test_spec_references_filed(self):
+        uut = self.DummyInfoExtractor
+        self.assertEqual(len(uut.spec_references), 2)
+        self.assertEqual(
+            uut.spec_references,
+            ['some/dummy/link', 'another/dummy/link'])
+
+        uut = self.DummyMultiInfoExtractor
+        self.assertEqual(uut.spec_references, [])

tests/info_extraction/InfoTest.py

@@ -27,9 +27,11 @@ def find_information(self, fname, parsed_file):
         class InfoA(Info):
             description = 'Information A'
             value_type = (str, int)
+            example_values = ['coala', 420]
 
         class InfoB(Info):
             description = "Info class without value_type"
+            example_values = [["literally", "anything"]]
 
         self.info_a = InfoA(
             'source_file',
@@ -43,7 +45,8 @@ def test_main(self):
         self.assertEqual(self.base_info.name, 'Info')
         self.assertEqual(self.base_info.value, 'base_info_value')
         self.assertEqual(self.base_info.source, 'source_file')
-        self.assertEqual(self.base_info.description, 'Some information')
+        self.assertEqual(self.base_info.description, 'Some Description.')
+        self.assertEqual(len(self.base_info.example_values), 0)
         self.assertIsInstance(self.base_info.extractor, InfoExtractor)
 
     def test_derived_instances(self):
@@ -52,6 +55,7 @@ def test_derived_instances(self):
         self.assertEqual(self.info_a.source, 'source_file')
         self.assertEqual(self.info_a.extra_param, 'extra_param_value')
         self.assertEqual(self.info_a.description, 'Information A')
+        self.assertEqual(self.info_a.example_values, ['coala', 420])
 
     def test_value_type(self):
         with self.assertRaisesRegexp(

tests/info_extractors/GemfileInfoExtractorTest.py

@@ -0,0 +1,72 @@
+import os
+import unittest
+
+from coala_quickstart.info_extractors.GemfileInfoExtractor import (
+    GemfileInfoExtractor)
+from coala_quickstart.info_extraction.Information import (
+    ProjectDependencyInfo, VersionInfo)
+from tests.TestUtilities import generate_files
+
+
+test_file = """
+source "https://rubygems.org"
+
+gem "puppet-lint", "2.1.1"
+gem "rubocop", "0.47.1"
+gem "scss_lint", require: false
+gem "RedCloth", :require => "redcloth"
+gem "omniauth", ">= 0.2.6", :git => "git://github.com/intridea/omniauth.git"
+
+group :assets do
+  gem 'some-gem', source: "https://gems.example.com"
+end
+gem "rspec-rails", ">= 2.6.1", :group => [:development, :test]
+
+# Some comment
+# gem "not_to_consider"
+
+group :development, :test, :cucumber do
+  gem "rspec-rails", "~> 2.0.0"
+  gem "ruby-debug19", :platforms => :mri_19
+end
+"""
+
+
+class GemfileInfoExtractorTest(unittest.TestCase):
+
+    def setUp(self):
+        self.current_dir = os.getcwd()
+
+    def test_extracted_information(self):
+
+        with generate_files(
+              ["Gemfile"],
+              [test_file],
+              self.current_dir) as gen_file:
+
+            self.uut = GemfileInfoExtractor(
+                ["Gemfile"],
+                self.current_dir)
+
+            extracted_info = self.uut.extract_information()
+            extracted_info = extracted_info[
+                os.path.normcase("Gemfile")]
+
+            information_types = extracted_info.keys()
+
+            self.assertIn("ProjectDependencyInfo", information_types)
+            dep_info = extracted_info["ProjectDependencyInfo"]
+            self.assertEqual(len(dep_info), 9)
+
+            gems = [('some-gem', ''), ('puppet-lint', '2.1.1'),
+                    ('rubocop', '0.47.1'), ('scss_lint', ''), ('RedCloth', ''),
+                    ('rspec-rails', '>= 2.6.1'), ('rspec-rails', '~> 2.0.0'),
+                    ('ruby-debug19', ''), ('omniauth', '>= 0.2.6')]
+
+            deps = [(d.value, d.version.value) for d in dep_info]
+            self.assertNotIn(("not_to_consider", ""), deps)
+            for gem in gems:
+                self.assertIn(gem, deps)
+
+            source_urls = [d.url for d in dep_info]
+            self.assertIn("https://gems.example.com", source_urls)