Improved Maven POM detection

* rename class to MavenPomPackage for clarity * fix the metafiles and the recognition for #631 * add additional api and recognize tests * clean up the API doc and otehr minor refactorings Signed-off-by: Philippe Ombredanne <[email protected]>
aboutcode-org · Jun 23, 2017 · a79dd9d · a79dd9d
1 parent ce2eff8
commit a79dd9d
Show file tree

Hide file tree

Showing 511 changed files with 755 additions and 566 deletions.
diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py
@@ -37,7 +37,7 @@
     models.JavaJar,
     models.JavaEar,
     models.JavaWar,
-    maven.MavenJar,
+    maven.MavenPomPackage,
     models.IvyJar,
     models.JBossSar,
     models.Axis2Mar,

diff --git a/src/packagedcode/maven.py b/src/packagedcode/maven.py
@@ -56,11 +56,13 @@
 Attempts to resolve Maven properties when possible.
 """
 
-
-class MavenJar(models.JavaJar):
-    metafiles = ('META-INF/**/*.pom', 'pom.xml',)
+class MavenPomPackage(models.Package):
+    metafiles = ('.pom', 'pom.xml',)
+    extensions = ('.pom','.xml', )
     repo_types = (models.repo_maven,)
-    type = models.StringType(default='Apache Maven')
+    type = models.StringType(default='Apache Maven POM')
+    packaging = models.StringType(default=models.as_archive)
+    primary_language = models.StringType(default='Java')
 
     @classmethod
     def recognize(cls, location):
@@ -650,7 +652,7 @@ def parse(location):
             ))
 
     # FIXME: there are still a lot of other data to map in a Package
-    package = MavenJar(
+    package = MavenPomPackage(
         location=location,
         name='{group_id}:{artifact_id}'.format(**pom),
         version=pom['version'],

diff --git a/src/packagedcode/recognize.py b/src/packagedcode/recognize.py
@@ -41,7 +41,7 @@
 """
 
 
-def recognize_packaged_archives(location):
+def recognize_package(location):
     """
     Return a Package object if one was recognized or None for this `location`.
     """

diff --git a/src/scancode/api.py b/src/scancode/api.py
@@ -36,24 +36,19 @@
 
 def extract_archives(location, recurse=True):
     """
-    Extract recursively any archives found at location and yield an iterable of
-    ExtractEvents.
-    If verbose is False, only the "done" event is returned at extraction
-    completion.
-    If verbose is True, both "start" and "done" events are returned.
+    Extract any archives found at `location` and yield ExtractEvents. If
+    `recurse` is True, extracts nested archives-in- archives
+    recursively.
     """
     from extractcode.extract import extract
     from extractcode import default_kinds
-
     for xevent in extract(location, kinds=default_kinds, recurse=recurse):
         yield xevent
 
 
 def get_copyrights(location):
     """
-    Yield an iterable of dictionaries of copyright data detected in the file at
-    location. Each item contains a list of copyright statements and a start and
-    end line.
+    Yield mappings of copyright data detected in the file at `location`.
     """
     from cluecode.copyrights import detect_copyrights
 
@@ -72,8 +67,7 @@ def get_copyrights(location):
 
 def get_emails(location):
     """
-    Yield an iterable of dictionaries of emails detected in the file at
-    location.
+    Yield mappings of emails detected in the file at `location`.
     """
     from cluecode.finder import find_emails
     for email, line_num  in find_emails(location):
@@ -88,8 +82,7 @@ def get_emails(location):
 
 def get_urls(location):
     """
-    Yield an iterable of dictionaries of urls detected in the file at
-    location.
+    Yield mappings of urls detected in the file at `location`.
     """
     from cluecode.finder import find_urls
     for urls, line_num  in find_urls(location):
@@ -108,14 +101,18 @@ def get_urls(location):
 
 def get_licenses(location, min_score=0, include_text=False, diag=False):
     """
-    Yield dictionaries of license data detected in the file at location.
+    Yield mappings of license data detected in the file at `location`.
+
+    `minimum_score` is a minimum score threshold from 0 to 100. The
+    default is 0 means that all license matches will be returned. With
+    any other value matches that have a score below minimum score with
+    not be returned.
 
-    `minimum_score` is a minimum score threshold from 0 to 100. The default is 0
-    means that all license matches will be returned. With any other value matches
-    that have a score below minimum score with not be returned.
+    if `include_text` is True, the matched text is included in the
+    returned data.
 
-    If `diag` is True, additional match details are returned with the matched_rule
-    key of the returned mapping.
+    If `diag` is True, additional match details are returned with the
+    matched_rule key of the returned mapping.
     """
     from licensedcode.cache import get_index
     from licensedcode.cache import get_licenses_db
@@ -164,15 +161,14 @@ def get_licenses(location, min_score=0, include_text=False, diag=False):
             yield result
 
 
-def get_file_infos(location, as_list=True):
+def get_file_infos(location):
     """
-    Return a list of dictionaries of informations collected from the file or
-    directory at location.
+    Return a mapping of file information collected from the file or
+    directory at `location`.
     """
     from commoncode import fileutils
     from commoncode import filetype
     from commoncode.hash import multi_checksums
-    from scancode import utils
     from typecode import contenttype
 
     infos = OrderedDict()
@@ -198,16 +194,13 @@ def get_file_infos(location, as_list=True):
     infos['is_source'] = bool(is_file and T.is_source)
     infos['is_script'] = bool(is_file and T.is_script)
 
-    if as_list:
-        return [infos]
-    else:
-        return infos
+    return infos
 
 
 # FIXME: this smells bad
 def _empty_file_infos():
     """
-    Return an empty mapping of info key/values, used in case of failure.
+    Return an empty mapping of file info, used in case of failure.
     """
     infos = OrderedDict()
     infos['type'] = None
@@ -232,11 +225,11 @@ def _empty_file_infos():
 
 def get_package_infos(location):
     """
-    Return a list of dictionaries of package information
-    collected from the location or an empty list.
+    Return a list of mappings of package information collected from the
+    `location` or an empty list.
     """
-    from packagedcode.recognize import recognize_packaged_archives
-    package = recognize_packaged_archives(location)
+    from packagedcode.recognize import recognize_package
+    package = recognize_package(location)
     if not package:
         return []
     return [package.to_dict()]
diff --git a/src/scancode/cli.py b/src/scancode/cli.py
@@ -689,7 +689,7 @@ def scan_infos(input_file, diag=False):
     """
     errors = []
     try:
-        infos = get_file_infos(input_file, as_list=False)
+        infos = get_file_infos(input_file)
     except Exception as e:
         # never fail but instead add an error message.
         infos = _empty_file_infos()

diff --git a/tests/packagedcode/data/m2/ant/ant-launcher/1.6.5/ant-launcher-1.6.5.pom.package.json b/tests/packagedcode/data/m2/ant/ant-launcher/1.6.5/ant-launcher-1.6.5.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "ant:ant-launcher", 
   "version": "1.6.5", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/ant/ant/1.6.5/ant-1.6.5.pom.package.json b/tests/packagedcode/data/m2/ant/ant/1.6.5/ant-1.6.5.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "ant:ant", 
   "version": "1.6.5", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/antlr/antlr/2.7.6/antlr-2.7.6.pom.package.json b/tests/packagedcode/data/m2/antlr/antlr/2.7.6/antlr-2.7.6.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "antlr:antlr", 
   "version": "2.7.6", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/aopalliance/aopalliance/1.0/aopalliance-1.0.pom.package.json b/tests/packagedcode/data/m2/aopalliance/aopalliance/1.0/aopalliance-1.0.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "aopalliance:aopalliance", 
   "version": "1.0", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/asm/asm-attrs/1.5.3/asm-attrs-1.5.3.pom.package.json b/tests/packagedcode/data/m2/asm/asm-attrs/1.5.3/asm-attrs-1.5.3.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "asm:asm-attrs", 
   "version": "1.5.3", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/asm/asm-parent/3.0/asm-parent-3.0.pom.package.json b/tests/packagedcode/data/m2/asm/asm-parent/3.0/asm-parent-3.0.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "asm:asm-parent", 
   "version": "3.0", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/asm/asm/1.5.3/asm-1.5.3.pom.package.json b/tests/packagedcode/data/m2/asm/asm/1.5.3/asm-1.5.3.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "asm:asm", 
   "version": "1.5.3", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/asm/asm/3.0/asm-3.0.pom.package.json b/tests/packagedcode/data/m2/asm/asm/3.0/asm-3.0.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "asm:asm", 
   "version": "3.0", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/aspectj/aspectjrt/1.5.3/aspectjrt-1.5.3.pom.package.json b/tests/packagedcode/data/m2/aspectj/aspectjrt/1.5.3/aspectjrt-1.5.3.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "aspectj:aspectjrt", 
   "version": "1.5.3", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/aspectj/aspectjrt/1.5.4/aspectjrt-1.5.4.pom.package.json b/tests/packagedcode/data/m2/aspectj/aspectjrt/1.5.4/aspectjrt-1.5.4.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "aspectj:aspectjrt", 
   "version": "1.5.4", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/aspectj/aspectjtools/1.5.4/aspectjtools-1.5.4.pom.package.json b/tests/packagedcode/data/m2/aspectj/aspectjtools/1.5.4/aspectjtools-1.5.4.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "aspectj:aspectjtools", 
   "version": "1.5.4", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/aspectj/aspectjweaver/1.5.4/aspectjweaver-1.5.4.pom.package.json b/tests/packagedcode/data/m2/aspectj/aspectjweaver/1.5.4/aspectjweaver-1.5.4.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "aspectj:aspectjweaver", 
   "version": "1.5.4", 
   "primary_language": "Java", 

diff --git a/...ackagedcode/data/m2/avalon/avalon-framework/4.1.3/avalon-framework-4.1.3.pom.package.json b/...ackagedcode/data/m2/avalon/avalon-framework/4.1.3/avalon-framework-4.1.3.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "avalon-framework:avalon-framework", 
   "version": "4.1.3", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/bcel/bcel/5.1/bcel-5.1.pom.package.json b/tests/packagedcode/data/m2/bcel/bcel/5.1/bcel-5.1.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "bcel:bcel", 
   "version": "5.1", 
   "primary_language": "Java", 

diff --git a/...ckagedcode/data/m2/beanutils/commons-beanutils/1.6/commons-beanutils-1.6.pom.package.json b/...ckagedcode/data/m2/beanutils/commons-beanutils/1.6/commons-beanutils-1.6.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "commons-beanutils:commons-beanutils", 
   "version": "1.6", 
   "primary_language": "Java", 

diff --git a/...edcode/data/m2/beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.pom.package.json b/...edcode/data/m2/beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "commons-beanutils:commons-beanutils", 
   "version": "1.7.0", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/biz/aQute/bndlib/0.0.203/bndlib-0.0.203.pom.package.json b/tests/packagedcode/data/m2/biz/aQute/bndlib/0.0.203/bndlib-0.0.203.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "biz.aQute:bndlib", 
   "version": "0.0.203", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/biz/aQute/bndlib/0.0.238/bndlib-0.0.238.pom.package.json b/tests/packagedcode/data/m2/biz/aQute/bndlib/0.0.238/bndlib-0.0.238.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "biz.aQute:bndlib", 
   "version": "0.0.238", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/c3p0/c3p0/0.9.0.4/c3p0-0.9.0.4.pom.package.json b/tests/packagedcode/data/m2/c3p0/c3p0/0.9.0.4/c3p0-0.9.0.4.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "c3p0:c3p0", 
   "version": "0.9.0.4", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/cglib/cglib-nodep/2.1_3/cglib-nodep-2.1_3.pom.package.json b/tests/packagedcode/data/m2/cglib/cglib-nodep/2.1_3/cglib-nodep-2.1_3.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "cglib:cglib-nodep", 
   "version": "2.1_3", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/cglib/cglib/2.1_3/cglib-2.1_3.pom.package.json b/tests/packagedcode/data/m2/cglib/cglib/2.1_3/cglib-2.1_3.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "cglib:cglib", 
   "version": "2.1_3", 
   "primary_language": "Java", 

diff --git a/...code/data/m2/classworlds/classworlds/1.1-alpha-2/classworlds-1.1-alpha-2.pom.package.json b/...code/data/m2/classworlds/classworlds/1.1-alpha-2/classworlds-1.1-alpha-2.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "classworlds:classworlds", 
   "version": "1.1-alpha-2", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/classworlds/classworlds/1.1/classworlds-1.1.pom.package.json b/tests/packagedcode/data/m2/classworlds/classworlds/1.1/classworlds-1.1.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "classworlds:classworlds", 
   "version": "1.1", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/cli/commons-cli/1.0/commons-cli-1.0.pom.package.json b/tests/packagedcode/data/m2/cli/commons-cli/1.0/commons-cli-1.0.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "commons-cli:commons-cli", 
   "version": "1.0", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/codec/commons-codec/1.3/commons-codec-1.3.pom.package.json b/tests/packagedcode/data/m2/codec/commons-codec/1.3/commons-codec-1.3.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "commons-codec:commons-codec", 
   "version": "1.3", 
   "primary_language": "Java", 

diff --git a/...code/data/m2/collections/commons-collections/2.0/commons-collections-2.0.pom.package.json b/...code/data/m2/collections/commons-collections/2.0/commons-collections-2.0.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "commons-collections:commons-collections", 
   "version": "2.0", 
   "primary_language": "Java", 

diff --git a/...code/data/m2/collections/commons-collections/2.1/commons-collections-2.1.pom.package.json b/...code/data/m2/collections/commons-collections/2.1/commons-collections-2.1.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "commons-collections:commons-collections", 
   "version": "2.1", 
   "primary_language": "Java", 

diff --git a/...code/data/m2/collections/commons-collections/3.1/commons-collections-3.1.pom.package.json b/...code/data/m2/collections/commons-collections/3.1/commons-collections-3.1.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "commons-collections:commons-collections", 
   "version": "3.1", 
   "primary_language": "Java", 

diff --git a/...code/data/m2/collections/commons-collections/3.2/commons-collections-3.2.pom.package.json b/...code/data/m2/collections/commons-collections/3.2/commons-collections-3.2.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "commons-collections:commons-collections", 
   "version": "3.2", 
   "primary_language": "Java", 

diff --git a/....ss.c.g.common.collect/0.5.0.alpha/com.ss.c.g.common.collect-0.5.0.alpha.pom.package.json b/....ss.c.g.common.collect/0.5.0.alpha/com.ss.c.g.common.collect-0.5.0.alpha.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "com.google.collect:com.ss.c.g.common.collect", 
   "version": "0.5.0.alpha", 
   "primary_language": "Java", 

diff --git a/tests/packagedcode/data/m2/com/jcraft/jsch/0.1.23/jsch-0.1.23.pom.package.json b/tests/packagedcode/data/m2/com/jcraft/jsch/0.1.23/jsch-0.1.23.pom.package.json
@@ -1,5 +1,5 @@
 {
-  "type": "Apache Maven", 
+  "type": "Apache Maven POM", 
   "name": "com.jcraft:jsch", 
   "version": "0.1.23", 
   "primary_language": "Java",
-Original file line number
+Diff line change
@@ Expand Up / @@ -41,7 +41,7 @@ @@
     """
-    def recognize_packaged_archives(location):
+    def recognize_package(location):
         """
         Return a Package object if one was recognized or None for this `location`.
         """
@@ Expand Down @@