Merge pull request #326 from pyupio/nicholas/adding-documentation-lic…

…ense-command

License command documentation and cache

HISTORY.rst

@@ -9,6 +9,7 @@ History
 * Added README information about Python 2.7 workaround
 * Adjusted some pricing information
 * Fixed MacOS binary build through AppVeyor
+* Added the ability to check packages licenses
 
 1.9.0 (2020-04-27)
 ------------------

README.md

@@ -430,6 +430,111 @@ safety review --file report.json --bare
 ```
 django
 ```
+
+___
+
+# License
+
+Display packages licenses information (requires an api-key)
+
+## Options
+
+### `--key` (REQUIRED)
+
+*API Key for pyup.io's licenses database. Can be set as `SAFETY_API_KEY` environment variable.*
+
+**Example**
+```bash
+safety license --key=12345-ABCDEFGH
+```
+*Shows the license of each package in the current environment*
+
+
+```
++==============================================================================+
+|                                                                              |
+|                               /$$$$$$            /$$                         |
+|                              /$$__  $$          | $$                         |
+|           /$$$$$$$  /$$$$$$ | $$  \__//$$$$$$  /$$$$$$   /$$   /$$           |
+|          /$$_____/ |____  $$| $$$$   /$$__  $$|_  $$_/  | $$  | $$           |
+|         |  $$$$$$   /$$$$$$$| $$_/  | $$$$$$$$  | $$    | $$  | $$           |
+|          \____  $$ /$$__  $$| $$    | $$_____/  | $$ /$$| $$  | $$           |
+|          /$$$$$$$/|  $$$$$$$| $$    |  $$$$$$$  |  $$$$/|  $$$$$$$           |
+|         |_______/  \_______/|__/     \_______/   \___/   \____  $$           |
+|                                                          /$$  | $$           |
+|                                                         |  $$$$$$/           |
+|  by pyup.io                                              \______/            |
+|                                                                              |
++==============================================================================+
+| Packages licenses                                                            |
++=============================================+===========+====================+
+| package                                     |  version  | license            |
++=============================================+===========+====================+
+| requests                                    | 2.25.0    | Apache-2.0         |
+|------------------------------------------------------------------------------|
+| click                                       | 7.1.2     | BSD-3-Clause       |
+|------------------------------------------------------------------------------|
+| safety                                      | 1.10.0.de | MIT                |
++==============================================================================+
+```
+
+___
+
+### `--db`
+
+*Path to a directory with a local licenses database `licenses.json`*
+
+**Example**
+```bash
+safety license --key=12345-ABCDEFGH --db=/home/safety-db/data
+```
+___
+
+### `--no-cache`
+
+*Since PyUp.io licenses DB is updated once a week, the licenses database is cached locally for 7 days. You can use `--no-cache` to download it once again.*
+
+**Example**
+```bash
+safety license --key=12345-ABCDEFGH --no-cache
+```
+___
+
+### `--file`, `-r`
+
+*Read input from one (or multiple) requirement files.*
+
+**Example**
+```bash
+safety license --key=12345-ABCDEFGH -r requirements.txt
+```
+```bash
+safety license --key=12345-ABCDEFGH --file=requirements.txt
+```
+```bash
+safety license --key=12345-ABCDEFGH -r req_dev.txt -r req_prod.txt
+```
+
+___
+
+
+### `--proxy-host`, `-ph`
+
+*Proxy host IP or DNS*
+
+### `--proxy-port`, `-pp` 
+
+*Proxy port number*
+
+### `--proxy-protocol`, `-pr`
+
+*Proxy protocol (https or http)*
+
+**Example**
+```bash
+safety license --key=12345-ABCDEFGH -ph 127.0.0.1 -pp 8080 -pr https
+```
+
 ___
 
 # Python 2.7

safety/cli.py

@@ -7,7 +7,7 @@
 from safety.formatter import report, license_report
 import itertools
 from safety.util import read_requirements, read_vulnerabilities, get_proxy_dict, get_packages_licenses
-from safety.errors import DatabaseFetchError, DatabaseFileNotFoundError, InvalidKeyError
+from safety.errors import DatabaseFetchError, DatabaseFileNotFoundError, InvalidKeyError, TooManyRequestsError
 
 try:
     from json.decoder import JSONDecodeError
@@ -123,7 +123,7 @@ def review(full_report, bare, file):
 
 
 @cli.command()
-@click.option("--key", default="", envvar="SAFETY_API_KEY",
+@click.option("--key", required=True, envvar="SAFETY_API_KEY",
               help="API Key for pyup.io's vulnerability database. Can be set as SAFETY_API_KEY "
                    "environment variable. Default: empty")
 @click.option("--db", default="",
@@ -151,7 +151,26 @@ def license(key, db, cache, files, proxyprotocol, proxyhost, proxyport):
         ]  
 
     proxy_dictionary = get_proxy_dict(proxyprotocol, proxyhost, proxyport)
-    licenses_db = safety.get_licenses(key, db, cache, proxy_dictionary)
+    try:
+        licenses_db = safety.get_licenses(key, db, cache, proxy_dictionary)
+    except InvalidKeyError:
+        click.secho("Your API Key '{key}' is invalid. See {link}".format(
+            key=key, link='https://goo.gl/O7Y1rS'),
+            fg="red",
+            file=sys.stderr)
+        sys.exit(-1)
+    except DatabaseFileNotFoundError:
+        click.secho("Unable to load licenses database from {db}".format(db=db), fg="red", file=sys.stderr)
+        sys.exit(-1)
+    except TooManyRequestsError:
+        click.secho("Unable to load licenses database (Too many requests, please wait before another request)",
+            fg="red",
+            file=sys.stderr
+        )
+        sys.exit(-1)
+    except DatabaseFetchError:
+        click.secho("Unable to load licenses database", fg="red", file=sys.stderr)
+        sys.exit(-1)
     filtered_packages_licenses = get_packages_licenses(packages, licenses_db)
     output_report = license_report(packages=packages, licenses=filtered_packages_licenses)
     click.secho(output_report, nl=True)

safety/constants.py

@@ -13,6 +13,8 @@
 
 CACHE_VALID_SECONDS = 60 * 60 * 2  # 2 hours
 
+CACHE_LICENSES_VALID_SECONDS = 60 * 60 * 24 * 7  # one week
+
 CACHE_FILE = os.path.join(
     os.path.expanduser("~"),
     ".safety",

safety/errors.py

@@ -8,3 +8,7 @@ class DatabaseFileNotFoundError(DatabaseFetchError):
 
 class InvalidKeyError(DatabaseFetchError):
     pass
+
+
+class TooManyRequestsError(DatabaseFetchError):
+    pass

safety/safety.py

@@ -8,10 +8,10 @@
 import requests
 from packaging.specifiers import SpecifierSet
 
-from .constants import (API_MIRRORS, CACHE_FILE, CACHE_VALID_SECONDS,
-                        OPEN_MIRRORS, REQUEST_TIMEOUT)
+from .constants import (API_MIRRORS, CACHE_FILE, CACHE_LICENSES_VALID_SECONDS,
+                        CACHE_VALID_SECONDS, OPEN_MIRRORS, REQUEST_TIMEOUT)
 from .errors import (DatabaseFetchError, DatabaseFileNotFoundError,
-                     InvalidKeyError)
+                     InvalidKeyError, TooManyRequestsError)
 from .util import RequirementFile
 
 
@@ -27,7 +27,13 @@ def get_from_cache(db_name):
                 data = json.loads(f.read())
                 if db_name in data:
                     if "cached_at" in data[db_name]:
-                        if data[db_name]["cached_at"] + CACHE_VALID_SECONDS > time.time():
+                        if 'licenses.json' in db_name:
+                            # Getting the specific cache time for the licenses db.
+                            cache_valid_seconds = CACHE_LICENSES_VALID_SECONDS
+                        else:
+                            cache_valid_seconds = CACHE_VALID_SECONDS
+
+                        if data[db_name]["cached_at"] + cache_valid_seconds > time.time():
                             return data[db_name]["db"]
             except json.JSONDecodeError:
                 pass
@@ -89,6 +95,8 @@ def fetch_database_url(mirror, db_name, key, cached, proxy):
         return data
     elif r.status_code == 403:
         raise InvalidKeyError()
+    elif r.status_code == 429:
+        raise TooManyRequestsError()
 
 
 def fetch_database_file(path, db_name):
@@ -181,8 +189,8 @@ def review(vulnerabilities):
 def get_licenses(key, db_mirror, cached, proxy):
     key = key if key else os.environ.get("SAFETY_API_KEY", False)
 
-    if not key:
-        raise DatabaseFetchError("API-KEY not provided.")
+    if not key and not db_mirror:
+        raise InvalidKeyError("API-KEY not provided.")
     if db_mirror:
         mirrors = [db_mirror]
     else:

tests/test_safety.py

@@ -12,6 +12,7 @@
 import unittest
 import textwrap
 from click.testing import CliRunner
+from unittest.mock import Mock, patch
 
 from safety import safety
 from safety import cli
@@ -256,6 +257,136 @@ def test_get_packages_licenses(self):
                     "unexpected package '" + pkg_license['package'] + "' was found"
                 )
 
+    def test_get_packages_licenses_without_api_key(self):
+        from safety.errors import InvalidKeyError
+
+        # without providing an API-KEY 
+        with self.assertRaises(InvalidKeyError) as error:
+            safety.get_licenses(
+                db_mirror=False,
+                cached=False,
+                proxy={},
+                key=None
+            )
+        db_generic_exception = error.exception
+        self.assertEqual(str(db_generic_exception), 'API-KEY not provided.')
+
+    @patch("safety.safety.requests")
+    def test_get_packages_licenses_with_invalid_api_key(self, requests):
+        from safety.errors import InvalidKeyError
+
+        mock = Mock()
+        mock.status_code = 403
+        requests.get.return_value = mock
+
+        # proving an invalid API-KEY
+        with self.assertRaises(InvalidKeyError):
+            safety.get_licenses(
+                db_mirror=False,
+                cached=False,
+                proxy={},
+                key="INVALID"
+            )
+
+    @patch("safety.safety.requests")
+    def test_get_packages_licenses_db_fetch_error(self, requests):
+        from safety.errors import DatabaseFetchError
+
+        mock = Mock()
+        mock.status_code = 500
+        requests.get.return_value = mock
+
+        with self.assertRaises(DatabaseFetchError):
+            safety.get_licenses(
+                db_mirror=False,
+                cached=False,
+                proxy={},
+                key="MY-VALID-KEY"
+            )
+
+    def test_get_packages_licenses_with_invalid_db_file(self):
+        from safety.errors import DatabaseFileNotFoundError
+
+        with self.assertRaises(DatabaseFileNotFoundError):
+            safety.get_licenses(
+                db_mirror='/my/invalid/path',
+                cached=False,
+                proxy={},
+                key=None
+            )
+
+    @patch("safety.safety.requests")
+    def test_get_packages_licenses_very_often(self, requests):
+        from safety.errors import TooManyRequestsError
+
+        # if the request is made too often, an 429 error is raise by PyUp.io
+        mock = Mock()
+        mock.status_code = 429
+        requests.get.return_value = mock
+
+        with self.assertRaises(TooManyRequestsError):
+            safety.get_licenses(
+                db_mirror=False,
+                cached=False,
+                proxy={},
+                key="MY-VALID-KEY"
+            )
+
+    @patch("safety.safety.requests")
+    def test_get_cached_packages_licenses(self, requests):
+        import copy
+        from safety.constants import CACHE_FILE
+
+        licenses_db = {
+            "licenses": {
+                "BSD-3-Clause": 2
+            },
+            "packages": {
+                "django": [
+                    {
+                        "start_version": "0.0",
+                        "license_id": 2
+                    }
+                ]
+            }
+        }
+        original_db = copy.deepcopy(licenses_db)
+
+        mock = Mock()
+        mock.json.return_value = licenses_db
+        mock.status_code = 200
+        requests.get.return_value = mock
+
+        # lets clear the cache first
+        try:
+            with open(CACHE_FILE, 'w') as f:
+                f.write(json.dumps({}))
+        except Exception:
+            pass
+
+        # In order to cache the db (and get), we must set cached as True
+        response = safety.get_licenses(
+            db_mirror=False,
+            cached=True,
+            proxy={},
+            key="MY-VALID-KEY"
+        )
+        self.assertEqual(response, licenses_db)
+
+        # now we should have the db in cache
+        # changing the "live" db to test if we are getting the cached db
+        licenses_db['licenses']['BSD-3-Clause'] = 123
+
+        resp = safety.get_licenses(
+            db_mirror=False,
+            cached=True,
+            proxy={},
+            key="MY-VALID-KEY"
+        )
+
+        self.assertNotEqual(resp, licenses_db)
+        self.assertEqual(resp, original_db)
+
 
 class ReadRequirementsTestCase(unittest.TestCase):