From 82239a83b414f9497a63ad43416faee7a302c5d1 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Fri, 8 Dec 2017 12:31:20 -0500 Subject: [PATCH 01/13] Added a PyPI badge to the README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a6c3df6..2a18570b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## Pushing HTTPS :lock: +[![Latest Version](https://img.shields.io/pypi/v/pshtt.svg)](https://pypi.python.org/pypi/pshtt/) [![Coverage Status](https://coveralls.io/repos/github/dhs-ncats/pshtt/badge.svg)](https://coveralls.io/github/dhs-ncats/pshtt) - [![Build Status](https://travis-ci.org/dhs-ncats/pshtt.svg?branch=master)](https://travis-ci.org/dhs-ncats/pshtt) `pshtt` (_"pushed"_) is a tool to scan domains for HTTPS best practices. It saves its results to a CSV (or JSON) file. From 8d32da5cbeb5d2a3e5fa2e46801979bcc00672ef Mon Sep 17 00:00:00 2001 From: Eric Mill Date: Sun, 25 Feb 2018 13:25:24 -0500 Subject: [PATCH 02/13] redirects to www only matter if they're within the same hostname --- pshtt/pshtt.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pshtt/pshtt.py b/pshtt/pshtt.py index 66fa3492..57eea033 100644 --- a/pshtt/pshtt.py +++ b/pshtt/pshtt.py @@ -335,7 +335,6 @@ def basic_check(endpoint): base_immediate = parent_domain_for(subdomain_immediate) endpoint.redirect_immediately_to = immediate - endpoint.redirect_immediately_to_www = (re.match(r'^https?://www\.', immediate) is not None) endpoint.redirect_immediately_to_https = immediate.startswith("https://") endpoint.redirect_immediately_to_http = immediate.startswith("http://") endpoint.redirect_immediately_to_external = (base_original != base_immediate) @@ -344,6 +343,14 @@ def basic_check(endpoint): (subdomain_original != subdomain_immediate) ) + # We're interested in whether an endpoint redirects to the www version + # of itself (not whether it redirects to www prepended to any other + # hostname, even within the same parent domain). + endpoint.redirect_immediately_to_www = ( + subdomain_immediate.startswith("www.") and + (re.sub("www\.", "", subdomain_immediate) == subdomain_original) + ) + if ultimate_req is not None: # For ultimate destination, use the URL we arrived at, # not Location header. Auto-resolves relative redirects. @@ -592,12 +599,6 @@ def root_down(endpoint): ) ) - def goes_to_www(endpoint): - return ( - endpoint.redirect_immediately_to_www and - (not endpoint.redirect_immediately_to_external) - ) - all_roots_unused = root_unused(https) and root_unused(http) all_roots_down = root_down(https) and root_down(http) @@ -606,8 +607,8 @@ def goes_to_www(endpoint): at_least_one_www_used and all_roots_unused and ( all_roots_down or - goes_to_www(https) or - goes_to_www(http) + https.redirect_immediately_to_www or + http.redirect_immediately_to_www ) ) From 73417270e124f89187d4a94617c401429c0adc75 Mon Sep 17 00:00:00 2001 From: Eric Mill Date: Mon, 26 Feb 2018 10:56:12 -0500 Subject: [PATCH 03/13] minor cleanup of e9e212c after @PaulSD's suggestion --- pshtt/pshtt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pshtt/pshtt.py b/pshtt/pshtt.py index 57eea033..b1988a71 100644 --- a/pshtt/pshtt.py +++ b/pshtt/pshtt.py @@ -347,8 +347,7 @@ def basic_check(endpoint): # of itself (not whether it redirects to www prepended to any other # hostname, even within the same parent domain). endpoint.redirect_immediately_to_www = ( - subdomain_immediate.startswith("www.") and - (re.sub("www\.", "", subdomain_immediate) == subdomain_original) + subdomain_immediate == ("www.%s" % subdomain_original) ) if ultimate_req is not None: From ec7e74bcd11963aa9f81230111c4f3e5af8502d3 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Thu, 8 Mar 2018 15:20:49 -0500 Subject: [PATCH 04/13] Describe in more detail that pshtt only counts domains as HSTS preloaded only if they are fully HSTS preloaded, meaning that all subdomains are included as well. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2a18570b..7035a333 100644 --- a/README.md +++ b/README.md @@ -124,9 +124,9 @@ The following values are returned in `results.csv`: * `HSTS Max Age` - A domain's HSTS max-age is its canonical endpoint's max-age. * `HSTS Entire Domain` - A domain has HSTS enabled for the entire domain if its **root HTTPS endpoint** (_not the canonical HTTPS endpoint_) has HSTS enabled and uses the HSTS `includeSubDomains` flag. * `HSTS Preload Ready` - A domain is HSTS "preload ready" if its **root HTTPS endpoint** (_not the canonical HTTPS endpoint_) has HSTS enabled, has a max-age of at least 18 weeks, and uses the `includeSubDomains` and `preload` flag. -* `HSTS Preload Pending` - A domain is "preload pending" when it appears in the [Chrome preload pending list](https://hstspreload.org/api/v2/pending). -* `HSTS Preloaded` - A domain is HSTS preloaded if its domain name appears in the [Chrome preload list](https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json), regardless of what header is present on any endpoint. -* `Base Domain HSTS Preloaded` - A domain's base domain is HSTS preloaded. This is subtly different from `HSTS Entire Domain`, which inpects headers on the base domain to see if HSTS is set correctly to encompass the entire zone. This checks the preload list directly. +* `HSTS Preload Pending` - A domain is "preload pending" when it appears in the [Chrome preload pending list](https://hstspreload.org/api/v2/pending) with the `include_subdomains` flag equal to `true`. The intent of `pshtt` is to make sure that the user is _fully_ protected, so it only counts domains as HSTS preloaded if they are _fully_ HSTS preloaded (meaning that all subdomains are included as well). +* `HSTS Preloaded` - A domain is HSTS preloaded if its domain name appears in the [Chrome preload list](https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json) with the `include_subdomains` flag equal to `true`, regardless of what header is present on any endpoint. The intent of `pshtt` is to make sure that the user is _fully_ protected, so it only counts domains as HSTS preloaded if they are _fully_ HSTS preloaded (meaning that all subdomains are included as well). +* `Base Domain HSTS Preloaded` - A domain's base domain is HSTS preloaded if its base domain appears in the [Chrome preload list](https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json) with the `include_subdomains` flag equal to `true`. This is subtly different from `HSTS Entire Domain`, which inpects headers on the base domain to see if HSTS is set correctly to encompass the entire zone. #### Scoring From d6c4de281b041bbfa18d04ebcbec7d4e4bd25417 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Sun, 25 Mar 2018 09:00:27 -0400 Subject: [PATCH 05/13] Reverting to the latest 1.3 version of sslyze. The just-released 1.4 version seems to segfault, and it has a breaking API change. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d7dbe4d9..fe900756 100755 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ install_requires=[ 'requests>=2.18.4', - 'sslyze>=1.1.0', + 'sslyze>=1.3.4,<1.4.0', 'wget>=3.2', 'docopt', 'pytablereader', From af053ea74d6480b6679e666ef85bcd4598f695bd Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Tue, 3 Apr 2018 06:59:31 -0400 Subject: [PATCH 06/13] Upgrading to 1.4+ version of sslyze --- pshtt/pshtt.py | 13 +++---------- setup.py | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/pshtt/pshtt.py b/pshtt/pshtt.py index b1988a71..f6d67ef0 100644 --- a/pshtt/pshtt.py +++ b/pshtt/pshtt.py @@ -448,17 +448,10 @@ def https_check(endpoint): # remove the https:// from prefix for sslyze try: hostname = endpoint.url[8:] - server_info = sslyze.server_connectivity.ServerConnectivityInfo(hostname=hostname, port=443) - except Exception as err: - endpoint.unknown_error = True - logging.warn("Unknown exception when checking server connectivity info with sslyze.") - utils.debug("{0}".format(err)) - return - - try: - server_info.test_connectivity_to_server() + server_tester = sslyze.server_connectivity.ServerConnectivityTester(hostname=hostname, port=443) + server_info = server_tester.perform() except sslyze.server_connectivity.ServerConnectivityError as err: - logging.warn("Error in sslyze server connectivity check") + logging.warn("Error in sslyze server connectivity check when connecting to {}".format(err.server_info.hostname)) utils.debug("{0}".format(err)) return except Exception as err: diff --git a/setup.py b/setup.py index fe900756..105f6c4a 100755 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ install_requires=[ 'requests>=2.18.4', - 'sslyze>=1.3.4,<1.4.0', + 'sslyze>=1.4.1', 'wget>=3.2', 'docopt', 'pytablereader', From 33d5a452d2d34fd7156c050fd19d192183d0fa21 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Tue, 3 Apr 2018 07:08:59 -0400 Subject: [PATCH 07/13] Removing support for python 2.7, since the PyPI package for sslyze doesn't support it. See https://github.com/nabla-c0d3/nassl/issues/34#issuecomment-375896876. --- .travis.yml | 1 - setup.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8ca7a88e..29d38bc6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ language: python sudo: false python: - - '2.7' - '3.4' - '3.5' - '3.6' diff --git a/setup.py b/setup.py index 105f6c4a..148c7d3d 100755 --- a/setup.py +++ b/setup.py @@ -44,8 +44,6 @@ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', From 00e3c32ddb24ea870414ec6a457bf06bbbfb91a7 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Tue, 3 Apr 2018 08:03:24 -0400 Subject: [PATCH 08/13] Adding missing import --- pshtt/pshtt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pshtt/pshtt.py b/pshtt/pshtt.py index f6d67ef0..1e1d2538 100644 --- a/pshtt/pshtt.py +++ b/pshtt/pshtt.py @@ -26,6 +26,7 @@ from urllib2 import URLError import sslyze +import sslyze.server_connectivity import sslyze.synchronous_scanner # We're going to be making requests with certificate validation disabled. From a5ba3d50c72b832e6762835bc29c0e9b9a8cfd40 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Tue, 3 Apr 2018 08:09:33 -0400 Subject: [PATCH 09/13] Removing python 2.7 from tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 235cafd3..92a55cb1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36,flake8 +envlist = py34,py35,py36,flake8 skip_missing_interpreters = true ; usedevelop = true From b89e0fec07ffb8a9686e928a323fd3a096294eaf Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Tue, 3 Apr 2018 08:13:13 -0400 Subject: [PATCH 10/13] Doing some import-fu for the new sslyze stuff --- pshtt/pshtt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pshtt/pshtt.py b/pshtt/pshtt.py index 1e1d2538..55d083e7 100644 --- a/pshtt/pshtt.py +++ b/pshtt/pshtt.py @@ -26,7 +26,7 @@ from urllib2 import URLError import sslyze -import sslyze.server_connectivity +from sslyze.server_connectivity_tester import ServerConnectivityTester, ServerConnectivityError import sslyze.synchronous_scanner # We're going to be making requests with certificate validation disabled. @@ -449,9 +449,9 @@ def https_check(endpoint): # remove the https:// from prefix for sslyze try: hostname = endpoint.url[8:] - server_tester = sslyze.server_connectivity.ServerConnectivityTester(hostname=hostname, port=443) + server_tester = ServerConnectivityTester(hostname=hostname, port=443) server_info = server_tester.perform() - except sslyze.server_connectivity.ServerConnectivityError as err: + except ServerConnectivityError as err: logging.warn("Error in sslyze server connectivity check when connecting to {}".format(err.server_info.hostname)) utils.debug("{0}".format(err)) return From bc09c6fe0f7b28e6fae62feccb55a0c54ccaa0ca Mon Sep 17 00:00:00 2001 From: Eric Mill Date: Tue, 3 Apr 2018 15:17:14 -0400 Subject: [PATCH 11/13] Note the Python 3 requirement in the README. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7035a333..c6d76832 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ ## Getting Started +`pshtt` requires **Python 3.4+**. Python 2 is not supported. + `pshtt` can be installed as a module, or run directly from the repository. #### Installed as a module From 84434d55fe574ac0d238fe66cfe24005ea309413 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Wed, 4 Apr 2018 14:04:23 -0400 Subject: [PATCH 12/13] * Adding semver to dev requirements in setup.py * Creating bump_version.sh, a bash script for bumping version numbers --- bump_version.sh | 33 +++++++++++++++++++++++++++++++++ setup.py | 5 +++-- 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100755 bump_version.sh diff --git a/bump_version.sh b/bump_version.sh new file mode 100755 index 00000000..c133dda3 --- /dev/null +++ b/bump_version.sh @@ -0,0 +1,33 @@ +#/usr/bin/env bash + +# bump_version.sh (show|major|minor|patch|prerelease|build) + +VERSION_FILE=pshtt/__init__.py + +HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" + +old_version=$(sed "s/__version__ = '\(.*\)'/\1/" $VERSION_FILE) + +if [[ $# -ne 1 ]] +then + echo $HELP_INFORMATION +else + case $1 in + major|minor|patch|prerelease|build) + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") + echo Changing version from $old_version to $new_version + sed -i "s/$old_version/$new_version/" $VERSION_FILE + ;; + finalize) + new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") + echo Changing version from $old_version to $new_version + sed -i "s/$old_version/$new_version/" $VERSION_FILE + ;; + show) + echo $old_version + ;; + *) + echo $HELP_INFORMATION + ;; + esac +fi diff --git a/setup.py b/setup.py index 148c7d3d..9d4ebf2e 100755 --- a/setup.py +++ b/setup.py @@ -68,9 +68,10 @@ extras_require={ # 'dev': ['check-manifest'], - 'test': [ + 'dev': [ + 'pytest', + 'semver', 'tox', - 'pytest' ], }, From 730683a5d0a7ed537ec7368f9d84535a046c1abe Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Wed, 4 Apr 2018 14:05:56 -0400 Subject: [PATCH 13/13] Bumping the patch number in anticipation of pushing to PyPI --- pshtt/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pshtt/__init__.py b/pshtt/__init__.py index f0ca8cfe..f0ede3d3 100644 --- a/pshtt/__init__.py +++ b/pshtt/__init__.py @@ -1 +1 @@ -__version__ = '0.4.0-dev' +__version__ = '0.4.1'