Implemented a dynamic generation of SSL request certificates

README.md

@@ -253,6 +253,7 @@ Billy.configure do |c|
   c.non_successful_error_level = :warn
   c.non_whitelisted_requests_disabled = false
   c.cache_path = 'spec/req_cache/'
+  c.certs_path = 'spec/req_certs/'
   c.proxy_host = 'example.com' # defaults to localhost
   c.proxy_port = 12345 # defaults to random
   c.proxied_request_host = nil
@@ -291,7 +292,7 @@ using `c.dynamic_jsonp`. This is helpful when JSONP APIs use cache-busting
 parameters. For example, if you want `http://example.com/foo?callback=bar&id=1&cache_bust=12345` and `http://example.com/foo?callback=baz&id=1&cache_bust=98765` to be cache hits for each other, you would set `c.dynamic_jsonp_keys = ['callback', 'cache_bust']` to ignore both params. Note
 that in this example the `id` param would still be considered important.
 
-`c.dynamic_jsonp_callback_name` is used to configure the name of the JSONP callback 
+`c.dynamic_jsonp_callback_name` is used to configure the name of the JSONP callback
 parameter. The default is `callback`.
 
 `c.path_blacklist = []` is used to always cache specific paths on any hostnames,
@@ -323,6 +324,13 @@ allowed, all others will throw an error with the URL attempted to be accessed.
 This is useful for debugging issues in isolated environments (ie.
 continuous integration).
 
+`c.cache_path` can be used to locate the cache directory to a different place
+other than `system temp directory/puffing-billy`.
+
+`c.certs_path` can be used to locate the directory for dynamically generated
+SSL certificates to a different place other than `system temp
+directory/puffing-billy/certs`.
+
 `c.proxy_host` and `c.proxy_port` are used for the Billy proxy itself which runs locally.
 
 `c.proxied_request_host` and `c.proxied_request_port` are used if an internal proxy
@@ -500,6 +508,54 @@ end
 
 Note that this approach may cause unexpected behavior if your backend sends the Referer HTTP header (which is unlikely).
 
+## SSL usage
+
+Unfortunately we cannot setup the runtime certificate authority on your browser
+at time of configuring the Capybara driver.  So you need to take care of this
+step yourself as a prepartion. A good point would be directly after configuring
+this gem.
+
+### Google Chrome Headless example
+
+Google Chrome/Chromium is capable to run as a test browser with the new
+headless mode which is not able to handle the deprecated
+`--ignore-certificate-errors` flag. But the headless mode is capable of
+handling the user PKI certificate store.  So you just need to import the
+runtime Puffing Billy certificate authority on your system store, or generate a
+new store for your current session. The following examples demonstrates the
+former variant:
+
+```ruby
+# Overwrite the local home directory for chrome. We use this
+# to setup a custom SSL certificate store.
+ENV['HOME'] = "#{Dir.tmpdir}/chrome-home-#{Time.now.to_i}"
+
+# Clear and recreate the Chrome home directory.
+FileUtils.rm_rf(ENV['HOME'])
+FileUtils.mkdir_p(ENV['HOME'])
+
+# Setup a new pki certificate database for Chrome
+system <<~SCRIPT
+cd "#{ENV['HOME']}"
+curl -s -k -o "cacert-root.crt" "http://www.cacert.org/certs/root.crt"
+curl -s -k -o "cacert-class3.crt" "http://www.cacert.org/certs/class3.crt"
+echo > .password
+mkdir -p .pki/nssdb
+CERT_DIR=sql:$HOME/.pki/nssdb
+certutil -N -d .pki/nssdb -f .password
+certutil -d ${CERT_DIR}  -A -t TC \
+  -n "CAcert.org" -i cacert-root.crt
+certutil -d ${CERT_DIR} -A -t TC \
+  -n "CAcert.org Class 3" -i cacert-class3.crt
+certutil -d sql:$HOME/.pki/nssdb -A \
+  -n puffing-billy -t "CT,C,C" -i #{Billy.certificate_authority.cert_file}
+SCRIPT
+```
+
+Mind the reset of the `HOME` environment variable. Furtunately Chrome takes
+care of the users home, so we can setup a new temporary directory for the test
+run, without messing with potential user configurations.
+
 ## Resources
 
 * [Bring Ruby VCR to Javascript testing with Capybara and puffing-billy](http://architects.dzone.com/articles/bring-ruby-vcr-javascript)

lib/billy.rb

@@ -7,6 +7,9 @@
 require 'billy/handlers/cache_handler'
 require 'billy/proxy_request_stub'
 require 'billy/cache'
+require 'billy/ssl/authority'
+require 'billy/ssl/certificate'
+require 'billy/ssl/certificate_chain'
 require 'billy/proxy'
 require 'billy/proxy_connection'
 require 'billy/railtie' if defined?(Rails)
@@ -19,4 +22,8 @@ def self.proxy
       proxy
     )
   end
+
+  def self.certificate_authority
+    @certificate_authority ||= Billy::Authority.new
+  end
 end

lib/billy/config.rb

@@ -8,7 +8,7 @@ class Config
 
     attr_accessor :logger, :cache, :cache_request_headers, :whitelist, :path_blacklist, :ignore_params,
                   :persist_cache, :ignore_cache_port, :non_successful_cache_disabled, :non_successful_error_level,
-                  :non_whitelisted_requests_disabled, :cache_path, :proxy_host, :proxy_port, :proxied_request_inactivity_timeout,
+                  :non_whitelisted_requests_disabled, :cache_path, :certs_path, :proxy_host, :proxy_port, :proxied_request_inactivity_timeout,
                   :proxied_request_connect_timeout, :dynamic_jsonp, :dynamic_jsonp_keys, :dynamic_jsonp_callback_name, :merge_cached_responses_whitelist,
                   :strip_query_params, :proxied_request_host, :proxied_request_port, :cache_request_body_methods, :after_cache_handles_request,
                   :cache_simulates_network_delays, :cache_simulates_network_delay_time, :record_stub_requests
@@ -34,6 +34,7 @@ def reset
       @non_successful_error_level = :warn
       @non_whitelisted_requests_disabled = false
       @cache_path = File.join(Dir.tmpdir, 'puffing-billy')
+      @certs_path = File.join(Dir.tmpdir, 'puffing-billy', 'certs')
       @proxy_host = 'localhost'
       @proxy_port = RANDOM_AVAILABLE_PORT
       @proxied_request_inactivity_timeout = 10 # defaults from https://github.com/igrigorik/em-http-request/wiki/Redirects-and-Timeouts

lib/billy/proxy_connection.rb

@@ -52,11 +52,9 @@ def on_message_complete
     def restart_with_ssl(url)
       @ssl = url
       @parser = Http::Parser.new(self)
+      generate_certificate_chain(url)
       send_data("HTTP/1.0 200 Connection established\r\nProxy-agent: Puffing-Billy/0.0.0\r\n\r\n")
-      start_tls(
-        private_key_file: File.expand_path('../mitm.key', __FILE__),
-        cert_chain_file: File.expand_path('../mitm.crt', __FILE__)
-      )
+      start_tls(private_key_file: @key_file, cert_chain_file: @chain_file)
     end
 
     def handle_request
@@ -93,6 +91,15 @@ def send_response(response)
       res.content = response[:content]
       res.send_response
     end
-
+
+    def generate_certificate_chain(url)
+      domain = url.split(':').first
+      ca = Billy.certificate_authority.cert
+      cert = Billy::Certificate.new(domain)
+      chain = Billy::CertificateChain.new(domain, cert.cert, ca)
+
+      @chain_file = chain.file
+      @key_file = cert.key_file
+    end
   end
 end

lib/billy/ssl/authority.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'openssl'
+require 'fileutils'
+
+module Billy
+  # This class is dedicated to the generation of a brand new certificate
+  # authority which can be picked up by a browser to verify and secure any
+  # communication with puffing billy. This authority certificate will be
+  # generated once on runtime and will sign each request certificate. So
+  # we do not have to deal with outdated certificates or stuff like that.
+  #
+  # The resulting certificate authority is at its bare minimum to keep
+  # things simple and snappy. We do not handle a certificate revoke list
+  # (CRL) nor any other special key handling, even if we enable these
+  # extensions. It's just a mimic of the mighty mitmproxy certificate
+  # authority file.
+  class Authority
+    attr_reader :key, :cert
+
+    # The authority generation does not require any arguments from outside
+    # of this class definition. We just generate the certificate and thats
+    # it.
+    #
+    # Example:
+    #
+    #   ca = Billy::Authority.new
+    #   [ca.cert_file, ca.key_file]
+    def initialize
+      @key = OpenSSL::PKey::RSA.new(2048)
+      @cert = generate
+    end
+
+    # Write out the private key to file (PEM format) and give back the
+    # file path. This will produce a temporary file which will be remove
+    # after the current process terminates.
+    def key_file
+      path = File.join(Billy.config.certs_path, 'ca.key')
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, key.to_pem)
+      path
+    end
+
+    # Write out the certifcate to file (PEM format) and give back the
+    # file path. This will produce a temporary file which will be remove
+    # after the current process terminates.
+    def cert_file
+      path = File.join(Billy.config.certs_path, 'ca.crt')
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, cert.to_pem)
+      path
+    end
+
+    private
+
+    # Defines a static list of available extensions on the certificate.
+    def extensions
+      [
+        # ln_sn, value, critical
+        ['basicConstraints', 'CA:TRUE', true],
+        ['keyUsage', 'keyCertSign, cRLSign', true],
+        ['subjectKeyIdentifier', 'hash', false],
+        ['authorityKeyIdentifier', 'keyid:always', false]
+      ]
+    end
+
+    # Give back the static subject name of the certificate.
+    def name
+      ['CN=Puffing Billy', 'O=Puffing Billy'].join('/')
+                                             .prepend('/')
+                                             .concat('/')
+    end
+
+    # Give back an appropriate date for the beginning of this
+    # certificate life. We give back now 2 days ago.
+    def valid_from
+      Time.now - (2 * 24 * 60 * 60)
+    end
+
+    # Give back an appropriate date for the end of this certificate life.
+    # We give back now in 2 days.
+    def valid_to
+      Time.now + (2 * 24 * 60 * 60)
+    end
+
+    # Generate a random serial number for the certificate.
+    def serial
+      Time.now.to_i + rand(100_000_000_000)
+    end
+
+    # Generate a fresh new certificate for the configured domain.
+    def generate
+      cert = OpenSSL::X509::Certificate.new
+      configure(cert)
+      add_extensions(cert)
+      cert.sign(key, OpenSSL::Digest::SHA256.new)
+    end
+
+    # Setup all relevant properties of the given certificate to produce
+    # a valid and useable certificate.
+    def configure(cert)
+      cert.version = 2
+      cert.serial = serial
+      cert.subject = OpenSSL::X509::Name.parse(name)
+      cert.issuer = cert.subject
+      cert.public_key = key.public_key
+      cert.not_before = valid_from
+      cert.not_after = valid_to
+    end
+
+    # Add all extensions (defined by the +extensions+ method) to the given
+    # certificate.
+    def add_extensions(cert)
+      factory = OpenSSL::X509::ExtensionFactory.new
+      factory.subject_certificate = cert
+      factory.issuer_certificate = cert
+      extensions.each do |ln_sn, value, critical|
+        cert.add_extension(factory.create_extension(ln_sn, value, critical))
+      end
+    end
+  end
+end