Merge pull request #177 from adamdecaf/ftp-cafile

files: support setting additional root certificates for FTP connections

README.md

@@ -75,6 +75,7 @@ The following environmental variables can be set to configure behavior in paygat
 #### ACH file uploading / transfers
 
 - `ACH_FILE_BATCH_SIZE`: Number of Transfers to retrieve from the database in each batch for mergin before upload to Fed.
+- `ACH_FILE_TRANSFERS_CAFILE`: Filepath for additional (CA) certificates to be added into each FTP client used within paygate
 - `ACH_FILE_TRANSFER_INTERVAL`: Go duration for how often to check and sync ACH files on their FTP destinations.
    - Note: Set the value `off` to completely disable async file downloads and uploads.
 - `ACH_FILE_STORAGE_DIR`: Filepath for temporary storage of ACH files. This is used as a scratch directory to manage outbound and incoming/returned ACH files.

file_transfer.go

@@ -6,9 +6,12 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -70,11 +73,23 @@ func (agent *ftpFileTransferAgent) close() error {
 }
 
 // newFileTransferAgent returns an FTP implementation of a fileTransferAgent
+//
+// This function reads ACH_FILE_TRANSFERS_ROOT_CAFILE for a file with root certificates to be used
+// in all secured connections.
 func newFileTransferAgent(ftpConf *ftpConfig, conf *fileTransferConfig) (fileTransferAgent, error) {
-	timeout := ftp.DialWithTimeout(30 * time.Second)
-	epsv := ftp.DialWithDisabledEPSV(false)
-
-	conn, err := ftp.Dial(ftpConf.Hostname, timeout, epsv)
+	opts := []ftp.DialOption{
+		ftp.DialWithTimeout(30 * time.Second),
+		ftp.DialWithDisabledEPSV(false),
+	}
+	tlsOpt, err := tlsDialOption(os.Getenv("ACH_FILE_TRANSFERS_CAFILE"))
+	if err != nil {
+		return nil, err
+	}
+	if tlsOpt != nil {
+		opts = append(opts, *tlsOpt)
+	}
+	// Make the first connection
+	conn, err := ftp.Dial(ftpConf.Hostname, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -88,6 +103,29 @@ func newFileTransferAgent(ftpConf *ftpConfig, conf *fileTransferConfig) (fileTra
 	}, nil
 }
 
+func tlsDialOption(caFilePath string) (*ftp.DialOption, error) {
+	if caFilePath == "" {
+		return nil, nil
+	}
+	bs, err := ioutil.ReadFile(caFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("tlsDialOption: failed to read %s: %v", caFilePath, err)
+	}
+	pool, err := x509.SystemCertPool()
+	if pool == nil || err != nil {
+		pool = x509.NewCertPool()
+	}
+	ok := pool.AppendCertsFromPEM(bs)
+	if !ok {
+		return nil, fmt.Errorf("tlsDialOption: problem with AppendCertsFromPEM from %s", caFilePath)
+	}
+	cfg := &tls.Config{
+		RootCAs: pool,
+	}
+	opt := ftp.DialWithTLS(cfg)
+	return &opt, nil
+}
+
 func (agent *ftpFileTransferAgent) delete(path string) error {
 	if path == "" || strings.HasSuffix(path, "/") {
 		return fmt.Errorf("ftpFileTransferAgent: invalid path %v", path)

file_transfer_test.go

@@ -159,6 +159,26 @@ func TestFTP(t *testing.T) {
 	}
 }
 
+func TestFTP__tlsDialOption(t *testing.T) {
+	if testing.Short() {
+		return // skip network calls
+	}
+
+	cafile, err := grabConnectionCertificates(t, "google.com:443")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(cafile)
+
+	opt, err := tlsDialOption(cafile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if opt == nil {
+		t.Fatal("nil tls DialOption")
+	}
+}
+
 func createTestFileTransferAgent(t *testing.T) (*server.Server, fileTransferAgent) {
 	svc, err := createTestFTPServer(t)
 	if err != nil {

http_test.go

@@ -111,21 +111,38 @@ func TestHTTP__tlsHttpClient(t *testing.T) {
 	}
 
 	if testing.Short() {
-		return // skip network call on -short
+		return // skip network calls
 	}
 
+	cafile, err := grabConnectionCertificates(t, "google.com:443")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(cafile)
+
+	client, err = tlsHttpClient(cafile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if client == nil {
+		t.Error("empty http.Client")
+	}
+}
+
+// grabConnectionCertificates returns a filepath of certificate chain from a given address's
+// server. This is useful for adding extra root CA's to network clients
+func grabConnectionCertificates(t *testing.T, addr string) (string, error) {
 	dialer := &net.Dialer{Timeout: 10 * time.Second}
-	conn, err := tls.DialWithDialer(dialer, "tcp", "google.com:443", nil)
+	conn, err := tls.DialWithDialer(dialer, "tcp", addr, nil)
 	if err != nil {
 		t.Error(err)
 	}
 	defer conn.Close()
 
-	fd, err := ioutil.TempFile("", "tlsHttpClient")
+	fd, err := ioutil.TempFile("", "conn-certs")
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.Remove(fd.Name())
 
 	// Write x509 certs to disk
 	certs := conn.ConnectionState().PeerCertificates
@@ -142,12 +159,5 @@ func TestHTTP__tlsHttpClient(t *testing.T) {
 	if err := ioutil.WriteFile(fd.Name(), buf.Bytes(), 0644); err != nil {
 		t.Fatal(err)
 	}
-
-	client, err = tlsHttpClient(fd.Name())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if client == nil {
-		t.Error("empty http.Client")
-	}
+	return fd.Name(), nil
 }