CSV address data download link

This provides a CSV address history download link.

New URL path is /download/address/io/{address}.

The CSV contains rows of the addresses table, with column headers.

The rows returned are:

 - tx_hash
 - direction: [1,-1] Indicates whether this is input or output of the 
associated transaction. +1 for output (increasing address balance), -1 
for input. This can also be thought of as funding or spending.
 - io_index: The input or output index in the tx_hash, depending on the 
direction.
 - valid_mainchain: [0,1]
 - value: dcrutil.Amount(value).ToCoin()
 - time_stamp: uses TimeDef.String() YY-MM-DD HH:MM:SS format, but 
should maybe be either a timestamp or an ISO formatted string
 - tx_type: from txhelpers.TxTypeToString(txType)
 - matching_tx_hash: Can be empty if the tx is funding and the output is 
not spent.
 

The header sequence is "tx_hash", "direction", "io_index", 
"valid_mainchain", "value", "time_stamp", "tx_type", "matching_tx_hash".

Server's filename recommendation is address-io-[address]-[current best 
block height at time of download]-[UNIX epoch timestamp].csv. For 
example: 
address-io-DseXBL6g6GxvfYAnKqdao2f7WkXDmYTYW87-302678-1545405532.csv

A new glyph for the download icon is added to the fonts (SVG, TTF, EOT, 
and WOFF files).

README.md

@@ -543,6 +543,7 @@ The dcrdata block explorer is exposed by two APIs: a Decred implementation of
 the [Insight API](https://github.com/bitpay/insight-api) (EXPERIMENTAL), and its
 own JSON HTTP API. The Insight API uses the path prefix `/insight/api`. The
 dcrdata API uses the path prefix `/api`.
+File downloads are served from the `/download` path.
 
 ### Insight API (EXPERIMENTAL)
 
@@ -629,6 +630,7 @@ the `/api` path prefix.
 | Verbose transaction result for last <br> `N` transactions               | `/address/A/count/N/raw`        | `types.AddressTxRaw`  |
 | Summary of last `N` transactions, skipping `M`                          | `/address/A/count/N/skip/M`     | `types.Address`       |
 | Verbose transaction result for last <br> `N` transactions, skipping `M` | `/address/A/count/N/skip/M/raw` | `types.AddressTxRaw`  |
+| Transaction inputs and outputs as a CSV formatted file.                 | `/download/address/io/A`        | CSV file              |
 
 | Stake Difficulty (Ticket Price)        | Path                    | Type                               |
 | -------------------------------------- | ----------------------- | ---------------------------------- |

api/apirouter.go

@@ -17,21 +17,15 @@ type apiMux struct {
 	*chi.Mux
 }
 
+type fileMux struct {
+	*chi.Mux
+}
+
 // NewAPIRouter creates a new HTTP request path router/mux for the given API,
 // appContext.
 func NewAPIRouter(app *appContext, useRealIP bool) apiMux {
 	// chi router
-	mux := chi.NewRouter()
-
-	if useRealIP {
-		mux.Use(middleware.RealIP)
-	}
-	mux.Use(middleware.Logger)
-	mux.Use(middleware.Recoverer)
-	//mux.Use(middleware.DefaultCompress)
-	//mux.Use(middleware.Compress(2))
-	corsMW := cors.Default()
-	mux.Use(corsMW.Handler)
+	mux := stackedMux(useRealIP)
 
 	mux.Get("/", app.root)
 
@@ -240,6 +234,32 @@ func NewAPIRouter(app *appContext, useRealIP bool) apiMux {
 	return apiMux{mux}
 }
 
+// NewFileRouter creates a new HTTP request path router/mux for file downloads.
+func NewFileRouter(app *appContext, useRealIP bool) fileMux {
+	mux := stackedMux(useRealIP)
+
+	mux.Route("/address", func(rd chi.Router) {
+		rd.With(m.AddressPathCtx).Get("/io/{address}", app.addressIoCsv)
+	})
+
+	return fileMux{mux}
+}
+
+// Stacks some middleware common to both file and api router.
+func stackedMux(useRealIP bool) *chi.Mux {
+	mux := chi.NewRouter()
+	if useRealIP {
+		mux.Use(middleware.RealIP)
+	}
+	mux.Use(middleware.Logger)
+	mux.Use(middleware.Recoverer)
+	//mux.Use(middleware.DefaultCompress)
+	//mux.Use(middleware.Compress(2))
+	corsMW := cors.Default()
+	mux.Use(corsMW.Handler)
+	return mux
+}
+
 func (mux *apiMux) ListenAndServeProto(listen, proto string) {
 	apiLog.Infof("Now serving on %s://%v/", proto, listen)
 	if proto == "https" {

api/apiroutes.go

@@ -5,8 +5,11 @@
 package api
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"database/sql"
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -16,6 +19,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/decred/dcrd/chaincfg"
 	"github.com/decred/dcrd/dcrjson"
@@ -100,6 +104,8 @@ type DataSourceAux interface {
 	TicketPoolVisualization(interval dbtypes.TimeBasedGrouping) (
 		*dbtypes.PoolTicketsData, *dbtypes.PoolTicketsData, *dbtypes.PoolTicketsData, uint64, error)
 	AgendaVotes(agendaID string, chartType int) (*dbtypes.AgendaVoteChoices, error)
+	AddressTxIoCsv(address string) ([][]string, error)
+	Height() uint64
 }
 
 // dcrdata application context used by all route handlers
@@ -110,7 +116,6 @@ type appContext struct {
 	AuxDataSource DataSourceAux
 	LiteMode      bool
 	Status        apitypes.Status
-	statusMtx     sync.RWMutex
 	JSONIndent    string
 }
 
@@ -155,21 +160,20 @@ out:
 				break out
 			}
 
-			c.statusMtx.Lock()
+			c.Status.Lock()
 			c.Status.Height = height
-
 			// if DB height agrees with node height, then we're ready
 			c.Status.Ready = c.Status.Height == c.Status.DBHeight
 
 			var err error
 			c.Status.NodeConnections, err = c.nodeClient.GetConnectionCount()
 			if err != nil {
 				c.Status.Ready = false
-				c.statusMtx.Unlock()
+				c.Status.Unlock()
 				log.Warn("Failed to get connection count: ", err)
 				break keepon
 			}
-			c.statusMtx.Unlock()
+			c.Status.Unlock()
 
 		case height, ok := <-notify.NtfnChans.UpdateStatusDBHeight:
 			if !ok {
@@ -187,7 +191,7 @@ out:
 				break keepon
 			}
 
-			c.statusMtx.Lock()
+			c.Status.Lock()
 			c.Status.DBHeight = height
 			c.Status.DBLastBlockTime = summary.Time.S.T.Unix()
 
@@ -196,12 +200,12 @@ out:
 				height == uint32(bdHeight) {
 				// if DB height agrees with node height, then we're ready
 				c.Status.Ready = c.Status.Height == c.Status.DBHeight
-				c.statusMtx.Unlock()
+				c.Status.Unlock()
 				break keepon
 			}
 
 			c.Status.Ready = false
-			c.statusMtx.Unlock()
+			c.Status.Unlock()
 			log.Errorf("New DB height (%d) and stored block data (%d, %d) not consistent.",
 				height, bdHeight, summary.Height)
 
@@ -233,6 +237,32 @@ func writeJSON(w http.ResponseWriter, thing interface{}, indent string) {
 	}
 }
 
+// Measures length, sets common headers, formats, and sends CSV data.
+func writeCSV(w http.ResponseWriter, rows [][]string, filename string) {
+	w.Header().Set("Content-Disposition",
+		fmt.Sprintf("attachment;filename=%s", filename))
+	w.Header().Set("Content-Type", "text/csv")
+
+	buffer := new(bytes.Buffer)
+	bufferWriter := bufio.NewWriter(buffer)
+	writer := csv.NewWriter(bufferWriter)
+	err := writer.WriteAll(rows)
+	if err != nil {
+		log.Errorf("Failed to write address rows to buffer: %v", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Length", strconv.FormatInt(int64(buffer.Len()), 10))
+
+	written, err := buffer.WriteTo(w)
+	if err != nil {
+		log.Errorf("Failed to transfer address rows from buffer. %d bytes written. %v", written, err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+}
+
 func (c *appContext) getIndentQuery(r *http.Request) (indent string) {
 	useIndentation := r.URL.Query().Get("indent")
 	if useIndentation == "1" || useIndentation == "true" {
@@ -260,9 +290,9 @@ func getVoteVersionQuery(r *http.Request) (int32, string, error) {
 }
 
 func (c *appContext) status(w http.ResponseWriter, r *http.Request) {
-	c.statusMtx.RLock()
-	defer c.statusMtx.RUnlock()
-	writeJSON(w, c.Status, c.getIndentQuery(r))
+	c.Status.RLock()
+	defer c.Status.RUnlock()
+	writeJSON(w, &c.Status, c.getIndentQuery(r))
 }
 
 func (c *appContext) coinSupply(w http.ResponseWriter, r *http.Request) {
@@ -278,7 +308,7 @@ func (c *appContext) coinSupply(w http.ResponseWriter, r *http.Request) {
 
 func (c *appContext) currentHeight(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-	if _, err := io.WriteString(w, strconv.Itoa(int(c.Status.Height))); err != nil {
+	if _, err := io.WriteString(w, strconv.Itoa(int(c.Status.GetHeight()))); err != nil {
 		apiLog.Infof("failed to write height response: %v", err)
 	}
 }
@@ -1299,6 +1329,40 @@ func (c *appContext) addressTotals(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, totals, c.getIndentQuery(r))
 }
 
+// For /download/address/io/{address}
+func (c *appContext) addressIoCsv(w http.ResponseWriter, r *http.Request) {
+	if c.LiteMode {
+		http.Error(w, http.StatusText(http.StatusNotImplemented), http.StatusNotImplemented)
+		return
+	}
+
+	address := m.GetAddressCtx(r)
+	if address == "" {
+		log.Debugf("Failed to parse address from request")
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		return
+	}
+
+	_, _, addrErr := txhelpers.AddressValidation(address, c.Params)
+	if addrErr != nil {
+		log.Debugf("Error validating address %s: %v", address, addrErr)
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+
+	rows, err := c.AuxDataSource.AddressTxIoCsv(address)
+	if err != nil {
+		log.Errorf("Failed to fetch AddressTxIoCsv: %v", err)
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+
+	filename := fmt.Sprintf("address-io-%s-%d-%s.csv", address,
+		c.Status.GetHeight(), strconv.FormatInt(time.Now().Unix(), 10))
+
+	writeCSV(w, rows, filename)
+}
+
 func (c *appContext) getAddressTxTypesData(w http.ResponseWriter, r *http.Request) {
 	if c.LiteMode {
 		http.Error(w, "not available in lite mode", 422)

api/insight/apiroutes.go

@@ -470,7 +470,7 @@ func (c *insightApiContext) getTransactions(w http.ResponseWriter, r *http.Reque
 		}
 		addresses := []string{address}
 		rawTxs, recentTxs, err :=
-			c.BlockData.ChainDB.InsightAddressTransactions(addresses, int64(c.Status.Height-2))
+			c.BlockData.ChainDB.InsightAddressTransactions(addresses, int64(c.Status.GetHeight()-2))
 		if dbtypes.IsTimeoutErr(err) {
 			apiLog.Errorf("InsightAddressTransactions: %v", err)
 			http.Error(w, "Database timeout.", http.StatusServiceUnavailable)
@@ -573,7 +573,7 @@ func (c *insightApiContext) getAddressesTxn(w http.ResponseWriter, r *http.Reque
 	UnconfirmedTxs := []string{}
 
 	rawTxs, recentTxs, err :=
-		c.BlockData.ChainDB.InsightAddressTransactions(addresses, int64(c.Status.Height-2))
+		c.BlockData.ChainDB.InsightAddressTransactions(addresses, int64(c.Status.GetHeight()-2))
 	if dbtypes.IsTimeoutErr(err) {
 		apiLog.Errorf("InsightAddressTransactions: %v", err)
 		http.Error(w, "Database timeout.", http.StatusServiceUnavailable)
@@ -785,10 +785,10 @@ func (c *insightApiContext) getStatusInfo(w http.ResponseWriter, r *http.Request
 			writeInsightError(w, fmt.Sprintf("Error getting block hash %d (%s)", infoResult.Blocks, err))
 			return
 		}
-		lastblockhash, err := c.nodeClient.GetBlockHash(int64(c.Status.Height))
+		lastblockhash, err := c.nodeClient.GetBlockHash(int64(c.Status.GetHeight()))
 		if err != nil {
-			apiLog.Errorf("Error getting block hash %d (%s)", c.Status.Height, err)
-			writeInsightError(w, fmt.Sprintf("Error getting block hash %d (%s)", c.Status.Height, err))
+			apiLog.Errorf("Error getting block hash %d (%s)", c.Status.GetHeight(), err)
+			writeInsightError(w, fmt.Sprintf("Error getting block hash %d (%s)", c.Status.GetHeight(), err))
 			return
 		}
 
@@ -954,7 +954,7 @@ func (c *insightApiContext) getAddressInfo(w http.ResponseWriter, r *http.Reques
 
 	// Get confirmed transactions.
 	rawTxs, recentTxs, err :=
-		c.BlockData.ChainDB.InsightAddressTransactions(addresses, int64(c.Status.Height-2))
+		c.BlockData.ChainDB.InsightAddressTransactions(addresses, int64(c.Status.GetHeight()-2))
 	if dbtypes.IsTimeoutErr(err) {
 		apiLog.Errorf("InsightAddressTransactions: %v", err)
 		http.Error(w, "Database timeout.", http.StatusServiceUnavailable)

api/types/apitypes.go

@@ -5,6 +5,7 @@ package types
 
 import (
 	"encoding/json"
+	"sync"
 
 	"github.com/decred/dcrd/dcrjson"
 	"github.com/decred/dcrdata/v4/db/dbtypes"
@@ -261,6 +262,7 @@ type VinPrevOut struct {
 // Status indicates the state of the server, including the API version and the
 // software version.
 type Status struct {
+	sync.RWMutex    `json:"-"`
 	Ready           bool   `json:"ready"`
 	DBHeight        uint32 `json:"db_height"`
 	DBLastBlockTime int64  `json:"db_block_time"`
@@ -271,6 +273,12 @@ type Status struct {
 	NetworkName     string `json:"network_name"`
 }
 
+func (s *Status) GetHeight() uint32 {
+	s.RLock()
+	defer s.RUnlock()
+	return s.Height
+}
+
 // CoinSupply models the coin supply at a certain best block.
 type CoinSupply struct {
 	Height   int64  `json:"block_height"`

db/dcrpg/internal/addrstmts.go

@@ -175,6 +175,9 @@ const (
 		GROUP BY (tx_hash, valid_mainchain, block_time)  -- merging common transactions in same valid mainchain block
 		ORDER BY block_time DESC LIMIT $2 OFFSET $3;`
 
+	SelectAddressCsvView = "SELECT tx_hash, valid_mainchain, matching_tx_hash, value, block_time, is_funding, " +
+		"tx_vin_vout_index, tx_type FROM addresses WHERE address=$1 ORDER BY block_time DESC"
+
 	SelectAddressDebitsLimitNByAddress = `SELECT ` + addrsColumnNames + `
 		FROM addresses WHERE address=$1 AND is_funding = FALSE AND valid_mainchain = TRUE
 		ORDER BY block_time DESC LIMIT $2 OFFSET $3;`

db/dcrpg/pgblockchain.go

@@ -719,6 +719,11 @@ func (pgb *ChainDB) HeightHashDB() (uint64, string, error) {
 	return height, hash, pgb.replaceCancelError(err)
 }
 
+// Getter for ChainDB.bestBlock.height
+func (pgb *ChainDB) Height() uint64 {
+	return pgb.bestBlock.Height()
+}
+
 // Height uses the last stored height.
 func (block *BestBlock) Height() uint64 {
 	block.RLock()
@@ -1669,6 +1674,20 @@ func (pgb *ChainDB) AddressTotals(address string) (*apitypes.AddressTotals, erro
 	}, nil
 }
 
+// AddressTxIoCsv grabs rows of an address' transaction input/output data as a
+// 2-D array of strings to be CSV-formatted.
+func (pgb *ChainDB) AddressTxIoCsv(address string) (rows [][]string, err error) {
+	ctx, cancel := context.WithTimeout(pgb.ctx, pgb.queryTimeout)
+	defer cancel()
+
+	rows, err = retrieveAddressIoCsv(ctx, pgb.db, address)
+	if err != nil {
+		return nil, fmt.Errorf("AddressTxIoCsv error: %v", err)
+	}
+
+	return
+}
+
 func (pgb *ChainDB) addressInfo(addr string, count, skip int64, txnType dbtypes.AddrTxnType) (*dbtypes.AddressInfo, *dbtypes.AddressBalance, error) {
 	address, err := dcrutil.DecodeAddress(addr)
 	if err != nil {