Merge pull request #50 from auth0/logs-formatting-tweaks

Logs: polish the streaming and display

go.mod

@@ -9,9 +9,11 @@ require (
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/golang/mock v1.4.4
 	github.com/google/go-cmp v0.5.4
+	github.com/jsanda/tablewriter v0.0.2-0.20190614032957-c4e45dc9c708
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/mattn/go-colorable v0.1.6 // indirect
-	github.com/olekukonko/tablewriter v0.0.4
+	github.com/mattn/go-runewidth v0.0.10 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.5.1

go.sum

@@ -82,8 +82,6 @@ github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-auth0/auth0 v1.3.1-0.20210126044025-c2f94c36f593 h1:Ty0FV0S+59z1ioKa6uM0WW9i39HgZM46oAtFh++rUHQ=
-github.com/go-auth0/auth0 v1.3.1-0.20210126044025-c2f94c36f593/go.mod h1:pbIRmwBulkHNKKsUGGvhyIOI2itMhz2OfwtPaFXBSSQ=
 github.com/go-auth0/auth0 v1.3.1-0.20210126164439-7451a183abbd h1:O1ZCQlPODGWJNDT0qH6SpGdFEcEBWz7dd/mHFw6Kh10=
 github.com/go-auth0/auth0 v1.3.1-0.20210126164439-7451a183abbd/go.mod h1:QQ9fgGj2Wpza15+Ho3mM6amMeKfhzHo2cixcOqdkoKk=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -158,6 +156,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jsanda/tablewriter v0.0.2-0.20190614032957-c4e45dc9c708 h1:iHNpdSksqNoMQLzebZoWhbRNlQbgaKKB+x/VrP+UDH8=
+github.com/jsanda/tablewriter v0.0.2-0.20190614032957-c4e45dc9c708/go.mod h1:9Nb9KesU8Gu6NiQmPbF7ZV+lVoUvvIoO3GASEhHBdb0=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -185,8 +185,10 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
-github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -196,8 +198,8 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
-github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
+github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
+github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -212,6 +214,10 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

internal/cli/cli.go

@@ -182,6 +182,8 @@ func (c *cli) init() error {
 	}
 	c.renderer.Format = display.OutputFormat(format)
 
+	c.renderer.Tenant = c.tenant
+
 	// Once initialized, we'll keep returning the same err that was
 	// originally encountered.
 	return c.errOnce

internal/cli/cli_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/olekukonko/tablewriter"
+	"github.com/jsanda/tablewriter"
 )
 
 // TODO(cyx): think about whether we should extract this function in the

internal/cli/logs.go

@@ -5,51 +5,25 @@ import (
 	"sort"
 	"time"
 
-	"github.com/auth0/auth0-cli/internal/ansi"
 	"github.com/spf13/cobra"
 	"gopkg.in/auth0.v5/management"
 )
 
-func getLatestLogs(cli *cli, n int) (result []*management.Log, err error) {
-	var list []*management.Log
+func getLatestLogs(cli *cli, n int) ([]*management.Log, error) {
 	page := 0
-	perPage := 100
-	var count int
-	if count = n; n > 1000 {
+	perPage := n
+
+	if perPage > 1000 {
 		// Pagination max out at 1000 entries in total
 		// https://auth0.com/docs/logs/retrieve-log-events-using-mgmt-api#limitations
-		count = 1000
-	}
-	if perPage > count {
-		perPage = count
+		perPage = 1000
 	}
 
-	err = ansi.Spinner("Getting logs", func() error {
-		for count > len(result) {
-			var err error
-			list, err = cli.api.Log.List(
-				management.Parameter("sort", "date:-1"),
-				management.Parameter("page", fmt.Sprintf("%d", page)),
-				management.Parameter("per_page", fmt.Sprintf("%d", perPage)),
-			)
-			if err != nil {
-				return err
-			}
-
-			sort.Slice(list, func(i, j int) bool {
-				return list[i].GetDate().Before(list[j].GetDate())
-			})
-			result = append(list, result...)
-			if len(list) < perPage {
-				// We've got all
-				break
-			}
-			page++
-		}
-		return err
-	})
-
-	return
+	return cli.api.Log.List(
+		management.Parameter("sort", "date:-1"),
+		management.Parameter("page", fmt.Sprintf("%d", page)),
+		management.Parameter("per_page", fmt.Sprintf("%d", perPage)),
+	)
 }
 
 func logsCmd(cli *cli) *cobra.Command {
@@ -68,29 +42,54 @@ Show the tenant logs.
 			if err != nil {
 				return err
 			}
+
+			// TODO(cyx): This is a hack for now to make the
+			// streaming work faster.
+			//
+			// Create a `set` to detect duplicates clientside.
+			set := make(map[string]struct{})
+			list = dedupLogs(list, set)
+
 			if len(list) > 0 {
 				lastLogID = list[len(list)-1].GetLogID()
-				cli.renderer.LogList(list, noColor)
 			}
+
+			var logsCh chan []*management.Log
 			if follow {
-				for {
-					list, err = cli.api.Log.List(
-						management.Parameter("from", lastLogID),
-						management.Parameter("take", "100"),
-					)
-					if err != nil {
-						return err
-					}
-					if len(list) > 0 {
-						cli.renderer.LogList(list, noColor)
-						lastLogID = list[len(list)-1].GetLogID()
-					}
-					if len(list) < 90 {
-						// Not a lot is happening, sleep on it
-						time.Sleep(1 * time.Second)
+				logsCh = make(chan []*management.Log)
+
+				go func() {
+					// This is pretty important and allows
+					// us to close / terminate the command.
+					defer close(logsCh)
+
+					for {
+						list, err = cli.api.Log.List(
+							management.Query(fmt.Sprintf("log_id:[%s TO *]", lastLogID)),
+							management.Parameter("page", "0"),
+							management.Parameter("per_page", "100"),
+							management.Parameter("sort", "date:-1"),
+						)
+						if err != nil {
+							cli.renderer.Errorf("Error: %v", err)
+							return
+						}
+
+						if len(list) > 0 {
+							logsCh <- dedupLogs(list, set)
+							lastLogID = list[len(list)-1].GetLogID()
+						}
+
+						if len(list) < 90 {
+							// Not a lot is happening, sleep on it
+							time.Sleep(1 * time.Second)
+						}
 					}
-				}
+
+				}()
 			}
+
+			cli.renderer.LogList(list, logsCh, noColor)
 			return nil
 		},
 	}
@@ -101,3 +100,21 @@ Show the tenant logs.
 
 	return cmd
 }
+
+func dedupLogs(list []*management.Log, set map[string]struct{}) []*management.Log {
+	res := make([]*management.Log, 0, len(list))
+
+	for _, l := range list {
+		if _, ok := set[l.GetID()]; !ok {
+			// It's not a duplicate, track it, and take it.
+			set[l.GetID()] = struct{}{}
+			res = append(res, l)
+		}
+	}
+
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].GetDate().Before(res[j].GetDate())
+	})
+
+	return res
+}

internal/display/actions.go

@@ -62,7 +62,7 @@ func (r *Renderer) ActionList(actions []*management.Action) {
 
 func (r *Renderer) ActionTest(payload management.Object) {
 	r.Heading(ansi.Bold(r.Tenant), "Actions test result\n")
-	r.JSONResult(payload)
+	r.JSONResult(payload, nil)
 }
 
 func (r *Renderer) ActionCreate(action *management.Action) {

internal/display/display.go

@@ -9,8 +9,8 @@ import (
 	"time"
 
 	"github.com/auth0/auth0-cli/internal/ansi"
+	"github.com/jsanda/tablewriter"
 	"github.com/logrusorgru/aurora"
-	"github.com/olekukonko/tablewriter"
 )
 
 type OutputFormat string
@@ -63,7 +63,7 @@ type View interface {
 	AsTableRow() []string
 }
 
-func (r *Renderer) JSONResult(data interface{}) {
+func (r *Renderer) JSONResult(data interface{}, ch <-chan View) {
 	b, err := json.MarshalIndent(data, "", "    ")
 	if err != nil {
 		r.Errorf("couldn't marshal results as JSON: %v", err)
@@ -73,24 +73,27 @@ func (r *Renderer) JSONResult(data interface{}) {
 }
 
 func (r *Renderer) Results(data []View) {
+	r.Stream(data, nil)
+}
+
+func (r *Renderer) Stream(data []View, ch <-chan View) {
 	if len(data) > 0 {
 		switch r.Format {
 		case OutputFormatJSON:
-			r.JSONResult(data)
+			r.JSONResult(data, ch)
 
 		default:
 			rows := make([][]string, len(data))
 			for i, d := range data {
 				rows[i] = d.AsTableRow()
 			}
-			writeTable(r.ResultWriter, data[0].AsTableHeader(), rows)
+			writeTable(r.ResultWriter, data[0].AsTableHeader(), rows, ch)
 		}
 	}
 }
 
-func writeTable(w io.Writer, header []string, data [][]string) {
-	tableString := &strings.Builder{}
-	table := tablewriter.NewWriter(tableString)
+func writeTable(w io.Writer, header []string, data [][]string, ch <-chan View) {
+	table := tablewriter.NewWriter(w)
 	table.SetHeader(header)
 
 	table.SetAutoWrapText(false)
@@ -103,12 +106,31 @@ func writeTable(w io.Writer, header []string, data [][]string) {
 	table.SetHeaderLine(false)
 	table.SetBorder(false)
 
-	for _, v := range data {
-		table.Append(v)
+	if ch == nil {
+		for _, v := range data {
+			table.Append(v)
+		}
+		table.Render()
+		return
 	}
 
-	table.Render()
-	fmt.Fprint(w, tableString.String())
+	done := make(chan struct{})
+	strCh := make(chan []string)
+	go func() {
+		defer close(done)
+
+		for _, v := range data {
+			strCh <- v
+		}
+
+		for v := range ch {
+			strCh <- v.AsTableRow()
+		}
+	}()
+
+	go table.ContinuousRender(strCh)
+
+	<-done
 }
 
 func timeAgo(ts time.Time) string {
@@ -119,6 +141,9 @@ func timeAgo(ts time.Time) string {
 
 	v := time.Since(ts)
 	switch {
+	case v < time.Minute:
+		return fmt.Sprintf("%d seconds ago", v/time.Second)
+
 	case v < 2*time.Minute:
 		return "a minute ago"
 

internal/display/display_test.go

@@ -13,7 +13,7 @@ func TestTimeAgo(t *testing.T) {
 		ts   time.Time
 		want string
 	}{
-		{t0, "a minute ago"},
+		{t0, "0 seconds ago"},
 		{t0.Add(-2 * time.Minute), "2 minutes ago"},
 		{t0.Add(-119 * time.Minute), "an hour ago"},
 		{t0.Add(-3 * time.Hour), "3 hours ago"},