Skip to content

Commit

Permalink
Merge branch 'johan/textstyles'
Browse files Browse the repository at this point in the history
This moves text styling code into its own package.
walles committed Jan 5, 2024
2 parents 192838a + c702282 commit 54c88d3
Showing 10 changed files with 161 additions and 121 deletions.
63 changes: 63 additions & 0 deletions m/line.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package m

import (
"regexp"

"github.com/walles/moar/textstyles"
"github.com/walles/moar/twin"
)

// A Line represents a line of text that can / will be paged
type Line struct {
raw string
plain *string
}

// NewLine creates a new Line from a (potentially ANSI / man page formatted) string
func NewLine(raw string) Line {
return Line{
raw: raw,
plain: nil,
}
}

// Returns a representation of the string split into styled tokens. Any regexp
// matches are highlighted. A nil regexp means no highlighting.
//
//revive:disable-next-line:unexported-return
func (line *Line) HighlightedTokens(linePrefix string, search *regexp.Regexp, lineNumberOneBased *int) textstyles.CellsWithTrailer {
plain := line.Plain(lineNumberOneBased)
matchRanges := getMatchRanges(&plain, search)

fromString := textstyles.CellsFromString(linePrefix+line.raw, lineNumberOneBased)
returnCells := make([]twin.Cell, 0, len(fromString.Cells))
for _, token := range fromString.Cells {
style := token.Style
if matchRanges.InRange(len(returnCells)) {
if standoutStyle != nil {
style = *standoutStyle
} else {
style = style.WithAttr(twin.AttrReverse)
}
}

returnCells = append(returnCells, twin.Cell{
Rune: token.Rune,
Style: style,
})
}

return textstyles.CellsWithTrailer{
Cells: returnCells,
Trailer: fromString.Trailer,
}
}

// Plain returns a plain text representation of the initial string
func (line *Line) Plain(lineNumberOneBased *int) string {
if line.plain == nil {
plain := textstyles.WithoutFormatting(line.raw, lineNumberOneBased)
line.plain = &plain
}
return *line.plain
}
17 changes: 3 additions & 14 deletions m/pager.go
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import (

"github.com/alecthomas/chroma/v2"
log "github.com/sirupsen/logrus"
"github.com/walles/moar/textstyles"
"github.com/walles/moar/twin"
)

@@ -32,18 +33,6 @@ const (
STATUSBAR_STYLE_BOLD
)

// How do we render unprintable characters?
type UnprintableStyle int

const (
//revive:disable-next-line:var-naming
UNPRINTABLE_STYLE_HIGHLIGHT UnprintableStyle = iota
//revive:disable-next-line:var-naming
UNPRINTABLE_STYLE_WHITESPACE
)

var unprintableStyle UnprintableStyle

type eventSpinnerUpdate struct {
spinner string
}
@@ -79,7 +68,7 @@ type Pager struct {
StatusBarStyle StatusBarOption
ShowStatusBar bool

UnprintableStyle UnprintableStyle
UnprintableStyle textstyles.UnprintableStyleT

WrapLongLines bool

@@ -478,7 +467,7 @@ func (p *Pager) StartPaging(screen twin.Screen, chromaStyle *chroma.Style, chrom
}
}()

unprintableStyle = p.UnprintableStyle
textstyles.UnprintableStyle = p.UnprintableStyle
consumeLessTermcapEnvs(chromaStyle, chromaFormatter)
styleUI(chromaStyle, chromaFormatter, p.StatusBarStyle)

5 changes: 3 additions & 2 deletions m/pager_test.go
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ import (
"github.com/alecthomas/chroma/v2/lexers"
"github.com/alecthomas/chroma/v2/styles"
"github.com/google/go-cmp/cmp"
"github.com/walles/moar/textstyles"
"github.com/walles/moar/twin"
"gotest.tools/v3/assert"
)
@@ -208,8 +209,8 @@ func TestUnicodePrivateUse(t *testing.T) {
}

func resetManPageFormat() {
manPageBold = twin.StyleDefault.WithAttr(twin.AttrBold)
manPageUnderline = twin.StyleDefault.WithAttr(twin.AttrUnderline)
textstyles.ManPageBold = twin.StyleDefault.WithAttr(twin.AttrBold)
textstyles.ManPageUnderline = twin.StyleDefault.WithAttr(twin.AttrUnderline)
}

func testManPageFormatting(t *testing.T, input string, expected twin.Cell) {
3 changes: 2 additions & 1 deletion m/screenLines.go
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ package m
import (
"fmt"

"github.com/walles/moar/textstyles"
"github.com/walles/moar/twin"
)

@@ -52,7 +53,7 @@ func (p *Pager) redraw(spinner string) overflowState {
// This happens when we're done
eofSpinner = "---"
}
spinnerLine := cellsFromString(_EofMarkerFormat+eofSpinner, nil).Cells
spinnerLine := textstyles.CellsFromString(_EofMarkerFormat+eofSpinner, nil).Cells
for column, cell := range spinnerLine {
p.screen.SetCell(column, lastUpdatedScreenLineNumber+1, cell)
}
12 changes: 5 additions & 7 deletions m/styling.go
Original file line number Diff line number Diff line change
@@ -7,15 +7,13 @@ import (

"github.com/alecthomas/chroma/v2"
log "github.com/sirupsen/logrus"
"github.com/walles/moar/textstyles"
"github.com/walles/moar/twin"
)

// From LESS_TERMCAP_so, overrides statusbarStyle from the Chroma style if set
var standoutStyle *twin.Style

var manPageBold = twin.StyleDefault.WithAttr(twin.AttrBold)
var manPageUnderline = twin.StyleDefault.WithAttr(twin.AttrUnderline)

var lineNumbersStyle = twin.StyleDefault.WithAttr(twin.AttrDim)
var statusbarStyle = twin.StyleDefault.WithAttr(twin.AttrReverse)

@@ -46,7 +44,7 @@ func twinStyleFromChroma(chromaStyle *chroma.Style, chromaFormatter *chroma.Form
}

formatted := stringBuilder.String()
cells := cellsFromString(formatted, nil).Cells
cells := textstyles.CellsFromString(formatted, nil).Cells
if len(cells) != 1 {
log.Warnf("Chroma formatter didn't return exactly one cell: %#v", cells)
return nil
@@ -60,8 +58,8 @@ func twinStyleFromChroma(chromaStyle *chroma.Style, chromaFormatter *chroma.Form
func consumeLessTermcapEnvs(chromaStyle *chroma.Style, chromaFormatter *chroma.Formatter) {
// Requested here: https://github.com/walles/moar/issues/14

setStyle(&manPageBold, "LESS_TERMCAP_md", twinStyleFromChroma(chromaStyle, chromaFormatter, chroma.GenericStrong))
setStyle(&manPageUnderline, "LESS_TERMCAP_us", twinStyleFromChroma(chromaStyle, chromaFormatter, chroma.GenericUnderline))
setStyle(&textstyles.ManPageBold, "LESS_TERMCAP_md", twinStyleFromChroma(chromaStyle, chromaFormatter, chroma.GenericStrong))
setStyle(&textstyles.ManPageUnderline, "LESS_TERMCAP_us", twinStyleFromChroma(chromaStyle, chromaFormatter, chroma.GenericUnderline))

// Since standoutStyle defaults to nil we can't just pass it to setStyle().
// Instead we give it special treatment here and set it only if its
@@ -114,6 +112,6 @@ func styleUI(chromaStyle *chroma.Style, chromaFormatter *chroma.Formatter, statu

func termcapToStyle(termcap string) twin.Style {
// Add a character to be sure we have one to take the format from
cells := cellsFromString(termcap+"x", nil).Cells
cells := textstyles.CellsFromString(termcap+"x", nil).Cells
return cells[len(cells)-1].Style
}
9 changes: 5 additions & 4 deletions moar.go
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@ import (
"golang.org/x/term"

"github.com/walles/moar/m"
"github.com/walles/moar/textstyles"
"github.com/walles/moar/twin"
)

@@ -222,12 +223,12 @@ func parseStatusBarStyle(styleOption string) (m.StatusBarOption, error) {
return 0, fmt.Errorf("Good ones are inverse, plain and bold")
}

func parseUnprintableStyle(styleOption string) (m.UnprintableStyle, error) {
func parseUnprintableStyle(styleOption string) (textstyles.UnprintableStyleT, error) {
if styleOption == "highlight" {
return m.UNPRINTABLE_STYLE_HIGHLIGHT, nil
return textstyles.UNPRINTABLE_STYLE_HIGHLIGHT, nil
}
if styleOption == "whitespace" {
return m.UNPRINTABLE_STYLE_WHITESPACE, nil
return textstyles.UNPRINTABLE_STYLE_WHITESPACE, nil
}

return 0, fmt.Errorf("Good ones are highlight or whitespace")
@@ -416,7 +417,7 @@ func main() {
noClearOnExit := flagSet.Bool("no-clear-on-exit", false, "Retain screen contents when exiting moar")
statusBarStyle := flagSetFunc(flagSet, "statusbar", m.STATUSBAR_STYLE_INVERSE,
"Status bar style: inverse, plain or bold", parseStatusBarStyle)
unprintableStyle := flagSetFunc(flagSet, "render-unprintable", m.UNPRINTABLE_STYLE_HIGHLIGHT,
unprintableStyle := flagSetFunc(flagSet, "render-unprintable", textstyles.UNPRINTABLE_STYLE_HIGHLIGHT,
"How unprintable characters are rendered: highlight or whitespace", parseUnprintableStyle)
scrollLeftHint := flagSetFunc(flagSet, "scroll-left-hint",
twin.NewCell('<', twin.StyleDefault.WithAttr(twin.AttrReverse)),
103 changes: 31 additions & 72 deletions m/ansiTokenizer.go → textstyles/ansiTokenizer.go
Original file line number Diff line number Diff line change
@@ -1,76 +1,35 @@
package m
package textstyles

import (
"fmt"
"regexp"
"strconv"
"strings"

"github.com/walles/moar/twin"
)

const _TabSize = 4

const BACKSPACE = '\b'
// How do we render unprintable characters?
type UnprintableStyleT int

// A Line represents a line of text that can / will be paged
type Line struct {
raw string
plain *string
}
const (
//revive:disable-next-line:var-naming
UNPRINTABLE_STYLE_HIGHLIGHT UnprintableStyleT = iota
//revive:disable-next-line:var-naming
UNPRINTABLE_STYLE_WHITESPACE
)

type cellsWithTrailer struct {
Cells []twin.Cell
Trailer twin.Style
}
var UnprintableStyle UnprintableStyleT

// NewLine creates a new Line from a (potentially ANSI / man page formatted) string
func NewLine(raw string) Line {
return Line{
raw: raw,
plain: nil,
}
}
var ManPageBold = twin.StyleDefault.WithAttr(twin.AttrBold)
var ManPageUnderline = twin.StyleDefault.WithAttr(twin.AttrUnderline)

// Returns a representation of the string split into styled tokens. Any regexp
// matches are highlighted. A nil regexp means no highlighting.
//
//revive:disable-next-line:unexported-return
func (line *Line) HighlightedTokens(linePrefix string, search *regexp.Regexp, lineNumberOneBased *int) cellsWithTrailer {
plain := line.Plain(lineNumberOneBased)
matchRanges := getMatchRanges(&plain, search)

fromString := cellsFromString(linePrefix+line.raw, lineNumberOneBased)
returnCells := make([]twin.Cell, 0, len(fromString.Cells))
for _, token := range fromString.Cells {
style := token.Style
if matchRanges.InRange(len(returnCells)) {
if standoutStyle != nil {
style = *standoutStyle
} else {
style = style.WithAttr(twin.AttrReverse)
}
}

returnCells = append(returnCells, twin.Cell{
Rune: token.Rune,
Style: style,
})
}
const _TabSize = 4

return cellsWithTrailer{
Cells: returnCells,
Trailer: fromString.Trailer,
}
}
const BACKSPACE = '\b'

// Plain returns a plain text representation of the initial string
func (line *Line) Plain(lineNumberOneBased *int) string {
if line.plain == nil {
plain := withoutFormatting(line.raw, lineNumberOneBased)
line.plain = &plain
}
return *line.plain
type CellsWithTrailer struct {
Cells []twin.Cell
Trailer twin.Style
}

func isPlain(s string) bool {
@@ -87,7 +46,7 @@ func isPlain(s string) bool {
return true
}

func withoutFormatting(s string, lineNumberOneBased *int) string {
func WithoutFormatting(s string, lineNumberOneBased *int) string {
if isPlain(s) {
return s
}
@@ -116,12 +75,12 @@ func withoutFormatting(s string, lineNumberOneBased *int) string {
}

case '�': // Go's broken-UTF8 marker
if unprintableStyle == UNPRINTABLE_STYLE_HIGHLIGHT {
if UnprintableStyle == UNPRINTABLE_STYLE_HIGHLIGHT {
stripped.WriteRune('?')
} else if unprintableStyle == UNPRINTABLE_STYLE_WHITESPACE {
} else if UnprintableStyle == UNPRINTABLE_STYLE_WHITESPACE {
stripped.WriteRune(' ')
} else {
panic(fmt.Errorf("Unsupported unprintable-style: %#v", unprintableStyle))
panic(fmt.Errorf("Unsupported unprintable-style: %#v", UnprintableStyle))
}
runeCount++

@@ -145,7 +104,7 @@ func withoutFormatting(s string, lineNumberOneBased *int) string {
}

// Turn a (formatted) string into a series of screen cells
func cellsFromString(s string, lineNumberOneBased *int) cellsWithTrailer {
func CellsFromString(s string, lineNumberOneBased *int) CellsWithTrailer {
var cells []twin.Cell

// Specs: https://en.wikipedia.org/wiki/ANSI_escape_code#3-bit_and_4-bit
@@ -169,18 +128,18 @@ func cellsFromString(s string, lineNumberOneBased *int) cellsWithTrailer {
}

case '�': // Go's broken-UTF8 marker
if unprintableStyle == UNPRINTABLE_STYLE_HIGHLIGHT {
if UnprintableStyle == UNPRINTABLE_STYLE_HIGHLIGHT {
cells = append(cells, twin.Cell{
Rune: '?',
Style: styleUnprintable,
})
} else if unprintableStyle == UNPRINTABLE_STYLE_WHITESPACE {
} else if UnprintableStyle == UNPRINTABLE_STYLE_WHITESPACE {
cells = append(cells, twin.Cell{
Rune: '?',
Style: twin.StyleDefault,
})
} else {
panic(fmt.Errorf("Unsupported unprintable-style: %#v", unprintableStyle))
panic(fmt.Errorf("Unsupported unprintable-style: %#v", UnprintableStyle))
}

case BACKSPACE:
@@ -191,18 +150,18 @@ func cellsFromString(s string, lineNumberOneBased *int) cellsWithTrailer {

default:
if !twin.Printable(token.Rune) {
if unprintableStyle == UNPRINTABLE_STYLE_HIGHLIGHT {
if UnprintableStyle == UNPRINTABLE_STYLE_HIGHLIGHT {
cells = append(cells, twin.Cell{
Rune: '?',
Style: styleUnprintable,
})
} else if unprintableStyle == UNPRINTABLE_STYLE_WHITESPACE {
} else if UnprintableStyle == UNPRINTABLE_STYLE_WHITESPACE {
cells = append(cells, twin.Cell{
Rune: ' ',
Style: twin.StyleDefault,
})
} else {
panic(fmt.Errorf("Unsupported unprintable-style: %#v", unprintableStyle))
panic(fmt.Errorf("Unsupported unprintable-style: %#v", UnprintableStyle))
}
continue
}
@@ -211,7 +170,7 @@ func cellsFromString(s string, lineNumberOneBased *int) cellsWithTrailer {
}
})

return cellsWithTrailer{
return CellsWithTrailer{
Cells: cells,
Trailer: trailer,
}
@@ -237,7 +196,7 @@ func consumeBold(runes []rune, index int) (int, *twin.Cell) {
// We have a match!
return index + 3, &twin.Cell{
Rune: runes[index],
Style: manPageBold,
Style: ManPageBold,
}
}

@@ -261,7 +220,7 @@ func consumeUnderline(runes []rune, index int) (int, *twin.Cell) {
// We have a match!
return index + 3, &twin.Cell{
Rune: runes[index+2],
Style: manPageUnderline,
Style: ManPageUnderline,
}
}

66 changes: 47 additions & 19 deletions m/ansiTokenizer_test.go → textstyles/ansiTokenizer_test.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package m
package textstyles

import (
"bufio"
"fmt"
"os"
"path"
"runtime"
"strings"
"testing"
"unicode/utf8"

"github.com/alecthomas/chroma/v2"
"github.com/google/go-cmp/cmp"
log "github.com/sirupsen/logrus"

@@ -25,6 +27,30 @@ func cellsToPlainString(cells []twin.Cell) string {
return returnMe
}

func getSamplesDir() string {
// From: https://coderwall.com/p/_fmbug/go-get-path-to-current-file
_, filename, _, ok := runtime.Caller(0)
if !ok {
panic("Getting current filename failed")
}

return path.Join(path.Dir(filename), "../sample-files")
}

func getTestFiles() []string {
files, err := os.ReadDir(getSamplesDir())
if err != nil {
panic(err)
}

var filenames []string
for _, file := range files {
filenames = append(filenames, "../sample-files/"+file.Name())
}

return filenames
}

// Verify that we can tokenize all lines in ../sample-files/*
// without logging any errors
func TestTokenize(t *testing.T) {
@@ -41,24 +67,26 @@ func TestTokenize(t *testing.T) {
}
}()

myReader := NewReaderFromStream(fileName, file, chroma.Style{}, nil, nil)
//revive:disable-next-line:empty-block
for !myReader.done.Load() {
fileReader, err := os.Open(fileName)
if err != nil {
panic(err)
}

for lineNumber := 1; lineNumber <= myReader.GetLineCount(); lineNumber++ {
line := myReader.GetLine(lineNumber)
fileScanner := bufio.NewScanner(fileReader)
lineNumber := 1
for fileScanner.Scan() {
line := fileScanner.Text()
lineNumber++

var loglines strings.Builder
log.SetOutput(&loglines)

tokens := cellsFromString(line.raw, &lineNumber).Cells
plainString := withoutFormatting(line.raw, &lineNumber)
tokens := CellsFromString(line, &lineNumber).Cells
plainString := WithoutFormatting(line, &lineNumber)
if len(tokens) != utf8.RuneCountInString(plainString) {
t.Errorf("%s:%d: len(tokens)=%d, len(plainString)=%d for: <%s>",
fileName, lineNumber,
len(tokens), utf8.RuneCountInString(plainString), line.raw)
len(tokens), utf8.RuneCountInString(plainString), line)
continue
}

@@ -106,7 +134,7 @@ func TestTokenize(t *testing.T) {
}

func TestUnderline(t *testing.T) {
tokens := cellsFromString("a\x1b[4mb\x1b[24mc", nil).Cells
tokens := CellsFromString("a\x1b[4mb\x1b[24mc", nil).Cells
assert.Equal(t, len(tokens), 3)
assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
assert.Equal(t, tokens[1], twin.Cell{Rune: 'b', Style: twin.StyleDefault.WithAttr(twin.AttrUnderline)})
@@ -115,30 +143,30 @@ func TestUnderline(t *testing.T) {

func TestManPages(t *testing.T) {
// Bold
tokens := cellsFromString("ab\bbc", nil).Cells
tokens := CellsFromString("ab\bbc", nil).Cells
assert.Equal(t, len(tokens), 3)
assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
assert.Equal(t, tokens[1], twin.Cell{Rune: 'b', Style: twin.StyleDefault.WithAttr(twin.AttrBold)})
assert.Equal(t, tokens[2], twin.Cell{Rune: 'c', Style: twin.StyleDefault})

// Underline
tokens = cellsFromString("a_\bbc", nil).Cells
tokens = CellsFromString("a_\bbc", nil).Cells
assert.Equal(t, len(tokens), 3)
assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
assert.Equal(t, tokens[1], twin.Cell{Rune: 'b', Style: twin.StyleDefault.WithAttr(twin.AttrUnderline)})
assert.Equal(t, tokens[2], twin.Cell{Rune: 'c', Style: twin.StyleDefault})

// Bullet point 1, taken from doing this on my macOS system:
// env PAGER="hexdump -C" man printf | moar
tokens = cellsFromString("a+\b+\bo\bob", nil).Cells
tokens = CellsFromString("a+\b+\bo\bob", nil).Cells
assert.Equal(t, len(tokens), 3)
assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
assert.Equal(t, tokens[1], twin.Cell{Rune: '•', Style: twin.StyleDefault})
assert.Equal(t, tokens[2], twin.Cell{Rune: 'b', Style: twin.StyleDefault})

// Bullet point 2, taken from doing this using the "fish" shell on my macOS system:
// man printf | hexdump -C | moar
tokens = cellsFromString("a+\bob", nil).Cells
tokens = CellsFromString("a+\bob", nil).Cells
assert.Equal(t, len(tokens), 3)
assert.Equal(t, tokens[0], twin.Cell{Rune: 'a', Style: twin.StyleDefault})
assert.Equal(t, tokens[1], twin.Cell{Rune: '•', Style: twin.StyleDefault})
@@ -205,7 +233,7 @@ func TestRawUpdateStyle(t *testing.T) {
func TestHyperlink_escBackslash(t *testing.T) {
url := "http://example.com"

tokens := cellsFromString("a\x1b]8;;"+url+"\x1b\\bc\x1b]8;;\x1b\\d", nil).Cells
tokens := CellsFromString("a\x1b]8;;"+url+"\x1b\\bc\x1b]8;;\x1b\\d", nil).Cells

assert.DeepEqual(t, tokens, []twin.Cell{
{Rune: 'a', Style: twin.StyleDefault},
@@ -221,7 +249,7 @@ func TestHyperlink_escBackslash(t *testing.T) {
func TestHyperlink_bell(t *testing.T) {
url := "http://example.com"

tokens := cellsFromString("a\x1b]8;;"+url+"\x07bc\x1b]8;;\x07d", nil).Cells
tokens := CellsFromString("a\x1b]8;;"+url+"\x07bc\x1b]8;;\x07d", nil).Cells

assert.DeepEqual(t, tokens, []twin.Cell{
{Rune: 'a', Style: twin.StyleDefault},
@@ -234,7 +262,7 @@ func TestHyperlink_bell(t *testing.T) {
// Test with some other ESC sequence than ESC-backslash
func TestHyperlink_nonTerminatingEsc(t *testing.T) {
complete := "a\x1b]8;;https://example.com\x1bbc"
tokens := cellsFromString(complete, nil).Cells
tokens := CellsFromString(complete, nil).Cells

// This should not be treated as any link
for i := 0; i < len(complete); i++ {
@@ -254,7 +282,7 @@ func TestHyperlink_incomplete(t *testing.T) {
for l := len(complete) - 1; l >= 0; l-- {
incomplete := complete[:l]
t.Run(fmt.Sprintf("l=%d incomplete=<%s>", l, strings.ReplaceAll(incomplete, "\x1b", "ESC")), func(t *testing.T) {
tokens := cellsFromString(incomplete, nil).Cells
tokens := CellsFromString(incomplete, nil).Cells

for i := 0; i < l; i++ {
if complete[i] == '\x1b' {
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package m
package textstyles

import (
"fmt"
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package m
package textstyles

import (
"testing"

0 comments on commit 54c88d3

Please sign in to comment.