Bumped the github.com/zoomio/inout to the latest commit hash (#27)

* Bumped the github.com/zoomio/inout to the latest commit hash

* Introduced Timeout option to serve as an overall deadline for the operation

* Bumped inout, allow to pass user agent

* gitignore _dist

* Removed unknown "toolcahin" derective from go.mod

* Version bump + CHANGELOG

.gitignore

@@ -1,7 +1,9 @@
+/tagify
 /tagify_darwin*
 /tagify_linux*
 /cli
 vendor/
+/_dist/
 *.out
 *.test
 *.cpu

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v0.62.0
+
+- bumped Go to 1.22;
+- bumped `github.com/zoomio/inout` to v0.14.0;
+- introduced `UserAgent` (`-ua` in CLI mode) to allow to pass a custom user agent for headless HTTP calls.
+
 ## v0.61.0
 
 - bumped Go to 1.20;

Makefile

@@ -1,6 +1,6 @@
 .PHONY: deps clean build
 
-TAG=0.61.0
+TAG=0.62.0
 BINARY=tagify
 DIST_DIR=_dist
 OS=darwin

cmd/cli/cli.go

@@ -25,6 +25,7 @@ var (
 	ready = flag.String("r", "", "DOM CSS query, waits until certain element available, but fetches contents of the whole HTML document")
 	until = flag.Duration("u", 0, "duration to wait before getting HTML contents, handy for SPAs, because they keep loading in browsers for some time")
 	img   = flag.String("i", "", "enables capturing screenshot in the provided path")
+	ua    = flag.String("ua", "", "provide a custom user agent for headless HTTP calls")
 
 	limit       = flag.Int("l", 5, "number of tags to return")
 	verbose     = flag.Bool("v", false, "enables verbose mode")
@@ -93,6 +94,9 @@ func main() {
 	if len(*img) > 0 {
 		options = append(options, tagify.Screenshot(true))
 	}
+	if *ua != "" {
+		options = append(options, tagify.UserAgent(*ua))
+	}
 
 	if *verbose {
 		options = append(options, tagify.Verbose(*verbose))

config/config.go

@@ -45,11 +45,14 @@ type Config struct {
 	ContentType
 	Content string
 
+	Timeout time.Duration
+
 	// headless
 	Query      string
 	WaitFor    string
 	WaitUntil  time.Duration
 	Screenshot bool
+	UserAgent  string
 
 	// misc
 	Limit       int

config/options.go

@@ -36,6 +36,13 @@ var (
 		}
 	}
 
+	// Timeout sets the overall deadline for the operation.
+	Timeout = func(d time.Duration) Option {
+		return func(c *Config) {
+			c.Timeout = d
+		}
+	}
+
 	// WaitFor sets CSS query for the target of In-Out.
 	WaitFor = func(query string) Option {
 		return func(c *Config) {
@@ -57,6 +64,13 @@ var (
 		}
 	}
 
+	// UserAgent allows to specify custom User Agent of the HTTP headless calls.
+	UserAgent = func(ua string) Option {
+		return func(c *Config) {
+			c.UserAgent = ua
+		}
+	}
+
 	// Content sets content of the target.
 	Content = func(v string) Option {
 		return func(c *Config) {

go.mod

@@ -4,29 +4,29 @@ require (
 	github.com/abadojack/whatlanggo v1.0.1
 	github.com/go-ego/gse v0.70.2
 	github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a
-	github.com/stretchr/testify v1.7.0
-	github.com/zoomio/inout v0.12.0
+	github.com/stretchr/testify v1.8.4
+	github.com/zoomio/inout v0.14.0
 	github.com/zoomio/stopwords v0.11.0
 	golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d
 )
 
 require (
-	github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c // indirect
-	github.com/chromedp/chromedp v0.8.3 // indirect
+	github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89 // indirect
+	github.com/chromedp/chromedp v0.9.2 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/gobwas/ws v1.1.0 // indirect
+	github.com/gobwas/ws v1.2.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/vcaesar/cedar v0.20.1 // indirect
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/sys v0.6.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
-	gopkg.in/yaml.v3 v3.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-go 1.20
+go 1.22

go.sum

@@ -1,13 +1,12 @@
 github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
 github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
-github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c h1:Gm+DujZPVAtQNTLhbg5PExjRNfhdTCSMLvJ/pFfY4aY=
-github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0=
-github.com/chromedp/chromedp v0.8.3 h1:UwOY+fhC5Vv3uKgRpnvilCbWs/QPz8ciFwRB0q6pH8k=
-github.com/chromedp/chromedp v0.8.3/go.mod h1:9YfKSJnBNeP77vKecv+DNx2/Tcb+6Gli0d1aZPw/xbk=
+github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89 h1:aPflPkRFkVwbW6dmcVqfgwp1i+UWGFH6VgR1Jim5Ygc=
+github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
+github.com/chromedp/chromedp v0.9.2 h1:dKtNz4kApb06KuSXoTQIyUC2TrA0fhGDwNZf3bcgfKw=
+github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs=
 github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
 github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-ego/gse v0.70.2 h1:y2UMOHJMtI+0b2GjxTtQfKON5DMmlyX1hOQHTo8UVVs=
@@ -16,8 +15,8 @@ github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU
 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
 github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
 github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
-github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
-github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
+github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk=
+github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a h1:eeaG9XMUvRBYXJi4pg1ZKM7nxc5AfXfojeLLW7O5J3k=
 github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -26,31 +25,32 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
+github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5 h1:1SoBaSPudixRecmlHXb/GxmaD3fLMtHIDN13QujwQuc=
+github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
+github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/vcaesar/cedar v0.20.1 h1:cDOmYWdprO7ZW8cngJrDi8Zivnscj9dA/y8Y+2SB1P0=
 github.com/vcaesar/cedar v0.20.1/go.mod h1:iMDweyuW76RvSrCkQeZeQk4iCbshiPzcCvcGCtpM7iI=
 github.com/vcaesar/tt v0.20.0 h1:9t2Ycb9RNHcP0WgQgIaRKJBB+FrRdejuaL6uWIHuoBA=
-github.com/zoomio/inout v0.12.0 h1:B+LcJFSdSn7Vgt4dQim8c8bDBBA3wAbl9EVV+B9Pr2Y=
-github.com/zoomio/inout v0.12.0/go.mod h1:QGuD4z2/PEqRwlqCLG3H5Ye35zzQg4kj1rr7By7Gxco=
+github.com/vcaesar/tt v0.20.0/go.mod h1:GHPxQYhn+7OgKakRusH7KJ0M5MhywoeLb8Fcffs/Gtg=
+github.com/zoomio/inout v0.14.0 h1:Q/9EJudnSM71CuwqWsfr7TqNhhsPBaCT+gxh/57ozi4=
+github.com/zoomio/inout v0.14.0/go.mod h1:JFLyhAGsULpcarxp0hbLQEB1PGfkyRv1Y5bbiZ3vqcc=
 github.com/zoomio/stopwords v0.11.0 h1:O2JY3zFymge1u/9FkEjG7PtloJMiqWH+P6NKtTDKvLY=
 github.com/zoomio/stopwords v0.11.0/go.mod h1:G4EP6eLrE9A9HhG0JcOVcv9UdPpjthtqaA5MpgkKmoc=
 golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d h1:62NvYBuaanGXR2ZOfwDFkhhl6X1DUgf8qg3GuQvxZsE=
 golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
-gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

in.go

@@ -20,23 +20,25 @@ type in struct {
 //
 // source - the filename or web page source, reads from STDIN if source is empty.
 // Panics on errors.
-func newIn(ctx context.Context, c *Config) (in, error) {
-	in := in{source: c.Source}
+func newIn(ctx context.Context, cfg *Config) (in, error) {
+	in := in{source: cfg.Source}
 
-	if strings.HasPrefix(c.Source, "http://") || strings.HasPrefix(c.Source, "https://") || c.Query != "" {
+	if strings.HasPrefix(cfg.Source, "http://") || strings.HasPrefix(cfg.Source, "https://") || cfg.Query != "" {
 		in.ContentType = HTML
-	} else if strings.ToLower(filepath.Ext(c.Source)) == ".md" {
+	} else if strings.ToLower(filepath.Ext(cfg.Source)) == ".md" {
 		in.ContentType = Markdown
 	}
 
 	r, err := inout.NewInOut(ctx,
-		inout.Source(c.Source),
-		inout.Query(c.Query),
-		inout.WaitFor(c.WaitFor),
-		inout.WaitUntil(c.WaitUntil),
-		inout.Screenshot(c.Screenshot),
-		inout.Timeout(0),
-		inout.Verbose(c.Verbose))
+		inout.Source(cfg.Source),
+		inout.Query(cfg.Query),
+		inout.WaitFor(cfg.WaitFor),
+		inout.WaitUntil(cfg.WaitUntil),
+		inout.Screenshot(cfg.Screenshot),
+		inout.Timeout(cfg.Timeout),
+		inout.Verbose(cfg.Verbose),
+		inout.UserAgent(cfg.UserAgent),
+	)
 	if err != nil {
 		return in, err
 	}

model.go

@@ -14,11 +14,14 @@ var (
 	Language = config.Language
 	Content  = config.Content
 
+	Timeout = config.Timeout
+
 	// headless
 	Query      = config.Query
 	WaitFor    = config.WaitFor
 	WaitUntil  = config.WaitUntil
 	Screenshot = config.Screenshot
+	UserAgent  = config.UserAgent
 
 	// misc
 	TargetType  = config.TargetType

tagify.go

@@ -15,32 +15,32 @@ import (
 // Run produces slice of tags ordered by frequency.
 func Run(ctx context.Context, options ...Option) (*model.Result, error) {
 
-	c := config.New(options...)
+	cfg := config.New(options...)
 
 	var in in
 	var err error
 
-	if c.Content != "" {
-		in = newInFromString(c.Content, c.ContentType)
+	if cfg.Content != "" {
+		in = newInFromString(cfg.Content, cfg.ContentType)
 	} else {
-		in, err = newIn(ctx, c)
-		if c.ContentType > Unknown {
-			in.ContentType = c.ContentType
+		in, err = newIn(ctx, cfg)
+		if cfg.ContentType > Unknown {
+			in.ContentType = cfg.ContentType
 		}
 	}
 
 	if err != nil {
 		return nil, err
 	}
 
-	res := processInput(&in, c)
+	res := processInput(&in, cfg)
 
 	if len(res.RawTags) > 0 {
-		if c.Verbose {
+		if cfg.Verbose {
 			fmt.Println("tagifying...")
 		}
-		res.Tags = processor.Run(c, res.Flatten())
-		if c.Verbose {
+		res.Tags = processor.Run(cfg, res.Flatten())
+		if cfg.Verbose {
 			fmt.Printf("\n%v\n", res.Tags)
 		}
 	}