From 8a9b629df3655cfa73299ae171df0d9090bf8d60 Mon Sep 17 00:00:00 2001 From: Sergey Date: Sat, 30 Mar 2024 14:58:57 +1100 Subject: [PATCH] Bumped the github.com/zoomio/inout to the latest commit hash (#27) * Bumped the github.com/zoomio/inout to the latest commit hash * Introduced Timeout option to serve as an overall deadline for the operation * Bumped inout, allow to pass user agent * gitignore _dist * Removed unknown "toolcahin" derective from go.mod * Version bump + CHANGELOG --- .gitignore | 2 ++ CHANGELOG.md | 6 ++++++ Makefile | 2 +- cmd/cli/cli.go | 4 ++++ config/config.go | 3 +++ config/options.go | 14 ++++++++++++++ go.mod | 16 ++++++++-------- go.sum | 38 +++++++++++++++++++------------------- in.go | 24 +++++++++++++----------- model.go | 3 +++ tagify.go | 20 ++++++++++---------- 11 files changed, 83 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index 0afa26c..5a7c319 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ +/tagify /tagify_darwin* /tagify_linux* /cli vendor/ +/_dist/ *.out *.test *.cpu diff --git a/CHANGELOG.md b/CHANGELOG.md index cd48399..0b7c980 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v0.62.0 + +- bumped Go to 1.22; +- bumped `github.com/zoomio/inout` to v0.14.0; +- introduced `UserAgent` (`-ua` in CLI mode) to allow to pass a custom user agent for headless HTTP calls. + ## v0.61.0 - bumped Go to 1.20; diff --git a/Makefile b/Makefile index 705b77e..68ec5fb 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: deps clean build -TAG=0.61.0 +TAG=0.62.0 BINARY=tagify DIST_DIR=_dist OS=darwin diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 4b983f2..909d775 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -25,6 +25,7 @@ var ( ready = flag.String("r", "", "DOM CSS query, waits until certain element available, but fetches contents of the whole HTML document") until = flag.Duration("u", 0, "duration to wait before getting HTML contents, handy for SPAs, because they keep loading in browsers for some time") img = flag.String("i", "", "enables capturing screenshot in the provided path") + ua = flag.String("ua", "", "provide a custom user agent for headless HTTP calls") limit = flag.Int("l", 5, "number of tags to return") verbose = flag.Bool("v", false, "enables verbose mode") @@ -93,6 +94,9 @@ func main() { if len(*img) > 0 { options = append(options, tagify.Screenshot(true)) } + if *ua != "" { + options = append(options, tagify.UserAgent(*ua)) + } if *verbose { options = append(options, tagify.Verbose(*verbose)) diff --git a/config/config.go b/config/config.go index 7f46635..81c53e5 100644 --- a/config/config.go +++ b/config/config.go @@ -45,11 +45,14 @@ type Config struct { ContentType Content string + Timeout time.Duration + // headless Query string WaitFor string WaitUntil time.Duration Screenshot bool + UserAgent string // misc Limit int diff --git a/config/options.go b/config/options.go index e866df0..b0b0cd0 100644 --- a/config/options.go +++ b/config/options.go @@ -36,6 +36,13 @@ var ( } } + // Timeout sets the overall deadline for the operation. + Timeout = func(d time.Duration) Option { + return func(c *Config) { + c.Timeout = d + } + } + // WaitFor sets CSS query for the target of In-Out. WaitFor = func(query string) Option { return func(c *Config) { @@ -57,6 +64,13 @@ var ( } } + // UserAgent allows to specify custom User Agent of the HTTP headless calls. + UserAgent = func(ua string) Option { + return func(c *Config) { + c.UserAgent = ua + } + } + // Content sets content of the target. Content = func(v string) Option { return func(c *Config) { diff --git a/go.mod b/go.mod index 6b264d2..f592c06 100644 --- a/go.mod +++ b/go.mod @@ -4,29 +4,29 @@ require ( github.com/abadojack/whatlanggo v1.0.1 github.com/go-ego/gse v0.70.2 github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a - github.com/stretchr/testify v1.7.0 - github.com/zoomio/inout v0.12.0 + github.com/stretchr/testify v1.8.4 + github.com/zoomio/inout v0.14.0 github.com/zoomio/stopwords v0.11.0 golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d ) require ( - github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c // indirect - github.com/chromedp/chromedp v0.8.3 // indirect + github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89 // indirect + github.com/chromedp/chromedp v0.9.2 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect - github.com/gobwas/ws v1.1.0 // indirect + github.com/gobwas/ws v1.2.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/vcaesar/cedar v0.20.1 // indirect - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + golang.org/x/sys v0.6.0 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect - gopkg.in/yaml.v3 v3.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) -go 1.20 +go 1.22 diff --git a/go.sum b/go.sum index ef1f6f7..2a61862 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,12 @@ github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4= github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc= -github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c h1:Gm+DujZPVAtQNTLhbg5PExjRNfhdTCSMLvJ/pFfY4aY= -github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0= -github.com/chromedp/chromedp v0.8.3 h1:UwOY+fhC5Vv3uKgRpnvilCbWs/QPz8ciFwRB0q6pH8k= -github.com/chromedp/chromedp v0.8.3/go.mod h1:9YfKSJnBNeP77vKecv+DNx2/Tcb+6Gli0d1aZPw/xbk= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89 h1:aPflPkRFkVwbW6dmcVqfgwp1i+UWGFH6VgR1Jim5Ygc= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2 h1:dKtNz4kApb06KuSXoTQIyUC2TrA0fhGDwNZf3bcgfKw= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-ego/gse v0.70.2 h1:y2UMOHJMtI+0b2GjxTtQfKON5DMmlyX1hOQHTo8UVVs= @@ -16,8 +15,8 @@ github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= -github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= +github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a h1:eeaG9XMUvRBYXJi4pg1ZKM7nxc5AfXfojeLLW7O5J3k= github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -26,31 +25,32 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5 h1:1SoBaSPudixRecmlHXb/GxmaD3fLMtHIDN13QujwQuc= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/vcaesar/cedar v0.20.1 h1:cDOmYWdprO7ZW8cngJrDi8Zivnscj9dA/y8Y+2SB1P0= github.com/vcaesar/cedar v0.20.1/go.mod h1:iMDweyuW76RvSrCkQeZeQk4iCbshiPzcCvcGCtpM7iI= github.com/vcaesar/tt v0.20.0 h1:9t2Ycb9RNHcP0WgQgIaRKJBB+FrRdejuaL6uWIHuoBA= -github.com/zoomio/inout v0.12.0 h1:B+LcJFSdSn7Vgt4dQim8c8bDBBA3wAbl9EVV+B9Pr2Y= -github.com/zoomio/inout v0.12.0/go.mod h1:QGuD4z2/PEqRwlqCLG3H5Ye35zzQg4kj1rr7By7Gxco= +github.com/vcaesar/tt v0.20.0/go.mod h1:GHPxQYhn+7OgKakRusH7KJ0M5MhywoeLb8Fcffs/Gtg= +github.com/zoomio/inout v0.14.0 h1:Q/9EJudnSM71CuwqWsfr7TqNhhsPBaCT+gxh/57ozi4= +github.com/zoomio/inout v0.14.0/go.mod h1:JFLyhAGsULpcarxp0hbLQEB1PGfkyRv1Y5bbiZ3vqcc= github.com/zoomio/stopwords v0.11.0 h1:O2JY3zFymge1u/9FkEjG7PtloJMiqWH+P6NKtTDKvLY= github.com/zoomio/stopwords v0.11.0/go.mod h1:G4EP6eLrE9A9HhG0JcOVcv9UdPpjthtqaA5MpgkKmoc= golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d h1:62NvYBuaanGXR2ZOfwDFkhhl6X1DUgf8qg3GuQvxZsE= golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/in.go b/in.go index b73234e..4e0073b 100644 --- a/in.go +++ b/in.go @@ -20,23 +20,25 @@ type in struct { // // source - the filename or web page source, reads from STDIN if source is empty. // Panics on errors. -func newIn(ctx context.Context, c *Config) (in, error) { - in := in{source: c.Source} +func newIn(ctx context.Context, cfg *Config) (in, error) { + in := in{source: cfg.Source} - if strings.HasPrefix(c.Source, "http://") || strings.HasPrefix(c.Source, "https://") || c.Query != "" { + if strings.HasPrefix(cfg.Source, "http://") || strings.HasPrefix(cfg.Source, "https://") || cfg.Query != "" { in.ContentType = HTML - } else if strings.ToLower(filepath.Ext(c.Source)) == ".md" { + } else if strings.ToLower(filepath.Ext(cfg.Source)) == ".md" { in.ContentType = Markdown } r, err := inout.NewInOut(ctx, - inout.Source(c.Source), - inout.Query(c.Query), - inout.WaitFor(c.WaitFor), - inout.WaitUntil(c.WaitUntil), - inout.Screenshot(c.Screenshot), - inout.Timeout(0), - inout.Verbose(c.Verbose)) + inout.Source(cfg.Source), + inout.Query(cfg.Query), + inout.WaitFor(cfg.WaitFor), + inout.WaitUntil(cfg.WaitUntil), + inout.Screenshot(cfg.Screenshot), + inout.Timeout(cfg.Timeout), + inout.Verbose(cfg.Verbose), + inout.UserAgent(cfg.UserAgent), + ) if err != nil { return in, err } diff --git a/model.go b/model.go index 1a4fca4..343b0e4 100644 --- a/model.go +++ b/model.go @@ -14,11 +14,14 @@ var ( Language = config.Language Content = config.Content + Timeout = config.Timeout + // headless Query = config.Query WaitFor = config.WaitFor WaitUntil = config.WaitUntil Screenshot = config.Screenshot + UserAgent = config.UserAgent // misc TargetType = config.TargetType diff --git a/tagify.go b/tagify.go index 1c823d9..a3a1899 100644 --- a/tagify.go +++ b/tagify.go @@ -15,17 +15,17 @@ import ( // Run produces slice of tags ordered by frequency. func Run(ctx context.Context, options ...Option) (*model.Result, error) { - c := config.New(options...) + cfg := config.New(options...) var in in var err error - if c.Content != "" { - in = newInFromString(c.Content, c.ContentType) + if cfg.Content != "" { + in = newInFromString(cfg.Content, cfg.ContentType) } else { - in, err = newIn(ctx, c) - if c.ContentType > Unknown { - in.ContentType = c.ContentType + in, err = newIn(ctx, cfg) + if cfg.ContentType > Unknown { + in.ContentType = cfg.ContentType } } @@ -33,14 +33,14 @@ func Run(ctx context.Context, options ...Option) (*model.Result, error) { return nil, err } - res := processInput(&in, c) + res := processInput(&in, cfg) if len(res.RawTags) > 0 { - if c.Verbose { + if cfg.Verbose { fmt.Println("tagifying...") } - res.Tags = processor.Run(c, res.Flatten()) - if c.Verbose { + res.Tags = processor.Run(cfg, res.Flatten()) + if cfg.Verbose { fmt.Printf("\n%v\n", res.Tags) } }