From 03251e311bc6e3d9e750f1554118b9dd7c5d6661 Mon Sep 17 00:00:00 2001 From: tobi Date: Sun, 15 Sep 2024 16:12:57 +0200 Subject: [PATCH 01/11] [bugfix] Use better plaintext representation of status for filtering --- .drone.yml | 4 +- .goreleaser.yml | 1 + go.mod | 2 + go.sum | 8 + internal/typeutils/converter.go | 29 +- internal/typeutils/internaltofrontend.go | 58 +- internal/typeutils/util.go | 77 + internal/typeutils/util_test.go | 87 + scripts/build.sh | 3 +- vendor/github.com/cespare/xxhash/LICENSE.txt | 22 + vendor/github.com/cespare/xxhash/README.md | 50 + vendor/github.com/cespare/xxhash/rotate.go | 14 + vendor/github.com/cespare/xxhash/rotate19.go | 14 + vendor/github.com/cespare/xxhash/xxhash.go | 168 ++ .../github.com/cespare/xxhash/xxhash_amd64.go | 12 + .../github.com/cespare/xxhash/xxhash_amd64.s | 233 ++ .../github.com/cespare/xxhash/xxhash_other.go | 75 + .../github.com/cespare/xxhash/xxhash_safe.go | 10 + .../cespare/xxhash/xxhash_unsafe.go | 30 + vendor/github.com/k3a/html2text/.travis.yml | 10 + vendor/github.com/k3a/html2text/LICENSE | 21 + vendor/github.com/k3a/html2text/README.md | 60 + vendor/github.com/k3a/html2text/entity.go | 2046 +++++++++++++++++ vendor/github.com/k3a/html2text/html2text.go | 333 +++ vendor/modules.txt | 6 + 25 files changed, 3316 insertions(+), 57 deletions(-) create mode 100644 vendor/github.com/cespare/xxhash/LICENSE.txt create mode 100644 vendor/github.com/cespare/xxhash/README.md create mode 100644 vendor/github.com/cespare/xxhash/rotate.go create mode 100644 vendor/github.com/cespare/xxhash/rotate19.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash_amd64.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash_amd64.s create mode 100644 vendor/github.com/cespare/xxhash/xxhash_other.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash_safe.go create mode 100644 vendor/github.com/cespare/xxhash/xxhash_unsafe.go create mode 100644 vendor/github.com/k3a/html2text/.travis.yml create mode 100644 vendor/github.com/k3a/html2text/LICENSE create mode 100644 vendor/github.com/k3a/html2text/README.md create mode 100644 vendor/github.com/k3a/html2text/entity.go create mode 100644 vendor/github.com/k3a/html2text/html2text.go diff --git a/.drone.yml b/.drone.yml index 1a5239a13e..ea945db12f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -45,7 +45,7 @@ steps: go test -failfast -timeout=20m - -tags "netgo osusergo static_build kvformat timetzdata" + -tags "netgo osusergo static_build kvformat timetzdata purego" ./... - ./test/envparsing.sh - ./test/swagger.sh @@ -207,6 +207,6 @@ steps: --- kind: signature -hmac: f4008d87e4e5b67251eb89f255c1224e6ab5818828cab24fc319b8f829176058 +hmac: 3f3a24557b67760dd0c4091eaaed4842b0545f5aa65f90ce70d5e45da23c5260 ... diff --git a/.goreleaser.yml b/.goreleaser.yml index 6a7fccfd09..3d1bedd11e 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -27,6 +27,7 @@ builds: - static_build - kvformat - timetzdata + - purego - >- {{ if and (index .Env "DEBUG") (.Env.DEBUG) }}debugenv{{ end }} - >- diff --git a/go.mod b/go.mod index 624486a416..8d23218d62 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/DmitriyVTitov/size v1.5.0 github.com/KimMachineGun/automemlimit v0.6.1 github.com/buckket/go-blurhash v1.1.0 + github.com/cespare/xxhash v1.1.0 github.com/coreos/go-oidc/v3 v3.11.0 github.com/gin-contrib/cors v1.7.2 github.com/gin-contrib/gzip v1.0.1 @@ -40,6 +41,7 @@ require ( github.com/gorilla/feeds v1.2.0 github.com/gorilla/websocket v1.5.2 github.com/jackc/pgx/v5 v5.6.0 + github.com/k3a/html2text v1.2.1 github.com/microcosm-cc/bluemonday v1.0.27 github.com/miekg/dns v1.1.62 github.com/minio/minio-go/v7 v7.0.76 diff --git a/go.sum b/go.sum index d0826e1b9e..db3fc7812c 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -118,6 +120,8 @@ github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -384,6 +388,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k3a/html2text v1.2.1 h1:nvnKgBvBR/myqrwfLuiqecUtaK1lB9hGziIJKatNFVY= +github.com/k3a/html2text v1.2.1/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -506,6 +512,8 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 311839dc0a..a4395163da 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -18,26 +18,37 @@ package typeutils import ( + "log" "sync" + "time" + "codeberg.org/gruf/go-cache/v3" "github.com/superseriousbusiness/gotosocial/internal/filter/interaction" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/state" ) type Converter struct { - state *state.State - defaultAvatars []string - randAvatars sync.Map - visFilter *visibility.Filter - intFilter *interaction.Filter + state *state.State + defaultAvatars []string + randAvatars sync.Map + visFilter *visibility.Filter + intFilter *interaction.Filter + statusHashesToFilterableText cache.TTLCache[string, string] } func NewConverter(state *state.State) *Converter { + statusHashesToFilterableText := cache.NewTTL[string, string](0, 512, 0) + statusHashesToFilterableText.SetTTL(time.Hour, true) + if !statusHashesToFilterableText.Start(time.Minute) { + log.Panic(nil, "failed to start statusHashesToFilterableText cache") + } + return &Converter{ - state: state, - defaultAvatars: populateDefaultAvatars(), - visFilter: visibility.NewFilter(state), - intFilter: interaction.NewFilter(state), + state: state, + defaultAvatars: populateDefaultAvatars(), + visFilter: visibility.NewFilter(state), + intFilter: interaction.NewFilter(state), + statusHashesToFilterableText: statusHashesToFilterableText, } } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 55af2c1f10..261d71bb4d 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -35,7 +35,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/language" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" - "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/uris" "github.com/superseriousbusiness/gotosocial/internal/util" ) @@ -939,8 +938,18 @@ func (c *Converter) statusToAPIFilterResults( return nil, nil } - // Extract text fields from the status that we will match filters against. - fields := filterableTextFields(s) + // Derive a hash of this status. + statusHash := StatusHash(s) + + // Check if we have the filterable + // text stored already for this hash. + statusText, stored := c.statusHashesToFilterableText.Get(statusHash) + if !stored { + // We don't have this filterable text + // cached, calculate + cache it now. + statusText = filterableText(s) + c.statusHashesToFilterableText.Set(statusHash, statusText) + } // Record all matching warn filters and the reasons they matched. filterResults := make([]apimodel.FilterResult, 0, len(filters)) @@ -956,14 +965,7 @@ func (c *Converter) statusToAPIFilterResults( // List all matching keywords. keywordMatches := make([]string, 0, len(filter.Keywords)) for _, filterKeyword := range filter.Keywords { - var isMatch bool - for _, field := range fields { - if filterKeyword.Regexp.MatchString(field) { - isMatch = true - break - } - } - if isMatch { + if filterKeyword.Regexp.MatchString(statusText) { keywordMatches = append(keywordMatches, filterKeyword.Keyword) } } @@ -1001,40 +1003,6 @@ func (c *Converter) statusToAPIFilterResults( return filterResults, nil } -// filterableTextFields returns all text from a status that we might want to filter on: -// - content -// - content warning -// - media descriptions -// - poll options -func filterableTextFields(s *gtsmodel.Status) []string { - fieldCount := 2 + len(s.Attachments) - if s.Poll != nil { - fieldCount += len(s.Poll.Options) - } - fields := make([]string, 0, fieldCount) - - if s.Content != "" { - fields = append(fields, text.SanitizeToPlaintext(s.Content)) - } - if s.ContentWarning != "" { - fields = append(fields, s.ContentWarning) - } - for _, attachment := range s.Attachments { - if attachment.Description != "" { - fields = append(fields, attachment.Description) - } - } - if s.Poll != nil { - for _, option := range s.Poll.Options { - if option != "" { - fields = append(fields, option) - } - } - } - - return fields -} - // filterAppliesInContext returns whether a given filter applies in a given context. func filterAppliesInContext(filter *gtsmodel.Filter, filterContext statusfilter.FilterContext) bool { switch filterContext { diff --git a/internal/typeutils/util.go b/internal/typeutils/util.go index 3441e89a98..f660efc842 100644 --- a/internal/typeutils/util.go +++ b/internal/typeutils/util.go @@ -19,6 +19,7 @@ package typeutils import ( "context" + "encoding/hex" "fmt" "math" "net/url" @@ -27,6 +28,8 @@ import ( "strconv" "strings" + "github.com/cespare/xxhash/v2" + "github.com/k3a/html2text" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -284,3 +287,77 @@ func ContentToContentLanguage( return contentStr, langTagStr } + +// StatusHash returns an xxhash of text +// from a status, taking account of: +// +// - content warning +// - content +// - media IDs + descriptions +// - poll options +func StatusHash(s *gtsmodel.Status) string { + hash := xxhash.New() + + // Content warning / title. + hash.WriteString(s.ContentWarning) + + // Status content. + hash.WriteString(s.Content) + + // Media IDs + descriptions. + for _, attachment := range s.Attachments { + hash.WriteString(attachment.ID) + hash.WriteString(attachment.Description) + } + + // Poll options. + if s.Poll != nil { + for _, option := range s.Poll.Options { + hash.WriteString(option) + } + } + + sum := hash.Sum(nil) + return hex.EncodeToString(sum) +} + +// filterableText concatenates text from a +// status that we might want to filter on: +// +// - content warning +// - content (converted to plaintext from HTML) +// - media descriptions +// - poll options +func filterableText(s *gtsmodel.Status) string { + fields := []string{} + + // Content warning / title. + fields = append(fields, s.ContentWarning) + + // Status content; use raw text if available, + // else use text parsed from content HTML. + if s.Text != "" { + fields = append(fields, s.Text) + } else { + text := html2text.HTML2TextWithOptions( + s.Content, + html2text.WithLinksInnerText(), + html2text.WithUnixLineBreaks(), + ) + fields = append(fields, text) + } + + // Media descriptions. + for _, attachment := range s.Attachments { + fields = append(fields, attachment.Description) + } + + // Poll options. + if s.Poll != nil { + for _, option := range s.Poll.Options { + fields = append(fields, option) + } + } + + return strings.Join(fields, " ") +} diff --git a/internal/typeutils/util_test.go b/internal/typeutils/util_test.go index 0f852d399a..23be0bbe6b 100644 --- a/internal/typeutils/util_test.go +++ b/internal/typeutils/util_test.go @@ -158,3 +158,90 @@ func TestContentToContentLanguage(t *testing.T) { } } } + +func TestFilterableText(t *testing.T) { + type testcase struct { + status *gtsmodel.Status + expectedText string + } + + for i, testcase := range []testcase{ + { + status: >smodel.Status{ + ContentWarning: "This is a test status", + Content: `

Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.

`, + }, + expectedText: `This is a test status Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.`, + }, + { + status: >smodel.Status{ + Content: `

@zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)

https://github.com/superseriousbusiness/gotosocial/pull/2863

`, + }, + expectedText: ` @zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag) + +https://github.com/superseriousbusiness/gotosocial/pull/2863 `, + }, + { + status: >smodel.Status{ + Content: `

Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg, both running on Wazero, and configured with a 50MiB db cache target. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.

`, + Attachments: []*gtsmodel.MediaAttachment{ + { + Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`, + }, + }, + }, + expectedText: ` Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg , both running on Wazero , and configured with a 50MiB db cache target . This is the version we'll be releasing soonish, now we're happy with how we've tamed everything. Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`, + }, + } { + text := filterableText(testcase.status) + if text != testcase.expectedText { + t.Errorf( + "test %d expected text '%s' got '%s'", + i, testcase.expectedText, text, + ) + } + } +} + +func TestStatusHash(t *testing.T) { + type testcase struct { + status *gtsmodel.Status + expectedHash string + } + + for i, testcase := range []testcase{ + { + status: >smodel.Status{ + ContentWarning: "This is a test status", + Content: `

Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.

`, + }, + expectedHash: `8bbb5439dbe62ae0`, + }, + { + status: >smodel.Status{ + Content: `

@zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)

https://github.com/superseriousbusiness/gotosocial/pull/2863

`, + }, + expectedHash: `d039dfb4d04752d5`, + }, + { + status: >smodel.Status{ + Content: `

Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg, both running on Wazero, and configured with a 50MiB db cache target. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.

`, + Attachments: []*gtsmodel.MediaAttachment{ + { + ID: "01J7TYSH1V5V4DCTVPASH3K9PQ", + Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`, + }, + }, + }, + expectedHash: `414d975b2ef9d112`, + }, + } { + hash := StatusHash(testcase.status) + if hash != testcase.expectedHash { + t.Errorf( + "test %d expected hash '%s' got '%s'", + i, testcase.expectedHash, hash, + ) + } + } +} diff --git a/scripts/build.sh b/scripts/build.sh index 5b10a5493b..1781731e3c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -6,7 +6,7 @@ set -e log_exec() { echo "$ ${*}"; "$@"; } # Grab environment variables and set defaults + requirements. -GO_BUILDTAGS="${GO_BUILDTAGS-} netgo osusergo static_build kvformat timetzdata" +GO_BUILDTAGS="${GO_BUILDTAGS-} netgo osusergo static_build kvformat timetzdata purego" GO_LDFLAGS="${GO_LDFLAGS-} -s -w -extldflags '-static' -X 'main.Version=${VERSION:-$(git describe --tags --abbrev=0)}'" GO_GCFLAGS=${GO_GCFLAGS-} @@ -17,6 +17,7 @@ GO_GCFLAGS=${GO_GCFLAGS-} # Available Go build tags, with explanation, followed by benefits of enabling it: # - kvformat: enables prettier output of log fields (slightly better performance) # - timetzdata: embed timezone database inside binary (allow setting local time inside Docker containers, at cost of 450KB) +# - purego: disable amd64/arm64 assembly implementation for xxhash (increase portability at marginal performance cost) # - notracing: disables compiling-in otel tracing support (reduced binary size, better performance) # - nometrics: disables compiling-in otel metrics support (reduced binary size, better performance) # - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability) diff --git a/vendor/github.com/cespare/xxhash/LICENSE.txt b/vendor/github.com/cespare/xxhash/LICENSE.txt new file mode 100644 index 0000000000..24b53065f4 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/cespare/xxhash/README.md b/vendor/github.com/cespare/xxhash/README.md new file mode 100644 index 0000000000..0982fd25e5 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/README.md @@ -0,0 +1,50 @@ +# xxhash + +[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash) + +xxhash is a Go implementation of the 64-bit +[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a +high-quality hashing algorithm that is much faster than anything in the Go +standard library. + +The API is very small, taking its cue from the other hashing packages in the +standard library: + + $ go doc github.com/cespare/xxhash ! + package xxhash // import "github.com/cespare/xxhash" + + Package xxhash implements the 64-bit variant of xxHash (XXH64) as described + at http://cyan4973.github.io/xxHash/. + + func New() hash.Hash64 + func Sum64(b []byte) uint64 + func Sum64String(s string) uint64 + +This implementation provides a fast pure-Go implementation and an even faster +assembly implementation for amd64. + +## Benchmarks + +Here are some quick benchmarks comparing the pure-Go and assembly +implementations of Sum64 against another popular Go XXH64 implementation, +[github.com/OneOfOne/xxhash](https://github.com/OneOfOne/xxhash): + +| input size | OneOfOne | cespare (purego) | cespare | +| --- | --- | --- | --- | +| 5 B | 416 MB/s | 720 MB/s | 872 MB/s | +| 100 B | 3980 MB/s | 5013 MB/s | 5252 MB/s | +| 4 KB | 12727 MB/s | 12999 MB/s | 13026 MB/s | +| 10 MB | 9879 MB/s | 10775 MB/s | 10913 MB/s | + +These numbers were generated with: + +``` +$ go test -benchtime 10s -bench '/OneOfOne,' +$ go test -tags purego -benchtime 10s -bench '/xxhash,' +$ go test -benchtime 10s -bench '/xxhash,' +``` + +## Projects using this package + +- [InfluxDB](https://github.com/influxdata/influxdb) +- [Prometheus](https://github.com/prometheus/prometheus) diff --git a/vendor/github.com/cespare/xxhash/rotate.go b/vendor/github.com/cespare/xxhash/rotate.go new file mode 100644 index 0000000000..f3eac5ebc0 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/rotate.go @@ -0,0 +1,14 @@ +// +build !go1.9 + +package xxhash + +// TODO(caleb): After Go 1.10 comes out, remove this fallback code. + +func rol1(x uint64) uint64 { return (x << 1) | (x >> (64 - 1)) } +func rol7(x uint64) uint64 { return (x << 7) | (x >> (64 - 7)) } +func rol11(x uint64) uint64 { return (x << 11) | (x >> (64 - 11)) } +func rol12(x uint64) uint64 { return (x << 12) | (x >> (64 - 12)) } +func rol18(x uint64) uint64 { return (x << 18) | (x >> (64 - 18)) } +func rol23(x uint64) uint64 { return (x << 23) | (x >> (64 - 23)) } +func rol27(x uint64) uint64 { return (x << 27) | (x >> (64 - 27)) } +func rol31(x uint64) uint64 { return (x << 31) | (x >> (64 - 31)) } diff --git a/vendor/github.com/cespare/xxhash/rotate19.go b/vendor/github.com/cespare/xxhash/rotate19.go new file mode 100644 index 0000000000..b99612bab8 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/rotate19.go @@ -0,0 +1,14 @@ +// +build go1.9 + +package xxhash + +import "math/bits" + +func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) } +func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) } +func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) } +func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) } +func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) } +func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) } +func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) } +func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) } diff --git a/vendor/github.com/cespare/xxhash/xxhash.go b/vendor/github.com/cespare/xxhash/xxhash.go new file mode 100644 index 0000000000..f896bd28f0 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash.go @@ -0,0 +1,168 @@ +// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described +// at http://cyan4973.github.io/xxHash/. +package xxhash + +import ( + "encoding/binary" + "hash" +) + +const ( + prime1 uint64 = 11400714785074694791 + prime2 uint64 = 14029467366897019727 + prime3 uint64 = 1609587929392839161 + prime4 uint64 = 9650029242287828579 + prime5 uint64 = 2870177450012600261 +) + +// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where +// possible in the Go code is worth a small (but measurable) performance boost +// by avoiding some MOVQs. Vars are needed for the asm and also are useful for +// convenience in the Go code in a few places where we need to intentionally +// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the +// result overflows a uint64). +var ( + prime1v = prime1 + prime2v = prime2 + prime3v = prime3 + prime4v = prime4 + prime5v = prime5 +) + +type xxh struct { + v1 uint64 + v2 uint64 + v3 uint64 + v4 uint64 + total int + mem [32]byte + n int // how much of mem is used +} + +// New creates a new hash.Hash64 that implements the 64-bit xxHash algorithm. +func New() hash.Hash64 { + var x xxh + x.Reset() + return &x +} + +func (x *xxh) Reset() { + x.n = 0 + x.total = 0 + x.v1 = prime1v + prime2 + x.v2 = prime2 + x.v3 = 0 + x.v4 = -prime1v +} + +func (x *xxh) Size() int { return 8 } +func (x *xxh) BlockSize() int { return 32 } + +// Write adds more data to x. It always returns len(b), nil. +func (x *xxh) Write(b []byte) (n int, err error) { + n = len(b) + x.total += len(b) + + if x.n+len(b) < 32 { + // This new data doesn't even fill the current block. + copy(x.mem[x.n:], b) + x.n += len(b) + return + } + + if x.n > 0 { + // Finish off the partial block. + copy(x.mem[x.n:], b) + x.v1 = round(x.v1, u64(x.mem[0:8])) + x.v2 = round(x.v2, u64(x.mem[8:16])) + x.v3 = round(x.v3, u64(x.mem[16:24])) + x.v4 = round(x.v4, u64(x.mem[24:32])) + b = b[32-x.n:] + x.n = 0 + } + + if len(b) >= 32 { + // One or more full blocks left. + b = writeBlocks(x, b) + } + + // Store any remaining partial block. + copy(x.mem[:], b) + x.n = len(b) + + return +} + +func (x *xxh) Sum(b []byte) []byte { + s := x.Sum64() + return append( + b, + byte(s>>56), + byte(s>>48), + byte(s>>40), + byte(s>>32), + byte(s>>24), + byte(s>>16), + byte(s>>8), + byte(s), + ) +} + +func (x *xxh) Sum64() uint64 { + var h uint64 + + if x.total >= 32 { + v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4 + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + h = mergeRound(h, v1) + h = mergeRound(h, v2) + h = mergeRound(h, v3) + h = mergeRound(h, v4) + } else { + h = x.v3 + prime5 + } + + h += uint64(x.total) + + i, end := 0, x.n + for ; i+8 <= end; i += 8 { + k1 := round(0, u64(x.mem[i:i+8])) + h ^= k1 + h = rol27(h)*prime1 + prime4 + } + if i+4 <= end { + h ^= uint64(u32(x.mem[i:i+4])) * prime1 + h = rol23(h)*prime2 + prime3 + i += 4 + } + for i < end { + h ^= uint64(x.mem[i]) * prime5 + h = rol11(h) * prime1 + i++ + } + + h ^= h >> 33 + h *= prime2 + h ^= h >> 29 + h *= prime3 + h ^= h >> 32 + + return h +} + +func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) } +func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) } + +func round(acc, input uint64) uint64 { + acc += input * prime2 + acc = rol31(acc) + acc *= prime1 + return acc +} + +func mergeRound(acc, val uint64) uint64 { + val = round(0, val) + acc ^= val + acc = acc*prime1 + prime4 + return acc +} diff --git a/vendor/github.com/cespare/xxhash/xxhash_amd64.go b/vendor/github.com/cespare/xxhash/xxhash_amd64.go new file mode 100644 index 0000000000..d617652680 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_amd64.go @@ -0,0 +1,12 @@ +// +build !appengine +// +build gc +// +build !purego + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +// +//go:noescape +func Sum64(b []byte) uint64 + +func writeBlocks(x *xxh, b []byte) []byte diff --git a/vendor/github.com/cespare/xxhash/xxhash_amd64.s b/vendor/github.com/cespare/xxhash/xxhash_amd64.s new file mode 100644 index 0000000000..757f2011f0 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_amd64.s @@ -0,0 +1,233 @@ +// +build !appengine +// +build gc +// +build !purego + +#include "textflag.h" + +// Register allocation: +// AX h +// CX pointer to advance through b +// DX n +// BX loop end +// R8 v1, k1 +// R9 v2 +// R10 v3 +// R11 v4 +// R12 tmp +// R13 prime1v +// R14 prime2v +// R15 prime4v + +// round reads from and advances the buffer pointer in CX. +// It assumes that R13 has prime1v and R14 has prime2v. +#define round(r) \ + MOVQ (CX), R12 \ + ADDQ $8, CX \ + IMULQ R14, R12 \ + ADDQ R12, r \ + ROLQ $31, r \ + IMULQ R13, r + +// mergeRound applies a merge round on the two registers acc and val. +// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v. +#define mergeRound(acc, val) \ + IMULQ R14, val \ + ROLQ $31, val \ + IMULQ R13, val \ + XORQ val, acc \ + IMULQ R13, acc \ + ADDQ R15, acc + +// func Sum64(b []byte) uint64 +TEXT ·Sum64(SB), NOSPLIT, $0-32 + // Load fixed primes. + MOVQ ·prime1v(SB), R13 + MOVQ ·prime2v(SB), R14 + MOVQ ·prime4v(SB), R15 + + // Load slice. + MOVQ b_base+0(FP), CX + MOVQ b_len+8(FP), DX + LEAQ (CX)(DX*1), BX + + // The first loop limit will be len(b)-32. + SUBQ $32, BX + + // Check whether we have at least one block. + CMPQ DX, $32 + JLT noBlocks + + // Set up initial state (v1, v2, v3, v4). + MOVQ R13, R8 + ADDQ R14, R8 + MOVQ R14, R9 + XORQ R10, R10 + XORQ R11, R11 + SUBQ R13, R11 + + // Loop until CX > BX. +blockLoop: + round(R8) + round(R9) + round(R10) + round(R11) + + CMPQ CX, BX + JLE blockLoop + + MOVQ R8, AX + ROLQ $1, AX + MOVQ R9, R12 + ROLQ $7, R12 + ADDQ R12, AX + MOVQ R10, R12 + ROLQ $12, R12 + ADDQ R12, AX + MOVQ R11, R12 + ROLQ $18, R12 + ADDQ R12, AX + + mergeRound(AX, R8) + mergeRound(AX, R9) + mergeRound(AX, R10) + mergeRound(AX, R11) + + JMP afterBlocks + +noBlocks: + MOVQ ·prime5v(SB), AX + +afterBlocks: + ADDQ DX, AX + + // Right now BX has len(b)-32, and we want to loop until CX > len(b)-8. + ADDQ $24, BX + + CMPQ CX, BX + JG fourByte + +wordLoop: + // Calculate k1. + MOVQ (CX), R8 + ADDQ $8, CX + IMULQ R14, R8 + ROLQ $31, R8 + IMULQ R13, R8 + + XORQ R8, AX + ROLQ $27, AX + IMULQ R13, AX + ADDQ R15, AX + + CMPQ CX, BX + JLE wordLoop + +fourByte: + ADDQ $4, BX + CMPQ CX, BX + JG singles + + MOVL (CX), R8 + ADDQ $4, CX + IMULQ R13, R8 + XORQ R8, AX + + ROLQ $23, AX + IMULQ R14, AX + ADDQ ·prime3v(SB), AX + +singles: + ADDQ $4, BX + CMPQ CX, BX + JGE finalize + +singlesLoop: + MOVBQZX (CX), R12 + ADDQ $1, CX + IMULQ ·prime5v(SB), R12 + XORQ R12, AX + + ROLQ $11, AX + IMULQ R13, AX + + CMPQ CX, BX + JL singlesLoop + +finalize: + MOVQ AX, R12 + SHRQ $33, R12 + XORQ R12, AX + IMULQ R14, AX + MOVQ AX, R12 + SHRQ $29, R12 + XORQ R12, AX + IMULQ ·prime3v(SB), AX + MOVQ AX, R12 + SHRQ $32, R12 + XORQ R12, AX + + MOVQ AX, ret+24(FP) + RET + +// writeBlocks uses the same registers as above except that it uses AX to store +// the x pointer. + +// func writeBlocks(x *xxh, b []byte) []byte +TEXT ·writeBlocks(SB), NOSPLIT, $0-56 + // Load fixed primes needed for round. + MOVQ ·prime1v(SB), R13 + MOVQ ·prime2v(SB), R14 + + // Load slice. + MOVQ b_base+8(FP), CX + MOVQ CX, ret_base+32(FP) // initialize return base pointer; see NOTE below + MOVQ b_len+16(FP), DX + LEAQ (CX)(DX*1), BX + SUBQ $32, BX + + // Load vN from x. + MOVQ x+0(FP), AX + MOVQ 0(AX), R8 // v1 + MOVQ 8(AX), R9 // v2 + MOVQ 16(AX), R10 // v3 + MOVQ 24(AX), R11 // v4 + + // We don't need to check the loop condition here; this function is + // always called with at least one block of data to process. +blockLoop: + round(R8) + round(R9) + round(R10) + round(R11) + + CMPQ CX, BX + JLE blockLoop + + // Copy vN back to x. + MOVQ R8, 0(AX) + MOVQ R9, 8(AX) + MOVQ R10, 16(AX) + MOVQ R11, 24(AX) + + // Construct return slice. + // NOTE: It's important that we don't construct a slice that has a base + // pointer off the end of the original slice, as in Go 1.7+ this will + // cause runtime crashes. (See discussion in, for example, + // https://github.com/golang/go/issues/16772.) + // Therefore, we calculate the length/cap first, and if they're zero, we + // keep the old base. This is what the compiler does as well if you + // write code like + // b = b[len(b):] + + // New length is 32 - (CX - BX) -> BX+32 - CX. + ADDQ $32, BX + SUBQ CX, BX + JZ afterSetBase + + MOVQ CX, ret_base+32(FP) + +afterSetBase: + MOVQ BX, ret_len+40(FP) + MOVQ BX, ret_cap+48(FP) // set cap == len + + RET diff --git a/vendor/github.com/cespare/xxhash/xxhash_other.go b/vendor/github.com/cespare/xxhash/xxhash_other.go new file mode 100644 index 0000000000..c68d13f89e --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_other.go @@ -0,0 +1,75 @@ +// +build !amd64 appengine !gc purego + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +func Sum64(b []byte) uint64 { + // A simpler version would be + // x := New() + // x.Write(b) + // return x.Sum64() + // but this is faster, particularly for small inputs. + + n := len(b) + var h uint64 + + if n >= 32 { + v1 := prime1v + prime2 + v2 := prime2 + v3 := uint64(0) + v4 := -prime1v + for len(b) >= 32 { + v1 = round(v1, u64(b[0:8:len(b)])) + v2 = round(v2, u64(b[8:16:len(b)])) + v3 = round(v3, u64(b[16:24:len(b)])) + v4 = round(v4, u64(b[24:32:len(b)])) + b = b[32:len(b):len(b)] + } + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + h = mergeRound(h, v1) + h = mergeRound(h, v2) + h = mergeRound(h, v3) + h = mergeRound(h, v4) + } else { + h = prime5 + } + + h += uint64(n) + + i, end := 0, len(b) + for ; i+8 <= end; i += 8 { + k1 := round(0, u64(b[i:i+8:len(b)])) + h ^= k1 + h = rol27(h)*prime1 + prime4 + } + if i+4 <= end { + h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 + h = rol23(h)*prime2 + prime3 + i += 4 + } + for ; i < end; i++ { + h ^= uint64(b[i]) * prime5 + h = rol11(h) * prime1 + } + + h ^= h >> 33 + h *= prime2 + h ^= h >> 29 + h *= prime3 + h ^= h >> 32 + + return h +} + +func writeBlocks(x *xxh, b []byte) []byte { + v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4 + for len(b) >= 32 { + v1 = round(v1, u64(b[0:8:len(b)])) + v2 = round(v2, u64(b[8:16:len(b)])) + v3 = round(v3, u64(b[16:24:len(b)])) + v4 = round(v4, u64(b[24:32:len(b)])) + b = b[32:len(b):len(b)] + } + x.v1, x.v2, x.v3, x.v4 = v1, v2, v3, v4 + return b +} diff --git a/vendor/github.com/cespare/xxhash/xxhash_safe.go b/vendor/github.com/cespare/xxhash/xxhash_safe.go new file mode 100644 index 0000000000..dfa15ab7e2 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_safe.go @@ -0,0 +1,10 @@ +// +build appengine + +// This file contains the safe implementations of otherwise unsafe-using code. + +package xxhash + +// Sum64String computes the 64-bit xxHash digest of s. +func Sum64String(s string) uint64 { + return Sum64([]byte(s)) +} diff --git a/vendor/github.com/cespare/xxhash/xxhash_unsafe.go b/vendor/github.com/cespare/xxhash/xxhash_unsafe.go new file mode 100644 index 0000000000..d2b64e8bb0 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/xxhash_unsafe.go @@ -0,0 +1,30 @@ +// +build !appengine + +// This file encapsulates usage of unsafe. +// xxhash_safe.go contains the safe implementations. + +package xxhash + +import ( + "reflect" + "unsafe" +) + +// Sum64String computes the 64-bit xxHash digest of s. +// It may be faster than Sum64([]byte(s)) by avoiding a copy. +// +// TODO(caleb): Consider removing this if an optimization is ever added to make +// it unnecessary: https://golang.org/issue/2205. +// +// TODO(caleb): We still have a function call; we could instead write Go/asm +// copies of Sum64 for strings to squeeze out a bit more speed. +func Sum64String(s string) uint64 { + // See https://groups.google.com/d/msg/golang-nuts/dcjzJy-bSpw/tcZYBzQqAQAJ + // for some discussion about this unsafe conversion. + var b []byte + bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data + bh.Len = len(s) + bh.Cap = len(s) + return Sum64(b) +} diff --git a/vendor/github.com/k3a/html2text/.travis.yml b/vendor/github.com/k3a/html2text/.travis.yml new file mode 100644 index 0000000000..c9f214d59a --- /dev/null +++ b/vendor/github.com/k3a/html2text/.travis.yml @@ -0,0 +1,10 @@ +language: go +go: + - master +before_install: + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls + - if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi +script: + - $HOME/gopath/bin/goveralls -service=travis-ci + diff --git a/vendor/github.com/k3a/html2text/LICENSE b/vendor/github.com/k3a/html2text/LICENSE new file mode 100644 index 0000000000..4838b48ab4 --- /dev/null +++ b/vendor/github.com/k3a/html2text/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Mario K3A Hros (www.k3a.me) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/k3a/html2text/README.md b/vendor/github.com/k3a/html2text/README.md new file mode 100644 index 0000000000..892665905e --- /dev/null +++ b/vendor/github.com/k3a/html2text/README.md @@ -0,0 +1,60 @@ +[![GoDoc](https://godoc.org/github.com/k3a/html2text?status.svg)](https://godoc.org/github.com/k3a/html2text) +[![Build Status](https://travis-ci.org/k3a/html2text.svg?branch=master)](https://travis-ci.org/k3a/html2text) +[![Coverage Status](https://coveralls.io/repos/github/k3a/html2text/badge.svg?branch=master)](https://coveralls.io/github/k3a/html2text?branch=master) +[![Report Card](https://goreportcard.com/badge/github.com/k3a/html2text)](https://goreportcard.com/report/github.com/k3a/html2text) + +# html2text + +A simple Golang package to convert HTML to plain text (without non-standard dependencies). + +It converts HTML tags to text and also parses HTML entities into characters they represent. +A `` section of the HTML document, as well as most other tags are stripped out but +links are properly converted into their href attribute. + +It can be used for converting HTML emails into text. + +Some tests are installed as well. +Uses semantic versioning and no breaking changes are planned. + +Fell free to publish a pull request if you have suggestions for improvement but please note that the library can now be considered feature-complete and API stable. If you need more than this basic conversion, please use an alternative mentioned at the bottom. + +## Install +```bash +go get github.com/k3a/html2text +``` + +## Usage + +```go +package main + +import ( + "fmt" + "github.com/k3a/html2text" +) + +func main() { + html := `Goodclean text` + + plain := html2text.HTML2Text(html) + + fmt.Println(plain) +} + +/* Outputs: + + clean text +*/ + +``` + +To see all features, please look info `html2text_test.go`. + +## Alternatives +- https://github.com/jaytaylor/html2text (heavier, with more features) +- https://git.alexwennerberg.com/nanohtml2text (rewrite of this module in Rust) + +## License + +MIT + diff --git a/vendor/github.com/k3a/html2text/entity.go b/vendor/github.com/k3a/html2text/entity.go new file mode 100644 index 0000000000..68f90068d7 --- /dev/null +++ b/vendor/github.com/k3a/html2text/entity.go @@ -0,0 +1,2046 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html2text + +// entity is a map from HTML entity names to their values. The semicolon matters: +// https://html.spec.whatwg.org/multipage/named-characters.html +// lists both "amp" and "amp;" as two separate entries. +// +// Note that the HTML5 list is larger than the HTML4 list at +// http://www.w3.org/TR/html4/sgml/entities.html +var entity = map[string]rune{ + "AElig": '\U000000C6', + "AMP": '\U00000026', + "Aacute": '\U000000C1', + "Abreve": '\U00000102', + "Acirc": '\U000000C2', + "Acy": '\U00000410', + "Afr": '\U0001D504', + "Agrave": '\U000000C0', + "Alpha": '\U00000391', + "Amacr": '\U00000100', + "And": '\U00002A53', + "Aogon": '\U00000104', + "Aopf": '\U0001D538', + "ApplyFunction": '\U00002061', + "Aring": '\U000000C5', + "Ascr": '\U0001D49C', + "Assign": '\U00002254', + "Atilde": '\U000000C3', + "Auml": '\U000000C4', + "Backslash": '\U00002216', + "Barv": '\U00002AE7', + "Barwed": '\U00002306', + "Bcy": '\U00000411', + "Because": '\U00002235', + "Bernoullis": '\U0000212C', + "Beta": '\U00000392', + "Bfr": '\U0001D505', + "Bopf": '\U0001D539', + "Breve": '\U000002D8', + "Bscr": '\U0000212C', + "Bumpeq": '\U0000224E', + "CHcy": '\U00000427', + "COPY": '\U000000A9', + "Cacute": '\U00000106', + "Cap": '\U000022D2', + "CapitalDifferentialD": '\U00002145', + "Cayleys": '\U0000212D', + "Ccaron": '\U0000010C', + "Ccedil": '\U000000C7', + "Ccirc": '\U00000108', + "Cconint": '\U00002230', + "Cdot": '\U0000010A', + "Cedilla": '\U000000B8', + "CenterDot": '\U000000B7', + "Cfr": '\U0000212D', + "Chi": '\U000003A7', + "CircleDot": '\U00002299', + "CircleMinus": '\U00002296', + "CirclePlus": '\U00002295', + "CircleTimes": '\U00002297', + "ClockwiseContourIntegral": '\U00002232', + "CloseCurlyDoubleQuote": '\U0000201D', + "CloseCurlyQuote": '\U00002019', + "Colon": '\U00002237', + "Colone": '\U00002A74', + "Congruent": '\U00002261', + "Conint": '\U0000222F', + "ContourIntegral": '\U0000222E', + "Copf": '\U00002102', + "Coproduct": '\U00002210', + "CounterClockwiseContourIntegral": '\U00002233', + "Cross": '\U00002A2F', + "Cscr": '\U0001D49E', + "Cup": '\U000022D3', + "CupCap": '\U0000224D', + "DD": '\U00002145', + "DDotrahd": '\U00002911', + "DJcy": '\U00000402', + "DScy": '\U00000405', + "DZcy": '\U0000040F', + "Dagger": '\U00002021', + "Darr": '\U000021A1', + "Dashv": '\U00002AE4', + "Dcaron": '\U0000010E', + "Dcy": '\U00000414', + "Del": '\U00002207', + "Delta": '\U00000394', + "Dfr": '\U0001D507', + "DiacriticalAcute": '\U000000B4', + "DiacriticalDot": '\U000002D9', + "DiacriticalDoubleAcute": '\U000002DD', + "DiacriticalGrave": '\U00000060', + "DiacriticalTilde": '\U000002DC', + "Diamond": '\U000022C4', + "DifferentialD": '\U00002146', + "Dopf": '\U0001D53B', + "Dot": '\U000000A8', + "DotDot": '\U000020DC', + "DotEqual": '\U00002250', + "DoubleContourIntegral": '\U0000222F', + "DoubleDot": '\U000000A8', + "DoubleDownArrow": '\U000021D3', + "DoubleLeftArrow": '\U000021D0', + "DoubleLeftRightArrow": '\U000021D4', + "DoubleLeftTee": '\U00002AE4', + "DoubleLongLeftArrow": '\U000027F8', + "DoubleLongLeftRightArrow": '\U000027FA', + "DoubleLongRightArrow": '\U000027F9', + "DoubleRightArrow": '\U000021D2', + "DoubleRightTee": '\U000022A8', + "DoubleUpArrow": '\U000021D1', + "DoubleUpDownArrow": '\U000021D5', + "DoubleVerticalBar": '\U00002225', + "DownArrow": '\U00002193', + "DownArrowBar": '\U00002913', + "DownArrowUpArrow": '\U000021F5', + "DownBreve": '\U00000311', + "DownLeftRightVector": '\U00002950', + "DownLeftTeeVector": '\U0000295E', + "DownLeftVector": '\U000021BD', + "DownLeftVectorBar": '\U00002956', + "DownRightTeeVector": '\U0000295F', + "DownRightVector": '\U000021C1', + "DownRightVectorBar": '\U00002957', + "DownTee": '\U000022A4', + "DownTeeArrow": '\U000021A7', + "Downarrow": '\U000021D3', + "Dscr": '\U0001D49F', + "Dstrok": '\U00000110', + "ENG": '\U0000014A', + "ETH": '\U000000D0', + "Eacute": '\U000000C9', + "Ecaron": '\U0000011A', + "Ecirc": '\U000000CA', + "Ecy": '\U0000042D', + "Edot": '\U00000116', + "Efr": '\U0001D508', + "Egrave": '\U000000C8', + "Element": '\U00002208', + "Emacr": '\U00000112', + "EmptySmallSquare": '\U000025FB', + "EmptyVerySmallSquare": '\U000025AB', + "Eogon": '\U00000118', + "Eopf": '\U0001D53C', + "Epsilon": '\U00000395', + "Equal": '\U00002A75', + "EqualTilde": '\U00002242', + "Equilibrium": '\U000021CC', + "Escr": '\U00002130', + "Esim": '\U00002A73', + "Eta": '\U00000397', + "Euml": '\U000000CB', + "Exists": '\U00002203', + "ExponentialE": '\U00002147', + "Fcy": '\U00000424', + "Ffr": '\U0001D509', + "FilledSmallSquare": '\U000025FC', + "FilledVerySmallSquare": '\U000025AA', + "Fopf": '\U0001D53D', + "ForAll": '\U00002200', + "Fouriertrf": '\U00002131', + "Fscr": '\U00002131', + "GJcy": '\U00000403', + "GT": '\U0000003E', + "Gamma": '\U00000393', + "Gammad": '\U000003DC', + "Gbreve": '\U0000011E', + "Gcedil": '\U00000122', + "Gcirc": '\U0000011C', + "Gcy": '\U00000413', + "Gdot": '\U00000120', + "Gfr": '\U0001D50A', + "Gg": '\U000022D9', + "Gopf": '\U0001D53E', + "GreaterEqual": '\U00002265', + "GreaterEqualLess": '\U000022DB', + "GreaterFullEqual": '\U00002267', + "GreaterGreater": '\U00002AA2', + "GreaterLess": '\U00002277', + "GreaterSlantEqual": '\U00002A7E', + "GreaterTilde": '\U00002273', + "Gscr": '\U0001D4A2', + "Gt": '\U0000226B', + "HARDcy": '\U0000042A', + "Hacek": '\U000002C7', + "Hat": '\U0000005E', + "Hcirc": '\U00000124', + "Hfr": '\U0000210C', + "HilbertSpace": '\U0000210B', + "Hopf": '\U0000210D', + "HorizontalLine": '\U00002500', + "Hscr": '\U0000210B', + "Hstrok": '\U00000126', + "HumpDownHump": '\U0000224E', + "HumpEqual": '\U0000224F', + "IEcy": '\U00000415', + "IJlig": '\U00000132', + "IOcy": '\U00000401', + "Iacute": '\U000000CD', + "Icirc": '\U000000CE', + "Icy": '\U00000418', + "Idot": '\U00000130', + "Ifr": '\U00002111', + "Igrave": '\U000000CC', + "Im": '\U00002111', + "Imacr": '\U0000012A', + "ImaginaryI": '\U00002148', + "Implies": '\U000021D2', + "Int": '\U0000222C', + "Integral": '\U0000222B', + "Intersection": '\U000022C2', + "InvisibleComma": '\U00002063', + "InvisibleTimes": '\U00002062', + "Iogon": '\U0000012E', + "Iopf": '\U0001D540', + "Iota": '\U00000399', + "Iscr": '\U00002110', + "Itilde": '\U00000128', + "Iukcy": '\U00000406', + "Iuml": '\U000000CF', + "Jcirc": '\U00000134', + "Jcy": '\U00000419', + "Jfr": '\U0001D50D', + "Jopf": '\U0001D541', + "Jscr": '\U0001D4A5', + "Jsercy": '\U00000408', + "Jukcy": '\U00000404', + "KHcy": '\U00000425', + "KJcy": '\U0000040C', + "Kappa": '\U0000039A', + "Kcedil": '\U00000136', + "Kcy": '\U0000041A', + "Kfr": '\U0001D50E', + "Kopf": '\U0001D542', + "Kscr": '\U0001D4A6', + "LJcy": '\U00000409', + "LT": '\U0000003C', + "Lacute": '\U00000139', + "Lambda": '\U0000039B', + "Lang": '\U000027EA', + "Laplacetrf": '\U00002112', + "Larr": '\U0000219E', + "Lcaron": '\U0000013D', + "Lcedil": '\U0000013B', + "Lcy": '\U0000041B', + "LeftAngleBracket": '\U000027E8', + "LeftArrow": '\U00002190', + "LeftArrowBar": '\U000021E4', + "LeftArrowRightArrow": '\U000021C6', + "LeftCeiling": '\U00002308', + "LeftDoubleBracket": '\U000027E6', + "LeftDownTeeVector": '\U00002961', + "LeftDownVector": '\U000021C3', + "LeftDownVectorBar": '\U00002959', + "LeftFloor": '\U0000230A', + "LeftRightArrow": '\U00002194', + "LeftRightVector": '\U0000294E', + "LeftTee": '\U000022A3', + "LeftTeeArrow": '\U000021A4', + "LeftTeeVector": '\U0000295A', + "LeftTriangle": '\U000022B2', + "LeftTriangleBar": '\U000029CF', + "LeftTriangleEqual": '\U000022B4', + "LeftUpDownVector": '\U00002951', + "LeftUpTeeVector": '\U00002960', + "LeftUpVector": '\U000021BF', + "LeftUpVectorBar": '\U00002958', + "LeftVector": '\U000021BC', + "LeftVectorBar": '\U00002952', + "Leftarrow": '\U000021D0', + "Leftrightarrow": '\U000021D4', + "LessEqualGreater": '\U000022DA', + "LessFullEqual": '\U00002266', + "LessGreater": '\U00002276', + "LessLess": '\U00002AA1', + "LessSlantEqual": '\U00002A7D', + "LessTilde": '\U00002272', + "Lfr": '\U0001D50F', + "Ll": '\U000022D8', + "Lleftarrow": '\U000021DA', + "Lmidot": '\U0000013F', + "LongLeftArrow": '\U000027F5', + "LongLeftRightArrow": '\U000027F7', + "LongRightArrow": '\U000027F6', + "Longleftarrow": '\U000027F8', + "Longleftrightarrow": '\U000027FA', + "Longrightarrow": '\U000027F9', + "Lopf": '\U0001D543', + "LowerLeftArrow": '\U00002199', + "LowerRightArrow": '\U00002198', + "Lscr": '\U00002112', + "Lsh": '\U000021B0', + "Lstrok": '\U00000141', + "Lt": '\U0000226A', + "Map": '\U00002905', + "Mcy": '\U0000041C', + "MediumSpace": '\U0000205F', + "Mellintrf": '\U00002133', + "Mfr": '\U0001D510', + "MinusPlus": '\U00002213', + "Mopf": '\U0001D544', + "Mscr": '\U00002133', + "Mu": '\U0000039C', + "NJcy": '\U0000040A', + "Nacute": '\U00000143', + "Ncaron": '\U00000147', + "Ncedil": '\U00000145', + "Ncy": '\U0000041D', + "NegativeMediumSpace": '\U0000200B', + "NegativeThickSpace": '\U0000200B', + "NegativeThinSpace": '\U0000200B', + "NegativeVeryThinSpace": '\U0000200B', + "NestedGreaterGreater": '\U0000226B', + "NestedLessLess": '\U0000226A', + "NewLine": '\U0000000A', + "Nfr": '\U0001D511', + "NoBreak": '\U00002060', + "NonBreakingSpace": '\U000000A0', + "Nopf": '\U00002115', + "Not": '\U00002AEC', + "NotCongruent": '\U00002262', + "NotCupCap": '\U0000226D', + "NotDoubleVerticalBar": '\U00002226', + "NotElement": '\U00002209', + "NotEqual": '\U00002260', + "NotExists": '\U00002204', + "NotGreater": '\U0000226F', + "NotGreaterEqual": '\U00002271', + "NotGreaterLess": '\U00002279', + "NotGreaterTilde": '\U00002275', + "NotLeftTriangle": '\U000022EA', + "NotLeftTriangleEqual": '\U000022EC', + "NotLess": '\U0000226E', + "NotLessEqual": '\U00002270', + "NotLessGreater": '\U00002278', + "NotLessTilde": '\U00002274', + "NotPrecedes": '\U00002280', + "NotPrecedesSlantEqual": '\U000022E0', + "NotReverseElement": '\U0000220C', + "NotRightTriangle": '\U000022EB', + "NotRightTriangleEqual": '\U000022ED', + "NotSquareSubsetEqual": '\U000022E2', + "NotSquareSupersetEqual": '\U000022E3', + "NotSubsetEqual": '\U00002288', + "NotSucceeds": '\U00002281', + "NotSucceedsSlantEqual": '\U000022E1', + "NotSupersetEqual": '\U00002289', + "NotTilde": '\U00002241', + "NotTildeEqual": '\U00002244', + "NotTildeFullEqual": '\U00002247', + "NotTildeTilde": '\U00002249', + "NotVerticalBar": '\U00002224', + "Nscr": '\U0001D4A9', + "Ntilde": '\U000000D1', + "Nu": '\U0000039D', + "OElig": '\U00000152', + "Oacute": '\U000000D3', + "Ocirc": '\U000000D4', + "Ocy": '\U0000041E', + "Odblac": '\U00000150', + "Ofr": '\U0001D512', + "Ograve": '\U000000D2', + "Omacr": '\U0000014C', + "Omega": '\U000003A9', + "Omicron": '\U0000039F', + "Oopf": '\U0001D546', + "OpenCurlyDoubleQuote": '\U0000201C', + "OpenCurlyQuote": '\U00002018', + "Or": '\U00002A54', + "Oscr": '\U0001D4AA', + "Oslash": '\U000000D8', + "Otilde": '\U000000D5', + "Otimes": '\U00002A37', + "Ouml": '\U000000D6', + "OverBar": '\U0000203E', + "OverBrace": '\U000023DE', + "OverBracket": '\U000023B4', + "OverParenthesis": '\U000023DC', + "PartialD": '\U00002202', + "Pcy": '\U0000041F', + "Pfr": '\U0001D513', + "Phi": '\U000003A6', + "Pi": '\U000003A0', + "PlusMinus": '\U000000B1', + "Poincareplane": '\U0000210C', + "Popf": '\U00002119', + "Pr": '\U00002ABB', + "Precedes": '\U0000227A', + "PrecedesEqual": '\U00002AAF', + "PrecedesSlantEqual": '\U0000227C', + "PrecedesTilde": '\U0000227E', + "Prime": '\U00002033', + "Product": '\U0000220F', + "Proportion": '\U00002237', + "Proportional": '\U0000221D', + "Pscr": '\U0001D4AB', + "Psi": '\U000003A8', + "QUOT": '\U00000022', + "Qfr": '\U0001D514', + "Qopf": '\U0000211A', + "Qscr": '\U0001D4AC', + "RBarr": '\U00002910', + "REG": '\U000000AE', + "Racute": '\U00000154', + "Rang": '\U000027EB', + "Rarr": '\U000021A0', + "Rarrtl": '\U00002916', + "Rcaron": '\U00000158', + "Rcedil": '\U00000156', + "Rcy": '\U00000420', + "Re": '\U0000211C', + "ReverseElement": '\U0000220B', + "ReverseEquilibrium": '\U000021CB', + "ReverseUpEquilibrium": '\U0000296F', + "Rfr": '\U0000211C', + "Rho": '\U000003A1', + "RightAngleBracket": '\U000027E9', + "RightArrow": '\U00002192', + "RightArrowBar": '\U000021E5', + "RightArrowLeftArrow": '\U000021C4', + "RightCeiling": '\U00002309', + "RightDoubleBracket": '\U000027E7', + "RightDownTeeVector": '\U0000295D', + "RightDownVector": '\U000021C2', + "RightDownVectorBar": '\U00002955', + "RightFloor": '\U0000230B', + "RightTee": '\U000022A2', + "RightTeeArrow": '\U000021A6', + "RightTeeVector": '\U0000295B', + "RightTriangle": '\U000022B3', + "RightTriangleBar": '\U000029D0', + "RightTriangleEqual": '\U000022B5', + "RightUpDownVector": '\U0000294F', + "RightUpTeeVector": '\U0000295C', + "RightUpVector": '\U000021BE', + "RightUpVectorBar": '\U00002954', + "RightVector": '\U000021C0', + "RightVectorBar": '\U00002953', + "Rightarrow": '\U000021D2', + "Ropf": '\U0000211D', + "RoundImplies": '\U00002970', + "Rrightarrow": '\U000021DB', + "Rscr": '\U0000211B', + "Rsh": '\U000021B1', + "RuleDelayed": '\U000029F4', + "SHCHcy": '\U00000429', + "SHcy": '\U00000428', + "SOFTcy": '\U0000042C', + "Sacute": '\U0000015A', + "Sc": '\U00002ABC', + "Scaron": '\U00000160', + "Scedil": '\U0000015E', + "Scirc": '\U0000015C', + "Scy": '\U00000421', + "Sfr": '\U0001D516', + "ShortDownArrow": '\U00002193', + "ShortLeftArrow": '\U00002190', + "ShortRightArrow": '\U00002192', + "ShortUpArrow": '\U00002191', + "Sigma": '\U000003A3', + "SmallCircle": '\U00002218', + "Sopf": '\U0001D54A', + "Sqrt": '\U0000221A', + "Square": '\U000025A1', + "SquareIntersection": '\U00002293', + "SquareSubset": '\U0000228F', + "SquareSubsetEqual": '\U00002291', + "SquareSuperset": '\U00002290', + "SquareSupersetEqual": '\U00002292', + "SquareUnion": '\U00002294', + "Sscr": '\U0001D4AE', + "Star": '\U000022C6', + "Sub": '\U000022D0', + "Subset": '\U000022D0', + "SubsetEqual": '\U00002286', + "Succeeds": '\U0000227B', + "SucceedsEqual": '\U00002AB0', + "SucceedsSlantEqual": '\U0000227D', + "SucceedsTilde": '\U0000227F', + "SuchThat": '\U0000220B', + "Sum": '\U00002211', + "Sup": '\U000022D1', + "Superset": '\U00002283', + "SupersetEqual": '\U00002287', + "Supset": '\U000022D1', + "THORN": '\U000000DE', + "TRADE": '\U00002122', + "TSHcy": '\U0000040B', + "TScy": '\U00000426', + "Tab": '\U00000009', + "Tau": '\U000003A4', + "Tcaron": '\U00000164', + "Tcedil": '\U00000162', + "Tcy": '\U00000422', + "Tfr": '\U0001D517', + "Therefore": '\U00002234', + "Theta": '\U00000398', + "ThinSpace": '\U00002009', + "Tilde": '\U0000223C', + "TildeEqual": '\U00002243', + "TildeFullEqual": '\U00002245', + "TildeTilde": '\U00002248', + "Topf": '\U0001D54B', + "TripleDot": '\U000020DB', + "Tscr": '\U0001D4AF', + "Tstrok": '\U00000166', + "Uacute": '\U000000DA', + "Uarr": '\U0000219F', + "Uarrocir": '\U00002949', + "Ubrcy": '\U0000040E', + "Ubreve": '\U0000016C', + "Ucirc": '\U000000DB', + "Ucy": '\U00000423', + "Udblac": '\U00000170', + "Ufr": '\U0001D518', + "Ugrave": '\U000000D9', + "Umacr": '\U0000016A', + "UnderBar": '\U0000005F', + "UnderBrace": '\U000023DF', + "UnderBracket": '\U000023B5', + "UnderParenthesis": '\U000023DD', + "Union": '\U000022C3', + "UnionPlus": '\U0000228E', + "Uogon": '\U00000172', + "Uopf": '\U0001D54C', + "UpArrow": '\U00002191', + "UpArrowBar": '\U00002912', + "UpArrowDownArrow": '\U000021C5', + "UpDownArrow": '\U00002195', + "UpEquilibrium": '\U0000296E', + "UpTee": '\U000022A5', + "UpTeeArrow": '\U000021A5', + "Uparrow": '\U000021D1', + "Updownarrow": '\U000021D5', + "UpperLeftArrow": '\U00002196', + "UpperRightArrow": '\U00002197', + "Upsi": '\U000003D2', + "Upsilon": '\U000003A5', + "Uring": '\U0000016E', + "Uscr": '\U0001D4B0', + "Utilde": '\U00000168', + "Uuml": '\U000000DC', + "VDash": '\U000022AB', + "Vbar": '\U00002AEB', + "Vcy": '\U00000412', + "Vdash": '\U000022A9', + "Vdashl": '\U00002AE6', + "Vee": '\U000022C1', + "Verbar": '\U00002016', + "Vert": '\U00002016', + "VerticalBar": '\U00002223', + "VerticalLine": '\U0000007C', + "VerticalSeparator": '\U00002758', + "VerticalTilde": '\U00002240', + "VeryThinSpace": '\U0000200A', + "Vfr": '\U0001D519', + "Vopf": '\U0001D54D', + "Vscr": '\U0001D4B1', + "Vvdash": '\U000022AA', + "Wcirc": '\U00000174', + "Wedge": '\U000022C0', + "Wfr": '\U0001D51A', + "Wopf": '\U0001D54E', + "Wscr": '\U0001D4B2', + "Xfr": '\U0001D51B', + "Xi": '\U0000039E', + "Xopf": '\U0001D54F', + "Xscr": '\U0001D4B3', + "YAcy": '\U0000042F', + "YIcy": '\U00000407', + "YUcy": '\U0000042E', + "Yacute": '\U000000DD', + "Ycirc": '\U00000176', + "Ycy": '\U0000042B', + "Yfr": '\U0001D51C', + "Yopf": '\U0001D550', + "Yscr": '\U0001D4B4', + "Yuml": '\U00000178', + "ZHcy": '\U00000416', + "Zacute": '\U00000179', + "Zcaron": '\U0000017D', + "Zcy": '\U00000417', + "Zdot": '\U0000017B', + "ZeroWidthSpace": '\U0000200B', + "Zeta": '\U00000396', + "Zfr": '\U00002128', + "Zopf": '\U00002124', + "Zscr": '\U0001D4B5', + "aacute": '\U000000E1', + "abreve": '\U00000103', + "ac": '\U0000223E', + "acd": '\U0000223F', + "acirc": '\U000000E2', + "acute": '\U000000B4', + "acy": '\U00000430', + "aelig": '\U000000E6', + "af": '\U00002061', + "afr": '\U0001D51E', + "agrave": '\U000000E0', + "alefsym": '\U00002135', + "aleph": '\U00002135', + "alpha": '\U000003B1', + "amacr": '\U00000101', + "amalg": '\U00002A3F', + "amp": '\U00000026', + "and": '\U00002227', + "andand": '\U00002A55', + "andd": '\U00002A5C', + "andslope": '\U00002A58', + "andv": '\U00002A5A', + "ang": '\U00002220', + "ange": '\U000029A4', + "angle": '\U00002220', + "angmsd": '\U00002221', + "angmsdaa": '\U000029A8', + "angmsdab": '\U000029A9', + "angmsdac": '\U000029AA', + "angmsdad": '\U000029AB', + "angmsdae": '\U000029AC', + "angmsdaf": '\U000029AD', + "angmsdag": '\U000029AE', + "angmsdah": '\U000029AF', + "angrt": '\U0000221F', + "angrtvb": '\U000022BE', + "angrtvbd": '\U0000299D', + "angsph": '\U00002222', + "angst": '\U000000C5', + "angzarr": '\U0000237C', + "aogon": '\U00000105', + "aopf": '\U0001D552', + "ap": '\U00002248', + "apE": '\U00002A70', + "apacir": '\U00002A6F', + "ape": '\U0000224A', + "apid": '\U0000224B', + "apos": '\U00000027', + "approx": '\U00002248', + "approxeq": '\U0000224A', + "aring": '\U000000E5', + "ascr": '\U0001D4B6', + "ast": '\U0000002A', + "asymp": '\U00002248', + "asympeq": '\U0000224D', + "atilde": '\U000000E3', + "auml": '\U000000E4', + "awconint": '\U00002233', + "awint": '\U00002A11', + "bNot": '\U00002AED', + "backcong": '\U0000224C', + "backepsilon": '\U000003F6', + "backprime": '\U00002035', + "backsim": '\U0000223D', + "backsimeq": '\U000022CD', + "barvee": '\U000022BD', + "barwed": '\U00002305', + "barwedge": '\U00002305', + "bbrk": '\U000023B5', + "bbrktbrk": '\U000023B6', + "bcong": '\U0000224C', + "bcy": '\U00000431', + "bdquo": '\U0000201E', + "becaus": '\U00002235', + "because": '\U00002235', + "bemptyv": '\U000029B0', + "bepsi": '\U000003F6', + "bernou": '\U0000212C', + "beta": '\U000003B2', + "beth": '\U00002136', + "between": '\U0000226C', + "bfr": '\U0001D51F', + "bigcap": '\U000022C2', + "bigcirc": '\U000025EF', + "bigcup": '\U000022C3', + "bigodot": '\U00002A00', + "bigoplus": '\U00002A01', + "bigotimes": '\U00002A02', + "bigsqcup": '\U00002A06', + "bigstar": '\U00002605', + "bigtriangledown": '\U000025BD', + "bigtriangleup": '\U000025B3', + "biguplus": '\U00002A04', + "bigvee": '\U000022C1', + "bigwedge": '\U000022C0', + "bkarow": '\U0000290D', + "blacklozenge": '\U000029EB', + "blacksquare": '\U000025AA', + "blacktriangle": '\U000025B4', + "blacktriangledown": '\U000025BE', + "blacktriangleleft": '\U000025C2', + "blacktriangleright": '\U000025B8', + "blank": '\U00002423', + "blk12": '\U00002592', + "blk14": '\U00002591', + "blk34": '\U00002593', + "block": '\U00002588', + "bnot": '\U00002310', + "bopf": '\U0001D553', + "bot": '\U000022A5', + "bottom": '\U000022A5', + "bowtie": '\U000022C8', + "boxDL": '\U00002557', + "boxDR": '\U00002554', + "boxDl": '\U00002556', + "boxDr": '\U00002553', + "boxH": '\U00002550', + "boxHD": '\U00002566', + "boxHU": '\U00002569', + "boxHd": '\U00002564', + "boxHu": '\U00002567', + "boxUL": '\U0000255D', + "boxUR": '\U0000255A', + "boxUl": '\U0000255C', + "boxUr": '\U00002559', + "boxV": '\U00002551', + "boxVH": '\U0000256C', + "boxVL": '\U00002563', + "boxVR": '\U00002560', + "boxVh": '\U0000256B', + "boxVl": '\U00002562', + "boxVr": '\U0000255F', + "boxbox": '\U000029C9', + "boxdL": '\U00002555', + "boxdR": '\U00002552', + "boxdl": '\U00002510', + "boxdr": '\U0000250C', + "boxh": '\U00002500', + "boxhD": '\U00002565', + "boxhU": '\U00002568', + "boxhd": '\U0000252C', + "boxhu": '\U00002534', + "boxminus": '\U0000229F', + "boxplus": '\U0000229E', + "boxtimes": '\U000022A0', + "boxuL": '\U0000255B', + "boxuR": '\U00002558', + "boxul": '\U00002518', + "boxur": '\U00002514', + "boxv": '\U00002502', + "boxvH": '\U0000256A', + "boxvL": '\U00002561', + "boxvR": '\U0000255E', + "boxvh": '\U0000253C', + "boxvl": '\U00002524', + "boxvr": '\U0000251C', + "bprime": '\U00002035', + "breve": '\U000002D8', + "brvbar": '\U000000A6', + "bscr": '\U0001D4B7', + "bsemi": '\U0000204F', + "bsim": '\U0000223D', + "bsime": '\U000022CD', + "bsol": '\U0000005C', + "bsolb": '\U000029C5', + "bsolhsub": '\U000027C8', + "bull": '\U00002022', + "bullet": '\U00002022', + "bump": '\U0000224E', + "bumpE": '\U00002AAE', + "bumpe": '\U0000224F', + "bumpeq": '\U0000224F', + "cacute": '\U00000107', + "cap": '\U00002229', + "capand": '\U00002A44', + "capbrcup": '\U00002A49', + "capcap": '\U00002A4B', + "capcup": '\U00002A47', + "capdot": '\U00002A40', + "caret": '\U00002041', + "caron": '\U000002C7', + "ccaps": '\U00002A4D', + "ccaron": '\U0000010D', + "ccedil": '\U000000E7', + "ccirc": '\U00000109', + "ccups": '\U00002A4C', + "ccupssm": '\U00002A50', + "cdot": '\U0000010B', + "cedil": '\U000000B8', + "cemptyv": '\U000029B2', + "cent": '\U000000A2', + "centerdot": '\U000000B7', + "cfr": '\U0001D520', + "chcy": '\U00000447', + "check": '\U00002713', + "checkmark": '\U00002713', + "chi": '\U000003C7', + "cir": '\U000025CB', + "cirE": '\U000029C3', + "circ": '\U000002C6', + "circeq": '\U00002257', + "circlearrowleft": '\U000021BA', + "circlearrowright": '\U000021BB', + "circledR": '\U000000AE', + "circledS": '\U000024C8', + "circledast": '\U0000229B', + "circledcirc": '\U0000229A', + "circleddash": '\U0000229D', + "cire": '\U00002257', + "cirfnint": '\U00002A10', + "cirmid": '\U00002AEF', + "cirscir": '\U000029C2', + "clubs": '\U00002663', + "clubsuit": '\U00002663', + "colon": '\U0000003A', + "colone": '\U00002254', + "coloneq": '\U00002254', + "comma": '\U0000002C', + "commat": '\U00000040', + "comp": '\U00002201', + "compfn": '\U00002218', + "complement": '\U00002201', + "complexes": '\U00002102', + "cong": '\U00002245', + "congdot": '\U00002A6D', + "conint": '\U0000222E', + "copf": '\U0001D554', + "coprod": '\U00002210', + "copy": '\U000000A9', + "copysr": '\U00002117', + "crarr": '\U000021B5', + "cross": '\U00002717', + "cscr": '\U0001D4B8', + "csub": '\U00002ACF', + "csube": '\U00002AD1', + "csup": '\U00002AD0', + "csupe": '\U00002AD2', + "ctdot": '\U000022EF', + "cudarrl": '\U00002938', + "cudarrr": '\U00002935', + "cuepr": '\U000022DE', + "cuesc": '\U000022DF', + "cularr": '\U000021B6', + "cularrp": '\U0000293D', + "cup": '\U0000222A', + "cupbrcap": '\U00002A48', + "cupcap": '\U00002A46', + "cupcup": '\U00002A4A', + "cupdot": '\U0000228D', + "cupor": '\U00002A45', + "curarr": '\U000021B7', + "curarrm": '\U0000293C', + "curlyeqprec": '\U000022DE', + "curlyeqsucc": '\U000022DF', + "curlyvee": '\U000022CE', + "curlywedge": '\U000022CF', + "curren": '\U000000A4', + "curvearrowleft": '\U000021B6', + "curvearrowright": '\U000021B7', + "cuvee": '\U000022CE', + "cuwed": '\U000022CF', + "cwconint": '\U00002232', + "cwint": '\U00002231', + "cylcty": '\U0000232D', + "dArr": '\U000021D3', + "dHar": '\U00002965', + "dagger": '\U00002020', + "daleth": '\U00002138', + "darr": '\U00002193', + "dash": '\U00002010', + "dashv": '\U000022A3', + "dbkarow": '\U0000290F', + "dblac": '\U000002DD', + "dcaron": '\U0000010F', + "dcy": '\U00000434', + "dd": '\U00002146', + "ddagger": '\U00002021', + "ddarr": '\U000021CA', + "ddotseq": '\U00002A77', + "deg": '\U000000B0', + "delta": '\U000003B4', + "demptyv": '\U000029B1', + "dfisht": '\U0000297F', + "dfr": '\U0001D521', + "dharl": '\U000021C3', + "dharr": '\U000021C2', + "diam": '\U000022C4', + "diamond": '\U000022C4', + "diamondsuit": '\U00002666', + "diams": '\U00002666', + "die": '\U000000A8', + "digamma": '\U000003DD', + "disin": '\U000022F2', + "div": '\U000000F7', + "divide": '\U000000F7', + "divideontimes": '\U000022C7', + "divonx": '\U000022C7', + "djcy": '\U00000452', + "dlcorn": '\U0000231E', + "dlcrop": '\U0000230D', + "dollar": '\U00000024', + "dopf": '\U0001D555', + "dot": '\U000002D9', + "doteq": '\U00002250', + "doteqdot": '\U00002251', + "dotminus": '\U00002238', + "dotplus": '\U00002214', + "dotsquare": '\U000022A1', + "doublebarwedge": '\U00002306', + "downarrow": '\U00002193', + "downdownarrows": '\U000021CA', + "downharpoonleft": '\U000021C3', + "downharpoonright": '\U000021C2', + "drbkarow": '\U00002910', + "drcorn": '\U0000231F', + "drcrop": '\U0000230C', + "dscr": '\U0001D4B9', + "dscy": '\U00000455', + "dsol": '\U000029F6', + "dstrok": '\U00000111', + "dtdot": '\U000022F1', + "dtri": '\U000025BF', + "dtrif": '\U000025BE', + "duarr": '\U000021F5', + "duhar": '\U0000296F', + "dwangle": '\U000029A6', + "dzcy": '\U0000045F', + "dzigrarr": '\U000027FF', + "eDDot": '\U00002A77', + "eDot": '\U00002251', + "eacute": '\U000000E9', + "easter": '\U00002A6E', + "ecaron": '\U0000011B', + "ecir": '\U00002256', + "ecirc": '\U000000EA', + "ecolon": '\U00002255', + "ecy": '\U0000044D', + "edot": '\U00000117', + "ee": '\U00002147', + "efDot": '\U00002252', + "efr": '\U0001D522', + "eg": '\U00002A9A', + "egrave": '\U000000E8', + "egs": '\U00002A96', + "egsdot": '\U00002A98', + "el": '\U00002A99', + "elinters": '\U000023E7', + "ell": '\U00002113', + "els": '\U00002A95', + "elsdot": '\U00002A97', + "emacr": '\U00000113', + "empty": '\U00002205', + "emptyset": '\U00002205', + "emptyv": '\U00002205', + "emsp": '\U00002003', + "emsp13": '\U00002004', + "emsp14": '\U00002005', + "eng": '\U0000014B', + "ensp": '\U00002002', + "eogon": '\U00000119', + "eopf": '\U0001D556', + "epar": '\U000022D5', + "eparsl": '\U000029E3', + "eplus": '\U00002A71', + "epsi": '\U000003B5', + "epsilon": '\U000003B5', + "epsiv": '\U000003F5', + "eqcirc": '\U00002256', + "eqcolon": '\U00002255', + "eqsim": '\U00002242', + "eqslantgtr": '\U00002A96', + "eqslantless": '\U00002A95', + "equals": '\U0000003D', + "equest": '\U0000225F', + "equiv": '\U00002261', + "equivDD": '\U00002A78', + "eqvparsl": '\U000029E5', + "erDot": '\U00002253', + "erarr": '\U00002971', + "escr": '\U0000212F', + "esdot": '\U00002250', + "esim": '\U00002242', + "eta": '\U000003B7', + "eth": '\U000000F0', + "euml": '\U000000EB', + "euro": '\U000020AC', + "excl": '\U00000021', + "exist": '\U00002203', + "expectation": '\U00002130', + "exponentiale": '\U00002147', + "fallingdotseq": '\U00002252', + "fcy": '\U00000444', + "female": '\U00002640', + "ffilig": '\U0000FB03', + "fflig": '\U0000FB00', + "ffllig": '\U0000FB04', + "ffr": '\U0001D523', + "filig": '\U0000FB01', + "flat": '\U0000266D', + "fllig": '\U0000FB02', + "fltns": '\U000025B1', + "fnof": '\U00000192', + "fopf": '\U0001D557', + "forall": '\U00002200', + "fork": '\U000022D4', + "forkv": '\U00002AD9', + "fpartint": '\U00002A0D', + "frac12": '\U000000BD', + "frac13": '\U00002153', + "frac14": '\U000000BC', + "frac15": '\U00002155', + "frac16": '\U00002159', + "frac18": '\U0000215B', + "frac23": '\U00002154', + "frac25": '\U00002156', + "frac34": '\U000000BE', + "frac35": '\U00002157', + "frac38": '\U0000215C', + "frac45": '\U00002158', + "frac56": '\U0000215A', + "frac58": '\U0000215D', + "frac78": '\U0000215E', + "frasl": '\U00002044', + "frown": '\U00002322', + "fscr": '\U0001D4BB', + "gE": '\U00002267', + "gEl": '\U00002A8C', + "gacute": '\U000001F5', + "gamma": '\U000003B3', + "gammad": '\U000003DD', + "gap": '\U00002A86', + "gbreve": '\U0000011F', + "gcirc": '\U0000011D', + "gcy": '\U00000433', + "gdot": '\U00000121', + "ge": '\U00002265', + "gel": '\U000022DB', + "geq": '\U00002265', + "geqq": '\U00002267', + "geqslant": '\U00002A7E', + "ges": '\U00002A7E', + "gescc": '\U00002AA9', + "gesdot": '\U00002A80', + "gesdoto": '\U00002A82', + "gesdotol": '\U00002A84', + "gesles": '\U00002A94', + "gfr": '\U0001D524', + "gg": '\U0000226B', + "ggg": '\U000022D9', + "gimel": '\U00002137', + "gjcy": '\U00000453', + "gl": '\U00002277', + "glE": '\U00002A92', + "gla": '\U00002AA5', + "glj": '\U00002AA4', + "gnE": '\U00002269', + "gnap": '\U00002A8A', + "gnapprox": '\U00002A8A', + "gne": '\U00002A88', + "gneq": '\U00002A88', + "gneqq": '\U00002269', + "gnsim": '\U000022E7', + "gopf": '\U0001D558', + "grave": '\U00000060', + "gscr": '\U0000210A', + "gsim": '\U00002273', + "gsime": '\U00002A8E', + "gsiml": '\U00002A90', + "gt": '\U0000003E', + "gtcc": '\U00002AA7', + "gtcir": '\U00002A7A', + "gtdot": '\U000022D7', + "gtlPar": '\U00002995', + "gtquest": '\U00002A7C', + "gtrapprox": '\U00002A86', + "gtrarr": '\U00002978', + "gtrdot": '\U000022D7', + "gtreqless": '\U000022DB', + "gtreqqless": '\U00002A8C', + "gtrless": '\U00002277', + "gtrsim": '\U00002273', + "hArr": '\U000021D4', + "hairsp": '\U0000200A', + "half": '\U000000BD', + "hamilt": '\U0000210B', + "hardcy": '\U0000044A', + "harr": '\U00002194', + "harrcir": '\U00002948', + "harrw": '\U000021AD', + "hbar": '\U0000210F', + "hcirc": '\U00000125', + "hearts": '\U00002665', + "heartsuit": '\U00002665', + "hellip": '\U00002026', + "hercon": '\U000022B9', + "hfr": '\U0001D525', + "hksearow": '\U00002925', + "hkswarow": '\U00002926', + "hoarr": '\U000021FF', + "homtht": '\U0000223B', + "hookleftarrow": '\U000021A9', + "hookrightarrow": '\U000021AA', + "hopf": '\U0001D559', + "horbar": '\U00002015', + "hscr": '\U0001D4BD', + "hslash": '\U0000210F', + "hstrok": '\U00000127', + "hybull": '\U00002043', + "hyphen": '\U00002010', + "iacute": '\U000000ED', + "ic": '\U00002063', + "icirc": '\U000000EE', + "icy": '\U00000438', + "iecy": '\U00000435', + "iexcl": '\U000000A1', + "iff": '\U000021D4', + "ifr": '\U0001D526', + "igrave": '\U000000EC', + "ii": '\U00002148', + "iiiint": '\U00002A0C', + "iiint": '\U0000222D', + "iinfin": '\U000029DC', + "iiota": '\U00002129', + "ijlig": '\U00000133', + "imacr": '\U0000012B', + "image": '\U00002111', + "imagline": '\U00002110', + "imagpart": '\U00002111', + "imath": '\U00000131', + "imof": '\U000022B7', + "imped": '\U000001B5', + "in": '\U00002208', + "incare": '\U00002105', + "infin": '\U0000221E', + "infintie": '\U000029DD', + "inodot": '\U00000131', + "int": '\U0000222B', + "intcal": '\U000022BA', + "integers": '\U00002124', + "intercal": '\U000022BA', + "intlarhk": '\U00002A17', + "intprod": '\U00002A3C', + "iocy": '\U00000451', + "iogon": '\U0000012F', + "iopf": '\U0001D55A', + "iota": '\U000003B9', + "iprod": '\U00002A3C', + "iquest": '\U000000BF', + "iscr": '\U0001D4BE', + "isin": '\U00002208', + "isinE": '\U000022F9', + "isindot": '\U000022F5', + "isins": '\U000022F4', + "isinsv": '\U000022F3', + "isinv": '\U00002208', + "it": '\U00002062', + "itilde": '\U00000129', + "iukcy": '\U00000456', + "iuml": '\U000000EF', + "jcirc": '\U00000135', + "jcy": '\U00000439', + "jfr": '\U0001D527', + "jmath": '\U00000237', + "jopf": '\U0001D55B', + "jscr": '\U0001D4BF', + "jsercy": '\U00000458', + "jukcy": '\U00000454', + "kappa": '\U000003BA', + "kappav": '\U000003F0', + "kcedil": '\U00000137', + "kcy": '\U0000043A', + "kfr": '\U0001D528', + "kgreen": '\U00000138', + "khcy": '\U00000445', + "kjcy": '\U0000045C', + "kopf": '\U0001D55C', + "kscr": '\U0001D4C0', + "lAarr": '\U000021DA', + "lArr": '\U000021D0', + "lAtail": '\U0000291B', + "lBarr": '\U0000290E', + "lE": '\U00002266', + "lEg": '\U00002A8B', + "lHar": '\U00002962', + "lacute": '\U0000013A', + "laemptyv": '\U000029B4', + "lagran": '\U00002112', + "lambda": '\U000003BB', + "lang": '\U000027E8', + "langd": '\U00002991', + "langle": '\U000027E8', + "lap": '\U00002A85', + "laquo": '\U000000AB', + "larr": '\U00002190', + "larrb": '\U000021E4', + "larrbfs": '\U0000291F', + "larrfs": '\U0000291D', + "larrhk": '\U000021A9', + "larrlp": '\U000021AB', + "larrpl": '\U00002939', + "larrsim": '\U00002973', + "larrtl": '\U000021A2', + "lat": '\U00002AAB', + "latail": '\U00002919', + "late": '\U00002AAD', + "lbarr": '\U0000290C', + "lbbrk": '\U00002772', + "lbrace": '\U0000007B', + "lbrack": '\U0000005B', + "lbrke": '\U0000298B', + "lbrksld": '\U0000298F', + "lbrkslu": '\U0000298D', + "lcaron": '\U0000013E', + "lcedil": '\U0000013C', + "lceil": '\U00002308', + "lcub": '\U0000007B', + "lcy": '\U0000043B', + "ldca": '\U00002936', + "ldquo": '\U0000201C', + "ldquor": '\U0000201E', + "ldrdhar": '\U00002967', + "ldrushar": '\U0000294B', + "ldsh": '\U000021B2', + "le": '\U00002264', + "leftarrow": '\U00002190', + "leftarrowtail": '\U000021A2', + "leftharpoondown": '\U000021BD', + "leftharpoonup": '\U000021BC', + "leftleftarrows": '\U000021C7', + "leftrightarrow": '\U00002194', + "leftrightarrows": '\U000021C6', + "leftrightharpoons": '\U000021CB', + "leftrightsquigarrow": '\U000021AD', + "leftthreetimes": '\U000022CB', + "leg": '\U000022DA', + "leq": '\U00002264', + "leqq": '\U00002266', + "leqslant": '\U00002A7D', + "les": '\U00002A7D', + "lescc": '\U00002AA8', + "lesdot": '\U00002A7F', + "lesdoto": '\U00002A81', + "lesdotor": '\U00002A83', + "lesges": '\U00002A93', + "lessapprox": '\U00002A85', + "lessdot": '\U000022D6', + "lesseqgtr": '\U000022DA', + "lesseqqgtr": '\U00002A8B', + "lessgtr": '\U00002276', + "lesssim": '\U00002272', + "lfisht": '\U0000297C', + "lfloor": '\U0000230A', + "lfr": '\U0001D529', + "lg": '\U00002276', + "lgE": '\U00002A91', + "lhard": '\U000021BD', + "lharu": '\U000021BC', + "lharul": '\U0000296A', + "lhblk": '\U00002584', + "ljcy": '\U00000459', + "ll": '\U0000226A', + "llarr": '\U000021C7', + "llcorner": '\U0000231E', + "llhard": '\U0000296B', + "lltri": '\U000025FA', + "lmidot": '\U00000140', + "lmoust": '\U000023B0', + "lmoustache": '\U000023B0', + "lnE": '\U00002268', + "lnap": '\U00002A89', + "lnapprox": '\U00002A89', + "lne": '\U00002A87', + "lneq": '\U00002A87', + "lneqq": '\U00002268', + "lnsim": '\U000022E6', + "loang": '\U000027EC', + "loarr": '\U000021FD', + "lobrk": '\U000027E6', + "longleftarrow": '\U000027F5', + "longleftrightarrow": '\U000027F7', + "longmapsto": '\U000027FC', + "longrightarrow": '\U000027F6', + "looparrowleft": '\U000021AB', + "looparrowright": '\U000021AC', + "lopar": '\U00002985', + "lopf": '\U0001D55D', + "loplus": '\U00002A2D', + "lotimes": '\U00002A34', + "lowast": '\U00002217', + "lowbar": '\U0000005F', + "loz": '\U000025CA', + "lozenge": '\U000025CA', + "lozf": '\U000029EB', + "lpar": '\U00000028', + "lparlt": '\U00002993', + "lrarr": '\U000021C6', + "lrcorner": '\U0000231F', + "lrhar": '\U000021CB', + "lrhard": '\U0000296D', + "lrm": '\U0000200E', + "lrtri": '\U000022BF', + "lsaquo": '\U00002039', + "lscr": '\U0001D4C1', + "lsh": '\U000021B0', + "lsim": '\U00002272', + "lsime": '\U00002A8D', + "lsimg": '\U00002A8F', + "lsqb": '\U0000005B', + "lsquo": '\U00002018', + "lsquor": '\U0000201A', + "lstrok": '\U00000142', + "lt": '\U0000003C', + "ltcc": '\U00002AA6', + "ltcir": '\U00002A79', + "ltdot": '\U000022D6', + "lthree": '\U000022CB', + "ltimes": '\U000022C9', + "ltlarr": '\U00002976', + "ltquest": '\U00002A7B', + "ltrPar": '\U00002996', + "ltri": '\U000025C3', + "ltrie": '\U000022B4', + "ltrif": '\U000025C2', + "lurdshar": '\U0000294A', + "luruhar": '\U00002966', + "mDDot": '\U0000223A', + "macr": '\U000000AF', + "male": '\U00002642', + "malt": '\U00002720', + "maltese": '\U00002720', + "map": '\U000021A6', + "mapsto": '\U000021A6', + "mapstodown": '\U000021A7', + "mapstoleft": '\U000021A4', + "mapstoup": '\U000021A5', + "marker": '\U000025AE', + "mcomma": '\U00002A29', + "mcy": '\U0000043C', + "mdash": '\U00002014', + "measuredangle": '\U00002221', + "mfr": '\U0001D52A', + "mho": '\U00002127', + "micro": '\U000000B5', + "mid": '\U00002223', + "midast": '\U0000002A', + "midcir": '\U00002AF0', + "middot": '\U000000B7', + "minus": '\U00002212', + "minusb": '\U0000229F', + "minusd": '\U00002238', + "minusdu": '\U00002A2A', + "mlcp": '\U00002ADB', + "mldr": '\U00002026', + "mnplus": '\U00002213', + "models": '\U000022A7', + "mopf": '\U0001D55E', + "mp": '\U00002213', + "mscr": '\U0001D4C2', + "mstpos": '\U0000223E', + "mu": '\U000003BC', + "multimap": '\U000022B8', + "mumap": '\U000022B8', + "nLeftarrow": '\U000021CD', + "nLeftrightarrow": '\U000021CE', + "nRightarrow": '\U000021CF', + "nVDash": '\U000022AF', + "nVdash": '\U000022AE', + "nabla": '\U00002207', + "nacute": '\U00000144', + "nap": '\U00002249', + "napos": '\U00000149', + "napprox": '\U00002249', + "natur": '\U0000266E', + "natural": '\U0000266E', + "naturals": '\U00002115', + "nbsp": '\U000000A0', + "ncap": '\U00002A43', + "ncaron": '\U00000148', + "ncedil": '\U00000146', + "ncong": '\U00002247', + "ncup": '\U00002A42', + "ncy": '\U0000043D', + "ndash": '\U00002013', + "ne": '\U00002260', + "neArr": '\U000021D7', + "nearhk": '\U00002924', + "nearr": '\U00002197', + "nearrow": '\U00002197', + "nequiv": '\U00002262', + "nesear": '\U00002928', + "nexist": '\U00002204', + "nexists": '\U00002204', + "nfr": '\U0001D52B', + "nge": '\U00002271', + "ngeq": '\U00002271', + "ngsim": '\U00002275', + "ngt": '\U0000226F', + "ngtr": '\U0000226F', + "nhArr": '\U000021CE', + "nharr": '\U000021AE', + "nhpar": '\U00002AF2', + "ni": '\U0000220B', + "nis": '\U000022FC', + "nisd": '\U000022FA', + "niv": '\U0000220B', + "njcy": '\U0000045A', + "nlArr": '\U000021CD', + "nlarr": '\U0000219A', + "nldr": '\U00002025', + "nle": '\U00002270', + "nleftarrow": '\U0000219A', + "nleftrightarrow": '\U000021AE', + "nleq": '\U00002270', + "nless": '\U0000226E', + "nlsim": '\U00002274', + "nlt": '\U0000226E', + "nltri": '\U000022EA', + "nltrie": '\U000022EC', + "nmid": '\U00002224', + "nopf": '\U0001D55F', + "not": '\U000000AC', + "notin": '\U00002209', + "notinva": '\U00002209', + "notinvb": '\U000022F7', + "notinvc": '\U000022F6', + "notni": '\U0000220C', + "notniva": '\U0000220C', + "notnivb": '\U000022FE', + "notnivc": '\U000022FD', + "npar": '\U00002226', + "nparallel": '\U00002226', + "npolint": '\U00002A14', + "npr": '\U00002280', + "nprcue": '\U000022E0', + "nprec": '\U00002280', + "nrArr": '\U000021CF', + "nrarr": '\U0000219B', + "nrightarrow": '\U0000219B', + "nrtri": '\U000022EB', + "nrtrie": '\U000022ED', + "nsc": '\U00002281', + "nsccue": '\U000022E1', + "nscr": '\U0001D4C3', + "nshortmid": '\U00002224', + "nshortparallel": '\U00002226', + "nsim": '\U00002241', + "nsime": '\U00002244', + "nsimeq": '\U00002244', + "nsmid": '\U00002224', + "nspar": '\U00002226', + "nsqsube": '\U000022E2', + "nsqsupe": '\U000022E3', + "nsub": '\U00002284', + "nsube": '\U00002288', + "nsubseteq": '\U00002288', + "nsucc": '\U00002281', + "nsup": '\U00002285', + "nsupe": '\U00002289', + "nsupseteq": '\U00002289', + "ntgl": '\U00002279', + "ntilde": '\U000000F1', + "ntlg": '\U00002278', + "ntriangleleft": '\U000022EA', + "ntrianglelefteq": '\U000022EC', + "ntriangleright": '\U000022EB', + "ntrianglerighteq": '\U000022ED', + "nu": '\U000003BD', + "num": '\U00000023', + "numero": '\U00002116', + "numsp": '\U00002007', + "nvDash": '\U000022AD', + "nvHarr": '\U00002904', + "nvdash": '\U000022AC', + "nvinfin": '\U000029DE', + "nvlArr": '\U00002902', + "nvrArr": '\U00002903', + "nwArr": '\U000021D6', + "nwarhk": '\U00002923', + "nwarr": '\U00002196', + "nwarrow": '\U00002196', + "nwnear": '\U00002927', + "oS": '\U000024C8', + "oacute": '\U000000F3', + "oast": '\U0000229B', + "ocir": '\U0000229A', + "ocirc": '\U000000F4', + "ocy": '\U0000043E', + "odash": '\U0000229D', + "odblac": '\U00000151', + "odiv": '\U00002A38', + "odot": '\U00002299', + "odsold": '\U000029BC', + "oelig": '\U00000153', + "ofcir": '\U000029BF', + "ofr": '\U0001D52C', + "ogon": '\U000002DB', + "ograve": '\U000000F2', + "ogt": '\U000029C1', + "ohbar": '\U000029B5', + "ohm": '\U000003A9', + "oint": '\U0000222E', + "olarr": '\U000021BA', + "olcir": '\U000029BE', + "olcross": '\U000029BB', + "oline": '\U0000203E', + "olt": '\U000029C0', + "omacr": '\U0000014D', + "omega": '\U000003C9', + "omicron": '\U000003BF', + "omid": '\U000029B6', + "ominus": '\U00002296', + "oopf": '\U0001D560', + "opar": '\U000029B7', + "operp": '\U000029B9', + "oplus": '\U00002295', + "or": '\U00002228', + "orarr": '\U000021BB', + "ord": '\U00002A5D', + "order": '\U00002134', + "orderof": '\U00002134', + "ordf": '\U000000AA', + "ordm": '\U000000BA', + "origof": '\U000022B6', + "oror": '\U00002A56', + "orslope": '\U00002A57', + "orv": '\U00002A5B', + "oscr": '\U00002134', + "oslash": '\U000000F8', + "osol": '\U00002298', + "otilde": '\U000000F5', + "otimes": '\U00002297', + "otimesas": '\U00002A36', + "ouml": '\U000000F6', + "ovbar": '\U0000233D', + "par": '\U00002225', + "para": '\U000000B6', + "parallel": '\U00002225', + "parsim": '\U00002AF3', + "parsl": '\U00002AFD', + "part": '\U00002202', + "pcy": '\U0000043F', + "percnt": '\U00000025', + "period": '\U0000002E', + "permil": '\U00002030', + "perp": '\U000022A5', + "pertenk": '\U00002031', + "pfr": '\U0001D52D', + "phi": '\U000003C6', + "phiv": '\U000003D5', + "phmmat": '\U00002133', + "phone": '\U0000260E', + "pi": '\U000003C0', + "pitchfork": '\U000022D4', + "piv": '\U000003D6', + "planck": '\U0000210F', + "planckh": '\U0000210E', + "plankv": '\U0000210F', + "plus": '\U0000002B', + "plusacir": '\U00002A23', + "plusb": '\U0000229E', + "pluscir": '\U00002A22', + "plusdo": '\U00002214', + "plusdu": '\U00002A25', + "pluse": '\U00002A72', + "plusmn": '\U000000B1', + "plussim": '\U00002A26', + "plustwo": '\U00002A27', + "pm": '\U000000B1', + "pointint": '\U00002A15', + "popf": '\U0001D561', + "pound": '\U000000A3', + "pr": '\U0000227A', + "prE": '\U00002AB3', + "prap": '\U00002AB7', + "prcue": '\U0000227C', + "pre": '\U00002AAF', + "prec": '\U0000227A', + "precapprox": '\U00002AB7', + "preccurlyeq": '\U0000227C', + "preceq": '\U00002AAF', + "precnapprox": '\U00002AB9', + "precneqq": '\U00002AB5', + "precnsim": '\U000022E8', + "precsim": '\U0000227E', + "prime": '\U00002032', + "primes": '\U00002119', + "prnE": '\U00002AB5', + "prnap": '\U00002AB9', + "prnsim": '\U000022E8', + "prod": '\U0000220F', + "profalar": '\U0000232E', + "profline": '\U00002312', + "profsurf": '\U00002313', + "prop": '\U0000221D', + "propto": '\U0000221D', + "prsim": '\U0000227E', + "prurel": '\U000022B0', + "pscr": '\U0001D4C5', + "psi": '\U000003C8', + "puncsp": '\U00002008', + "qfr": '\U0001D52E', + "qint": '\U00002A0C', + "qopf": '\U0001D562', + "qprime": '\U00002057', + "qscr": '\U0001D4C6', + "quaternions": '\U0000210D', + "quatint": '\U00002A16', + "quest": '\U0000003F', + "questeq": '\U0000225F', + "quot": '\U00000022', + "rAarr": '\U000021DB', + "rArr": '\U000021D2', + "rAtail": '\U0000291C', + "rBarr": '\U0000290F', + "rHar": '\U00002964', + "racute": '\U00000155', + "radic": '\U0000221A', + "raemptyv": '\U000029B3', + "rang": '\U000027E9', + "rangd": '\U00002992', + "range": '\U000029A5', + "rangle": '\U000027E9', + "raquo": '\U000000BB', + "rarr": '\U00002192', + "rarrap": '\U00002975', + "rarrb": '\U000021E5', + "rarrbfs": '\U00002920', + "rarrc": '\U00002933', + "rarrfs": '\U0000291E', + "rarrhk": '\U000021AA', + "rarrlp": '\U000021AC', + "rarrpl": '\U00002945', + "rarrsim": '\U00002974', + "rarrtl": '\U000021A3', + "rarrw": '\U0000219D', + "ratail": '\U0000291A', + "ratio": '\U00002236', + "rationals": '\U0000211A', + "rbarr": '\U0000290D', + "rbbrk": '\U00002773', + "rbrace": '\U0000007D', + "rbrack": '\U0000005D', + "rbrke": '\U0000298C', + "rbrksld": '\U0000298E', + "rbrkslu": '\U00002990', + "rcaron": '\U00000159', + "rcedil": '\U00000157', + "rceil": '\U00002309', + "rcub": '\U0000007D', + "rcy": '\U00000440', + "rdca": '\U00002937', + "rdldhar": '\U00002969', + "rdquo": '\U0000201D', + "rdquor": '\U0000201D', + "rdsh": '\U000021B3', + "real": '\U0000211C', + "realine": '\U0000211B', + "realpart": '\U0000211C', + "reals": '\U0000211D', + "rect": '\U000025AD', + "reg": '\U000000AE', + "rfisht": '\U0000297D', + "rfloor": '\U0000230B', + "rfr": '\U0001D52F', + "rhard": '\U000021C1', + "rharu": '\U000021C0', + "rharul": '\U0000296C', + "rho": '\U000003C1', + "rhov": '\U000003F1', + "rightarrow": '\U00002192', + "rightarrowtail": '\U000021A3', + "rightharpoondown": '\U000021C1', + "rightharpoonup": '\U000021C0', + "rightleftarrows": '\U000021C4', + "rightleftharpoons": '\U000021CC', + "rightrightarrows": '\U000021C9', + "rightsquigarrow": '\U0000219D', + "rightthreetimes": '\U000022CC', + "ring": '\U000002DA', + "risingdotseq": '\U00002253', + "rlarr": '\U000021C4', + "rlhar": '\U000021CC', + "rlm": '\U0000200F', + "rmoust": '\U000023B1', + "rmoustache": '\U000023B1', + "rnmid": '\U00002AEE', + "roang": '\U000027ED', + "roarr": '\U000021FE', + "robrk": '\U000027E7', + "ropar": '\U00002986', + "ropf": '\U0001D563', + "roplus": '\U00002A2E', + "rotimes": '\U00002A35', + "rpar": '\U00000029', + "rpargt": '\U00002994', + "rppolint": '\U00002A12', + "rrarr": '\U000021C9', + "rsaquo": '\U0000203A', + "rscr": '\U0001D4C7', + "rsh": '\U000021B1', + "rsqb": '\U0000005D', + "rsquo": '\U00002019', + "rsquor": '\U00002019', + "rthree": '\U000022CC', + "rtimes": '\U000022CA', + "rtri": '\U000025B9', + "rtrie": '\U000022B5', + "rtrif": '\U000025B8', + "rtriltri": '\U000029CE', + "ruluhar": '\U00002968', + "rx": '\U0000211E', + "sacute": '\U0000015B', + "sbquo": '\U0000201A', + "sc": '\U0000227B', + "scE": '\U00002AB4', + "scap": '\U00002AB8', + "scaron": '\U00000161', + "sccue": '\U0000227D', + "sce": '\U00002AB0', + "scedil": '\U0000015F', + "scirc": '\U0000015D', + "scnE": '\U00002AB6', + "scnap": '\U00002ABA', + "scnsim": '\U000022E9', + "scpolint": '\U00002A13', + "scsim": '\U0000227F', + "scy": '\U00000441', + "sdot": '\U000022C5', + "sdotb": '\U000022A1', + "sdote": '\U00002A66', + "seArr": '\U000021D8', + "searhk": '\U00002925', + "searr": '\U00002198', + "searrow": '\U00002198', + "sect": '\U000000A7', + "semi": '\U0000003B', + "seswar": '\U00002929', + "setminus": '\U00002216', + "setmn": '\U00002216', + "sext": '\U00002736', + "sfr": '\U0001D530', + "sfrown": '\U00002322', + "sharp": '\U0000266F', + "shchcy": '\U00000449', + "shcy": '\U00000448', + "shortmid": '\U00002223', + "shortparallel": '\U00002225', + "shy": '\U000000AD', + "sigma": '\U000003C3', + "sigmaf": '\U000003C2', + "sigmav": '\U000003C2', + "sim": '\U0000223C', + "simdot": '\U00002A6A', + "sime": '\U00002243', + "simeq": '\U00002243', + "simg": '\U00002A9E', + "simgE": '\U00002AA0', + "siml": '\U00002A9D', + "simlE": '\U00002A9F', + "simne": '\U00002246', + "simplus": '\U00002A24', + "simrarr": '\U00002972', + "slarr": '\U00002190', + "smallsetminus": '\U00002216', + "smashp": '\U00002A33', + "smeparsl": '\U000029E4', + "smid": '\U00002223', + "smile": '\U00002323', + "smt": '\U00002AAA', + "smte": '\U00002AAC', + "softcy": '\U0000044C', + "sol": '\U0000002F', + "solb": '\U000029C4', + "solbar": '\U0000233F', + "sopf": '\U0001D564', + "spades": '\U00002660', + "spadesuit": '\U00002660', + "spar": '\U00002225', + "sqcap": '\U00002293', + "sqcup": '\U00002294', + "sqsub": '\U0000228F', + "sqsube": '\U00002291', + "sqsubset": '\U0000228F', + "sqsubseteq": '\U00002291', + "sqsup": '\U00002290', + "sqsupe": '\U00002292', + "sqsupset": '\U00002290', + "sqsupseteq": '\U00002292', + "squ": '\U000025A1', + "square": '\U000025A1', + "squarf": '\U000025AA', + "squf": '\U000025AA', + "srarr": '\U00002192', + "sscr": '\U0001D4C8', + "ssetmn": '\U00002216', + "ssmile": '\U00002323', + "sstarf": '\U000022C6', + "star": '\U00002606', + "starf": '\U00002605', + "straightepsilon": '\U000003F5', + "straightphi": '\U000003D5', + "strns": '\U000000AF', + "sub": '\U00002282', + "subE": '\U00002AC5', + "subdot": '\U00002ABD', + "sube": '\U00002286', + "subedot": '\U00002AC3', + "submult": '\U00002AC1', + "subnE": '\U00002ACB', + "subne": '\U0000228A', + "subplus": '\U00002ABF', + "subrarr": '\U00002979', + "subset": '\U00002282', + "subseteq": '\U00002286', + "subseteqq": '\U00002AC5', + "subsetneq": '\U0000228A', + "subsetneqq": '\U00002ACB', + "subsim": '\U00002AC7', + "subsub": '\U00002AD5', + "subsup": '\U00002AD3', + "succ": '\U0000227B', + "succapprox": '\U00002AB8', + "succcurlyeq": '\U0000227D', + "succeq": '\U00002AB0', + "succnapprox": '\U00002ABA', + "succneqq": '\U00002AB6', + "succnsim": '\U000022E9', + "succsim": '\U0000227F', + "sum": '\U00002211', + "sung": '\U0000266A', + "sup": '\U00002283', + "sup1": '\U000000B9', + "sup2": '\U000000B2', + "sup3": '\U000000B3', + "supE": '\U00002AC6', + "supdot": '\U00002ABE', + "supdsub": '\U00002AD8', + "supe": '\U00002287', + "supedot": '\U00002AC4', + "suphsol": '\U000027C9', + "suphsub": '\U00002AD7', + "suplarr": '\U0000297B', + "supmult": '\U00002AC2', + "supnE": '\U00002ACC', + "supne": '\U0000228B', + "supplus": '\U00002AC0', + "supset": '\U00002283', + "supseteq": '\U00002287', + "supseteqq": '\U00002AC6', + "supsetneq": '\U0000228B', + "supsetneqq": '\U00002ACC', + "supsim": '\U00002AC8', + "supsub": '\U00002AD4', + "supsup": '\U00002AD6', + "swArr": '\U000021D9', + "swarhk": '\U00002926', + "swarr": '\U00002199', + "swarrow": '\U00002199', + "swnwar": '\U0000292A', + "szlig": '\U000000DF', + "target": '\U00002316', + "tau": '\U000003C4', + "tbrk": '\U000023B4', + "tcaron": '\U00000165', + "tcedil": '\U00000163', + "tcy": '\U00000442', + "tdot": '\U000020DB', + "telrec": '\U00002315', + "tfr": '\U0001D531', + "there4": '\U00002234', + "therefore": '\U00002234', + "theta": '\U000003B8', + "thetasym": '\U000003D1', + "thetav": '\U000003D1', + "thickapprox": '\U00002248', + "thicksim": '\U0000223C', + "thinsp": '\U00002009', + "thkap": '\U00002248', + "thksim": '\U0000223C', + "thorn": '\U000000FE', + "tilde": '\U000002DC', + "times": '\U000000D7', + "timesb": '\U000022A0', + "timesbar": '\U00002A31', + "timesd": '\U00002A30', + "tint": '\U0000222D', + "toea": '\U00002928', + "top": '\U000022A4', + "topbot": '\U00002336', + "topcir": '\U00002AF1', + "topf": '\U0001D565', + "topfork": '\U00002ADA', + "tosa": '\U00002929', + "tprime": '\U00002034', + "trade": '\U00002122', + "triangle": '\U000025B5', + "triangledown": '\U000025BF', + "triangleleft": '\U000025C3', + "trianglelefteq": '\U000022B4', + "triangleq": '\U0000225C', + "triangleright": '\U000025B9', + "trianglerighteq": '\U000022B5', + "tridot": '\U000025EC', + "trie": '\U0000225C', + "triminus": '\U00002A3A', + "triplus": '\U00002A39', + "trisb": '\U000029CD', + "tritime": '\U00002A3B', + "trpezium": '\U000023E2', + "tscr": '\U0001D4C9', + "tscy": '\U00000446', + "tshcy": '\U0000045B', + "tstrok": '\U00000167', + "twixt": '\U0000226C', + "twoheadleftarrow": '\U0000219E', + "twoheadrightarrow": '\U000021A0', + "uArr": '\U000021D1', + "uHar": '\U00002963', + "uacute": '\U000000FA', + "uarr": '\U00002191', + "ubrcy": '\U0000045E', + "ubreve": '\U0000016D', + "ucirc": '\U000000FB', + "ucy": '\U00000443', + "udarr": '\U000021C5', + "udblac": '\U00000171', + "udhar": '\U0000296E', + "ufisht": '\U0000297E', + "ufr": '\U0001D532', + "ugrave": '\U000000F9', + "uharl": '\U000021BF', + "uharr": '\U000021BE', + "uhblk": '\U00002580', + "ulcorn": '\U0000231C', + "ulcorner": '\U0000231C', + "ulcrop": '\U0000230F', + "ultri": '\U000025F8', + "umacr": '\U0000016B', + "uml": '\U000000A8', + "uogon": '\U00000173', + "uopf": '\U0001D566', + "uparrow": '\U00002191', + "updownarrow": '\U00002195', + "upharpoonleft": '\U000021BF', + "upharpoonright": '\U000021BE', + "uplus": '\U0000228E', + "upsi": '\U000003C5', + "upsih": '\U000003D2', + "upsilon": '\U000003C5', + "upuparrows": '\U000021C8', + "urcorn": '\U0000231D', + "urcorner": '\U0000231D', + "urcrop": '\U0000230E', + "uring": '\U0000016F', + "urtri": '\U000025F9', + "uscr": '\U0001D4CA', + "utdot": '\U000022F0', + "utilde": '\U00000169', + "utri": '\U000025B5', + "utrif": '\U000025B4', + "uuarr": '\U000021C8', + "uuml": '\U000000FC', + "uwangle": '\U000029A7', + "vArr": '\U000021D5', + "vBar": '\U00002AE8', + "vBarv": '\U00002AE9', + "vDash": '\U000022A8', + "vangrt": '\U0000299C', + "varepsilon": '\U000003F5', + "varkappa": '\U000003F0', + "varnothing": '\U00002205', + "varphi": '\U000003D5', + "varpi": '\U000003D6', + "varpropto": '\U0000221D', + "varr": '\U00002195', + "varrho": '\U000003F1', + "varsigma": '\U000003C2', + "vartheta": '\U000003D1', + "vartriangleleft": '\U000022B2', + "vartriangleright": '\U000022B3', + "vcy": '\U00000432', + "vdash": '\U000022A2', + "vee": '\U00002228', + "veebar": '\U000022BB', + "veeeq": '\U0000225A', + "vellip": '\U000022EE', + "verbar": '\U0000007C', + "vert": '\U0000007C', + "vfr": '\U0001D533', + "vltri": '\U000022B2', + "vopf": '\U0001D567', + "vprop": '\U0000221D', + "vrtri": '\U000022B3', + "vscr": '\U0001D4CB', + "vzigzag": '\U0000299A', + "wcirc": '\U00000175', + "wedbar": '\U00002A5F', + "wedge": '\U00002227', + "wedgeq": '\U00002259', + "weierp": '\U00002118', + "wfr": '\U0001D534', + "wopf": '\U0001D568', + "wp": '\U00002118', + "wr": '\U00002240', + "wreath": '\U00002240', + "wscr": '\U0001D4CC', + "xcap": '\U000022C2', + "xcirc": '\U000025EF', + "xcup": '\U000022C3', + "xdtri": '\U000025BD', + "xfr": '\U0001D535', + "xhArr": '\U000027FA', + "xharr": '\U000027F7', + "xi": '\U000003BE', + "xlArr": '\U000027F8', + "xlarr": '\U000027F5', + "xmap": '\U000027FC', + "xnis": '\U000022FB', + "xodot": '\U00002A00', + "xopf": '\U0001D569', + "xoplus": '\U00002A01', + "xotime": '\U00002A02', + "xrArr": '\U000027F9', + "xrarr": '\U000027F6', + "xscr": '\U0001D4CD', + "xsqcup": '\U00002A06', + "xuplus": '\U00002A04', + "xutri": '\U000025B3', + "xvee": '\U000022C1', + "xwedge": '\U000022C0', + "yacute": '\U000000FD', + "yacy": '\U0000044F', + "ycirc": '\U00000177', + "ycy": '\U0000044B', + "yen": '\U000000A5', + "yfr": '\U0001D536', + "yicy": '\U00000457', + "yopf": '\U0001D56A', + "yscr": '\U0001D4CE', + "yucy": '\U0000044E', + "yuml": '\U000000FF', + "zacute": '\U0000017A', + "zcaron": '\U0000017E', + "zcy": '\U00000437', + "zdot": '\U0000017C', + "zeetrf": '\U00002128', + "zeta": '\U000003B6', + "zfr": '\U0001D537', + "zhcy": '\U00000436', + "zigrarr": '\U000021DD', + "zopf": '\U0001D56B', + "zscr": '\U0001D4CF', + "zwj": '\U0000200D', + "zwnj": '\U0000200C', +} diff --git a/vendor/github.com/k3a/html2text/html2text.go b/vendor/github.com/k3a/html2text/html2text.go new file mode 100644 index 0000000000..f79fbe395e --- /dev/null +++ b/vendor/github.com/k3a/html2text/html2text.go @@ -0,0 +1,333 @@ +package html2text + +import ( + "bytes" + "regexp" + "strconv" + "strings" +) + +// Line break constants +// Deprecated: Please use HTML2TextWithOptions(text, WithUnixLineBreak()) +const ( + WIN_LBR = "\r\n" + UNIX_LBR = "\n" +) + +var legacyLBR = WIN_LBR +var badTagnamesRE = regexp.MustCompile(`^(head|script|style|a)($|\s+)`) +var linkTagRE = regexp.MustCompile(`^(?i:a)(?:$|\s).*(?i:href)\s*=\s*('([^']*?)'|"([^"]*?)"|([^\s"'` + "`" + `=<>]+))`) +var badLinkHrefRE = regexp.MustCompile(`javascript:`) +var headersRE = regexp.MustCompile(`^(\/)?h[1-6]`) +var numericEntityRE = regexp.MustCompile(`(?i)^#(x?[a-f0-9]+)$`) + +type options struct { + lbr string + linksInnerText bool + listPrefix string +} + +func newOptions() *options { + // apply defaults + return &options{ + lbr: WIN_LBR, + } +} + +// Option is a functional option +type Option func(*options) + +// WithUnixLineBreaks instructs the converter to use unix line breaks ("\n" instead of "\r\n" default) +func WithUnixLineBreaks() Option { + return func(o *options) { + o.lbr = UNIX_LBR + } +} + +// WithLinksInnerText instructs the converter to retain link tag inner text and append href URLs in angle brackets after the text +// Example: click news +func WithLinksInnerText() Option { + return func(o *options) { + o.linksInnerText = true + } +} + +// WithListSupportPrefix formats
    and
  • lists with the specified prefix +func WithListSupportPrefix(prefix string) Option { + return func(o *options) { + o.listPrefix = prefix + } +} + +// WithListSupport formats
      and
    • lists with " - " prefix +func WithListSupport() Option { + return WithListSupportPrefix(" - ") +} + +func parseHTMLEntity(entName string) (string, bool) { + if r, ok := entity[entName]; ok { + return string(r), true + } + + if match := numericEntityRE.FindStringSubmatch(entName); len(match) == 2 { + var ( + err error + n int64 + digits = match[1] + ) + + if digits != "" && (digits[0] == 'x' || digits[0] == 'X') { + n, err = strconv.ParseInt(digits[1:], 16, 64) + } else { + n, err = strconv.ParseInt(digits, 10, 64) + } + + if err == nil && (n == 9 || n == 10 || n == 13 || n > 31) { + return string(rune(n)), true + } + } + + return "", false +} + +// SetUnixLbr with argument true sets Unix-style line-breaks in output ("\n") +// with argument false sets Windows-style line-breaks in output ("\r\n", the default) +// Deprecated: Please use HTML2TextWithOptions(text, WithUnixLineBreak()) +func SetUnixLbr(b bool) { + if b { + legacyLBR = UNIX_LBR + } else { + legacyLBR = WIN_LBR + } +} + +// HTMLEntitiesToText decodes HTML entities inside a provided +// string and returns decoded text +func HTMLEntitiesToText(htmlEntsText string) string { + outBuf := bytes.NewBufferString("") + inEnt := false + + for i, r := range htmlEntsText { + switch { + case r == ';' && inEnt: + inEnt = false + continue + + case r == '&': //possible html entity + entName := "" + isEnt := false + + // parse the entity name - max 10 chars + chars := 0 + for _, er := range htmlEntsText[i+1:] { + if er == ';' { + isEnt = true + break + } else { + entName += string(er) + } + + chars++ + if chars == 10 { + break + } + } + + if isEnt { + if ent, isEnt := parseHTMLEntity(entName); isEnt { + outBuf.WriteString(ent) + inEnt = true + continue + } + } + } + + if !inEnt { + outBuf.WriteRune(r) + } + } + + return outBuf.String() +} + +func writeSpace(outBuf *bytes.Buffer) { + bts := outBuf.Bytes() + if len(bts) > 0 && bts[len(bts)-1] != ' ' { + outBuf.WriteString(" ") + } +} + +// HTML2Text converts html into a text form +func HTML2Text(html string) string { + var opts []Option + if legacyLBR == UNIX_LBR { + opts = append(opts, WithUnixLineBreaks()) + } + return HTML2TextWithOptions(html, opts...) +} + +// HTML2TextWithOptions converts html into a text form with additional options +func HTML2TextWithOptions(html string, reqOpts ...Option) string { + opts := newOptions() + for _, opt := range reqOpts { + opt(opts) + } + + inLen := len(html) + tagStart := 0 + inEnt := false + badTagStackDepth := 0 // if == 1 it means we are inside ... + shouldOutput := true + // maintain a stack of tag href links and output it after the tag's inner text (for opts.linksInnerText only) + hrefs := []string{} + // new line cannot be printed at the beginning or + // for

      after a new line created by previous

      + canPrintNewline := false + + outBuf := bytes.NewBufferString("") + + for i, r := range html { + if inLen > 0 && i == inLen-1 { + // prevent new line at the end of the document + canPrintNewline = false + } + + switch { + // skip new lines and spaces adding a single space if not there yet + case r <= 0xD, r == 0x85, r == 0x2028, r == 0x2029, // new lines + r == ' ', r >= 0x2008 && r <= 0x200B: // spaces + if shouldOutput && badTagStackDepth == 0 && !inEnt { + //outBuf.WriteString(fmt.Sprintf("{DBG r:%c, inEnt:%t, tag:%s}", r, inEnt, html[tagStart:i])) + writeSpace(outBuf) + } + continue + + case r == ';' && inEnt: // end of html entity + inEnt = false + continue + + case r == '&' && shouldOutput: // possible html entity + entName := "" + isEnt := false + + // parse the entity name - max 10 chars + chars := 0 + for _, er := range html[i+1:] { + if er == ';' { + isEnt = true + break + } else { + entName += string(er) + } + + chars++ + if chars == 10 { + break + } + } + + if isEnt { + if ent, isEnt := parseHTMLEntity(entName); isEnt { + outBuf.WriteString(ent) + inEnt = true + continue + } + } + + case r == '<': // start of a tag + tagStart = i + 1 + shouldOutput = false + continue + + case r == '>': // end of a tag + shouldOutput = true + tag := html[tagStart:i] + tagNameLowercase := strings.ToLower(tag) + + if tagNameLowercase == "/ul" || tagNameLowercase == "/ol" { + outBuf.WriteString(opts.lbr) + } else if tagNameLowercase == "li" || tagNameLowercase == "li/" { + if opts.listPrefix != "" { + outBuf.WriteString(opts.lbr + opts.listPrefix) + } else { + outBuf.WriteString(opts.lbr) + } + } else if headersRE.MatchString(tagNameLowercase) { + if canPrintNewline { + outBuf.WriteString(opts.lbr + opts.lbr) + } + canPrintNewline = false + } else if tagNameLowercase == "br" || tagNameLowercase == "br/" { + // new line + outBuf.WriteString(opts.lbr) + } else if tagNameLowercase == "p" || tagNameLowercase == "/p" { + if canPrintNewline { + outBuf.WriteString(opts.lbr + opts.lbr) + } + canPrintNewline = false + } else if opts.linksInnerText && tagNameLowercase == "/a" { + // end of link + // links can be empty can happen if the link matches the badLinkHrefRE + if len(hrefs) > 0 { + outBuf.WriteString(" <") + outBuf.WriteString(HTMLEntitiesToText(hrefs[0])) + outBuf.WriteString(">") + hrefs = hrefs[1:] + } + } else if opts.linksInnerText && linkTagRE.MatchString(tagNameLowercase) { + // parse link href + // add special handling for a tags + m := linkTagRE.FindStringSubmatch(tag) + if len(m) == 5 { + link := m[2] + if len(link) == 0 { + link = m[3] + if len(link) == 0 { + link = m[4] + } + } + + if opts.linksInnerText && !badLinkHrefRE.MatchString(link) { + hrefs = append(hrefs, link) + } + } + } else if badTagnamesRE.MatchString(tagNameLowercase) { + // unwanted block + badTagStackDepth++ + + // if link inner text preservation is not enabled + // and the current tag is a link tag, parse its href and output that + if !opts.linksInnerText { + // parse link href + m := linkTagRE.FindStringSubmatch(tag) + if len(m) == 5 { + link := m[2] + if len(link) == 0 { + link = m[3] + if len(link) == 0 { + link = m[4] + } + } + + if !badLinkHrefRE.MatchString(link) { + outBuf.WriteString(HTMLEntitiesToText(link)) + } + } + } + } else if len(tagNameLowercase) > 0 && tagNameLowercase[0] == '/' && + badTagnamesRE.MatchString(tagNameLowercase[1:]) { + // end of unwanted block + badTagStackDepth-- + } + continue + + } // switch end + + if shouldOutput && badTagStackDepth == 0 && !inEnt { + canPrintNewline = true + outBuf.WriteRune(r) + } + } + + return outBuf.String() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7cd7827d42..a0734eabb3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -140,6 +140,9 @@ github.com/bytedance/sonic/loader/internal/rt # github.com/cenkalti/backoff/v4 v4.3.0 ## explicit; go 1.18 github.com/cenkalti/backoff/v4 +# github.com/cespare/xxhash v1.1.0 +## explicit +github.com/cespare/xxhash # github.com/cespare/xxhash/v2 v2.3.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 @@ -446,6 +449,9 @@ github.com/josharian/intern # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go +# github.com/k3a/html2text v1.2.1 +## explicit; go 1.16 +github.com/k3a/html2text # github.com/klauspost/compress v1.17.9 ## explicit; go 1.20 github.com/klauspost/compress From 2c24409d50924bca66bff3202621b572fbcfc2cb Mon Sep 17 00:00:00 2001 From: tobi Date: Sun, 15 Sep 2024 16:20:05 +0200 Subject: [PATCH 02/11] add new deps to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 755a9f79f1..527344d84c 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,7 @@ For bugs and feature requests, please check to see if there's [already an issue] The following open source libraries, frameworks, and tools are used by GoToSocial, with gratitude 💕 - [buckket/go-blurhash](https://github.com/buckket/go-blurhash); used for generating image blurhashes. [GPL-3.0 License](https://spdx.org/licenses/GPL-3.0-only.html). +- [cespare/xxhash](https://github.com/cespare/xxhash); xxHash generation. [MIT License](https://spdx.org/licenses/MIT.html). - [coreos/go-oidc](https://github.com/coreos/go-oidc); OIDC client library. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html). - [DmitriyVTitov/size](https://github.com/DmitriyVTitov/size); runtime model memory size calculations. [MIT License](https://spdx.org/licenses/MIT.html). - Gin: @@ -273,6 +274,7 @@ The following open source libraries, frameworks, and tools are used by GoToSocia - [jackc/pgconn](https://github.com/jackc/pgconn); Postgres driver. [MIT License](https://spdx.org/licenses/MIT.html). - [jackc/pgx](https://github.com/jackc/pgx); Postgres driver and toolkit. [MIT License](https://spdx.org/licenses/MIT.html). - [KimMachineGun/automemlimit](https://github.com/KimMachineGun/automemlimit); cgroups memory limit checking. [MIT License](https://spdx.org/licenses/MIT.html). +- [k3a/html2text](https://github.com/k3a/html2text); HTML-to-text conversion. [MIT License](https://spdx.org/licenses/MIT.html). - [mcuadros/go-syslog](https://github.com/mcuadros/go-syslog); Syslog server library. [MIT License](https://spdx.org/licenses/MIT.html). - [microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday); HTML user-input sanitization. [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). - [miekg/dns](https://github.com/miekg/dns); DNS utilities. [Go License](https://go.dev/LICENSE). From 2b5ff69db3bed5477fc6217ef99812edbf0ce65d Mon Sep 17 00:00:00 2001 From: tobi Date: Sun, 15 Sep 2024 16:23:03 +0200 Subject: [PATCH 03/11] lint --- internal/typeutils/util.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/internal/typeutils/util.go b/internal/typeutils/util.go index f660efc842..bb61ffb66e 100644 --- a/internal/typeutils/util.go +++ b/internal/typeutils/util.go @@ -299,21 +299,21 @@ func StatusHash(s *gtsmodel.Status) string { hash := xxhash.New() // Content warning / title. - hash.WriteString(s.ContentWarning) + hash.WriteString(s.ContentWarning) // nolint:errcheck // Status content. - hash.WriteString(s.Content) + hash.WriteString(s.Content) // nolint:errcheck // Media IDs + descriptions. for _, attachment := range s.Attachments { - hash.WriteString(attachment.ID) - hash.WriteString(attachment.Description) + hash.WriteString(attachment.ID) // nolint:errcheck + hash.WriteString(attachment.Description) // nolint:errcheck } // Poll options. if s.Poll != nil { for _, option := range s.Poll.Options { - hash.WriteString(option) + hash.WriteString(option) // nolint:errcheck } } @@ -354,9 +354,7 @@ func filterableText(s *gtsmodel.Status) string { // Poll options. if s.Poll != nil { - for _, option := range s.Poll.Options { - fields = append(fields, option) - } + fields = append(fields, s.Poll.Options...) } return strings.Join(fields, " ") From 3cb6729f1b5cbda9aff03c87bc087411989d7b1a Mon Sep 17 00:00:00 2001 From: tobi Date: Sun, 15 Sep 2024 17:17:26 +0200 Subject: [PATCH 04/11] update tests --- internal/typeutils/internaltofrontend_test.go | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index 651ff867d1..569aef83ad 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -1063,15 +1063,22 @@ func (suite *InternalToFrontendTestSuite) TestHideFilteredBoostToFrontend() { // Test that a hashtag filter for a hashtag in Mastodon HTML content works the way most users would expect. func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wholeWord bool, boost bool) { - testStatus := suite.testStatuses["admin_account_status_1"] + testStatus := new(gtsmodel.Status) + *testStatus = *suite.testStatuses["admin_account_status_1"] testStatus.Content = `

      doggo doggin' it

      #dogsofmastodon

      ` + testStatus.Text = "doggo doggin' it\n\n#dogsofmastodon" if boost { - // Modify a fixture boost into a boost of the above status. - boostStatus := suite.testStatuses["admin_account_status_4"] - boostStatus.BoostOf = testStatus - boostStatus.BoostOfID = testStatus.ID - testStatus = boostStatus + boost, err := suite.typeconverter.StatusToBoost( + context.Background(), + testStatus, + suite.testAccounts["admin_account"], + "", + ) + if err != nil { + suite.FailNow(err.Error()) + } + testStatus = boost } requestingAccount := suite.testAccounts["local_account_1"] @@ -1103,9 +1110,11 @@ func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wh []*gtsmodel.Filter{filter}, nil, ) - if suite.NoError(err) { - suite.NotEmpty(apiStatus.Filtered) + if err != nil { + suite.FailNow(err.Error()) } + + suite.NotEmpty(apiStatus.Filtered) } func (suite *InternalToFrontendTestSuite) TestHashtagWholeWordFilteredStatusToFrontend() { From eb08ebe179425053d4822d4ac0ccadec6f245299 Mon Sep 17 00:00:00 2001 From: tobi Date: Sun, 15 Sep 2024 17:17:35 +0200 Subject: [PATCH 05/11] update regexes --- internal/gtsmodel/filter.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/gtsmodel/filter.go b/internal/gtsmodel/filter.go index e670e6fc0e..95047a44fb 100644 --- a/internal/gtsmodel/filter.go +++ b/internal/gtsmodel/filter.go @@ -20,6 +20,8 @@ package gtsmodel import ( "regexp" "time" + + "github.com/superseriousbusiness/gotosocial/internal/util" ) // Filter stores a filter created by a local account. @@ -61,14 +63,23 @@ type FilterKeyword struct { // Compile will compile this FilterKeyword as a prepared regular expression. func (k *FilterKeyword) Compile() (err error) { - var wordBreak string - if k.WholeWord != nil && *k.WholeWord { - wordBreak = `\b` + var ( + wordBreakStart string + wordBreakEnd string + ) + + if util.PtrOrZero(k.WholeWord) { + // Either word boundary or + // whitespace or start of line. + wordBreakStart = `(?:\b|\s|^)` + // Either word boundary or + // whitespace or end of line. + wordBreakEnd = `(?:\b|\s|$)` } // Compile keyword filter regexp. quoted := regexp.QuoteMeta(k.Keyword) - k.Regexp, err = regexp.Compile(`(?i)` + wordBreak + quoted + wordBreak) + k.Regexp, err = regexp.Compile(`(?i)` + wordBreakStart + quoted + wordBreakEnd) return // caller is expected to wrap this error } From ab4089d674ae9a70d98cc9ff5af956d6f256cbe6 Mon Sep 17 00:00:00 2001 From: tobi Date: Mon, 16 Sep 2024 11:21:16 +0200 Subject: [PATCH 06/11] address review comments --- .drone.yml | 4 +- .goreleaser.yml | 1 - go.mod | 1 - go.sum | 6 - internal/typeutils/converter.go | 28 ++- internal/typeutils/internaltofrontend.go | 41 +-- internal/typeutils/internaltofrontend_test.go | 1 - internal/typeutils/util.go | 85 +++---- internal/typeutils/util_test.go | 87 +++---- scripts/build.sh | 3 +- vendor/github.com/cespare/xxhash/LICENSE.txt | 22 -- vendor/github.com/cespare/xxhash/README.md | 50 ---- vendor/github.com/cespare/xxhash/rotate.go | 14 -- vendor/github.com/cespare/xxhash/rotate19.go | 14 -- vendor/github.com/cespare/xxhash/xxhash.go | 168 ------------- .../github.com/cespare/xxhash/xxhash_amd64.go | 12 - .../github.com/cespare/xxhash/xxhash_amd64.s | 233 ------------------ .../github.com/cespare/xxhash/xxhash_other.go | 75 ------ .../github.com/cespare/xxhash/xxhash_safe.go | 10 - .../cespare/xxhash/xxhash_unsafe.go | 30 --- vendor/modules.txt | 3 - 21 files changed, 112 insertions(+), 776 deletions(-) delete mode 100644 vendor/github.com/cespare/xxhash/LICENSE.txt delete mode 100644 vendor/github.com/cespare/xxhash/README.md delete mode 100644 vendor/github.com/cespare/xxhash/rotate.go delete mode 100644 vendor/github.com/cespare/xxhash/rotate19.go delete mode 100644 vendor/github.com/cespare/xxhash/xxhash.go delete mode 100644 vendor/github.com/cespare/xxhash/xxhash_amd64.go delete mode 100644 vendor/github.com/cespare/xxhash/xxhash_amd64.s delete mode 100644 vendor/github.com/cespare/xxhash/xxhash_other.go delete mode 100644 vendor/github.com/cespare/xxhash/xxhash_safe.go delete mode 100644 vendor/github.com/cespare/xxhash/xxhash_unsafe.go diff --git a/.drone.yml b/.drone.yml index ea945db12f..1a5239a13e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -45,7 +45,7 @@ steps: go test -failfast -timeout=20m - -tags "netgo osusergo static_build kvformat timetzdata purego" + -tags "netgo osusergo static_build kvformat timetzdata" ./... - ./test/envparsing.sh - ./test/swagger.sh @@ -207,6 +207,6 @@ steps: --- kind: signature -hmac: 3f3a24557b67760dd0c4091eaaed4842b0545f5aa65f90ce70d5e45da23c5260 +hmac: f4008d87e4e5b67251eb89f255c1224e6ab5818828cab24fc319b8f829176058 ... diff --git a/.goreleaser.yml b/.goreleaser.yml index 3d1bedd11e..6a7fccfd09 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -27,7 +27,6 @@ builds: - static_build - kvformat - timetzdata - - purego - >- {{ if and (index .Env "DEBUG") (.Env.DEBUG) }}debugenv{{ end }} - >- diff --git a/go.mod b/go.mod index 8d23218d62..67fe84d221 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/DmitriyVTitov/size v1.5.0 github.com/KimMachineGun/automemlimit v0.6.1 github.com/buckket/go-blurhash v1.1.0 - github.com/cespare/xxhash v1.1.0 github.com/coreos/go-oidc/v3 v3.11.0 github.com/gin-contrib/cors v1.7.2 github.com/gin-contrib/gzip v1.0.1 diff --git a/go.sum b/go.sum index db3fc7812c..749487a685 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,6 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= @@ -120,8 +118,6 @@ github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -512,8 +508,6 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index a4395163da..8284bda96c 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -29,26 +29,30 @@ import ( ) type Converter struct { - state *state.State - defaultAvatars []string - randAvatars sync.Map - visFilter *visibility.Filter - intFilter *interaction.Filter - statusHashesToFilterableText cache.TTLCache[string, string] + state *state.State + defaultAvatars []string + randAvatars sync.Map + visFilter *visibility.Filter + intFilter *interaction.Filter + + // TTL cache of statuses -> filterable text fields. + // To ensure up-to-date fields, cache is keyed as: + // [status.ID][status.UpdatedAt.Unix()]` + statusesFilterableFields cache.TTLCache[string, []string] } func NewConverter(state *state.State) *Converter { - statusHashesToFilterableText := cache.NewTTL[string, string](0, 512, 0) + statusHashesToFilterableText := cache.NewTTL[string, []string](0, 512, 0) statusHashesToFilterableText.SetTTL(time.Hour, true) if !statusHashesToFilterableText.Start(time.Minute) { log.Panic(nil, "failed to start statusHashesToFilterableText cache") } return &Converter{ - state: state, - defaultAvatars: populateDefaultAvatars(), - visFilter: visibility.NewFilter(state), - intFilter: interaction.NewFilter(state), - statusHashesToFilterableText: statusHashesToFilterableText, + state: state, + defaultAvatars: populateDefaultAvatars(), + visFilter: visibility.NewFilter(state), + intFilter: interaction.NewFilter(state), + statusesFilterableFields: statusHashesToFilterableText, } } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 261d71bb4d..58797e7901 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" "time" @@ -938,35 +939,47 @@ func (c *Converter) statusToAPIFilterResults( return nil, nil } - // Derive a hash of this status. - statusHash := StatusHash(s) + // Key this status based on ID + last updated time, + // to ensure we always filter on latest version. + statusKey := fmt.Sprintf("%s%d", s.ID, s.UpdatedAt.Unix()) - // Check if we have the filterable - // text stored already for this hash. - statusText, stored := c.statusHashesToFilterableText.Get(statusHash) + // Check if we have filterable fields cached for this status. + fields, stored := c.statusesFilterableFields.Get(statusKey) if !stored { - // We don't have this filterable text - // cached, calculate + cache it now. - statusText = filterableText(s) - c.statusHashesToFilterableText.Set(statusHash, statusText) + // We don't have filterable fields + // cached, calculate + cache now. + fields = filterableFields(s) + c.statusesFilterableFields.Set(statusKey, fields) } // Record all matching warn filters and the reasons they matched. filterResults := make([]apimodel.FilterResult, 0, len(filters)) for _, filter := range filters { if !filterAppliesInContext(filter, filterContext) { - // Filter doesn't apply to this context. + // Filter doesn't apply + // to this context. continue } + if filter.Expired(now) { + // Filter doesn't + // apply anymore. continue } - // List all matching keywords. + // Assemble matching keywords (if any) from this filter. keywordMatches := make([]string, 0, len(filter.Keywords)) - for _, filterKeyword := range filter.Keywords { - if filterKeyword.Regexp.MatchString(statusText) { - keywordMatches = append(keywordMatches, filterKeyword.Keyword) + for _, keyword := range filter.Keywords { + // Check if at least one filterable field + // in the status matches on this filter. + if slices.ContainsFunc( + fields, + func(field string) bool { + return keyword.Regexp.MatchString(field) + }, + ) { + // At least one field matched on this filter. + keywordMatches = append(keywordMatches, keyword.Keyword) } } diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index 569aef83ad..a44afe67ea 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -1066,7 +1066,6 @@ func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wh testStatus := new(gtsmodel.Status) *testStatus = *suite.testStatuses["admin_account_status_1"] testStatus.Content = `

      doggo doggin' it

      #dogsofmastodon

      ` - testStatus.Text = "doggo doggin' it\n\n#dogsofmastodon" if boost { boost, err := suite.typeconverter.StatusToBoost( diff --git a/internal/typeutils/util.go b/internal/typeutils/util.go index bb61ffb66e..3a867ba35c 100644 --- a/internal/typeutils/util.go +++ b/internal/typeutils/util.go @@ -19,7 +19,6 @@ package typeutils import ( "context" - "encoding/hex" "fmt" "math" "net/url" @@ -28,7 +27,6 @@ import ( "strconv" "strings" - "github.com/cespare/xxhash/v2" "github.com/k3a/html2text" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" @@ -288,74 +286,63 @@ func ContentToContentLanguage( return contentStr, langTagStr } -// StatusHash returns an xxhash of text -// from a status, taking account of: -// -// - content warning -// - content -// - media IDs + descriptions -// - poll options -func StatusHash(s *gtsmodel.Status) string { - hash := xxhash.New() - - // Content warning / title. - hash.WriteString(s.ContentWarning) // nolint:errcheck - - // Status content. - hash.WriteString(s.Content) // nolint:errcheck - - // Media IDs + descriptions. - for _, attachment := range s.Attachments { - hash.WriteString(attachment.ID) // nolint:errcheck - hash.WriteString(attachment.Description) // nolint:errcheck - } - - // Poll options. - if s.Poll != nil { - for _, option := range s.Poll.Options { - hash.WriteString(option) // nolint:errcheck - } - } - - sum := hash.Sum(nil) - return hex.EncodeToString(sum) -} - -// filterableText concatenates text from a -// status that we might want to filter on: +// filterableFields returns text fields from +// a status that we might want to filter on: // // - content warning // - content (converted to plaintext from HTML) // - media descriptions // - poll options -func filterableText(s *gtsmodel.Status) string { - fields := []string{} +// +// Each field should be filtered separately. +// This avoids scenarios where false-positive +// multiple-word matches can be made by matching +// the last word of one field + the first word +// of the next field together. +func filterableFields(s *gtsmodel.Status) []string { + // Estimate length of fields. + fieldCount := 2 + len(s.Attachments) + if s.Poll != nil { + fieldCount += len(s.Poll.Options) + } + fields := make([]string, 0, fieldCount) // Content warning / title. - fields = append(fields, s.ContentWarning) + if s.ContentWarning != "" { + fields = append(fields, s.ContentWarning) + } - // Status content; use raw text if available, - // else use text parsed from content HTML. - if s.Text != "" { - fields = append(fields, s.Text) - } else { + // Status content. Though we have raw text + // available for statuses created on our + // instance, use the html2text version to + // remove markdown-formatting characters + // and ensure more consistent filtering. + if s.Content != "" { text := html2text.HTML2TextWithOptions( s.Content, html2text.WithLinksInnerText(), html2text.WithUnixLineBreaks(), ) - fields = append(fields, text) + if text != "" { + fields = append(fields, text) + } } // Media descriptions. for _, attachment := range s.Attachments { - fields = append(fields, attachment.Description) + if attachment.Description != "" { + fields = append(fields, attachment.Description) + } } // Poll options. if s.Poll != nil { - fields = append(fields, s.Poll.Options...) + for _, opt := range s.Poll.Options { + if opt != "" { + fields = append(fields, opt) + } + } } - return strings.Join(fields, " ") + return fields } diff --git a/internal/typeutils/util_test.go b/internal/typeutils/util_test.go index 23be0bbe6b..ea66675192 100644 --- a/internal/typeutils/util_test.go +++ b/internal/typeutils/util_test.go @@ -21,6 +21,7 @@ import ( "context" "testing" + "github.com/stretchr/testify/assert" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/language" @@ -161,87 +162,59 @@ func TestContentToContentLanguage(t *testing.T) { func TestFilterableText(t *testing.T) { type testcase struct { - status *gtsmodel.Status - expectedText string + status *gtsmodel.Status + expectedFields []string } - for i, testcase := range []testcase{ + for _, testcase := range []testcase{ { status: >smodel.Status{ ContentWarning: "This is a test status", Content: `

      Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.

      `, }, - expectedText: `This is a test status Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.`, + expectedFields: []string{ + "This is a test status", + "Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.", + }, }, { status: >smodel.Status{ Content: `

      @zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)

      https://github.com/superseriousbusiness/gotosocial/pull/2863

      `, }, - expectedText: ` @zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag) - -https://github.com/superseriousbusiness/gotosocial/pull/2863 `, + expectedFields: []string{ + "@zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)\n\nhttps://github.com/superseriousbusiness/gotosocial/pull/2863 ", + }, }, { status: >smodel.Status{ - Content: `

      Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg, both running on Wazero, and configured with a 50MiB db cache target. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.

      `, + ContentWarning: "Nerd stuff", + Content: `

      Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg, both running on Wazero, and configured with a 50MiB db cache target. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.

      `, Attachments: []*gtsmodel.MediaAttachment{ { Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`, }, - }, - }, - expectedText: ` Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg , both running on Wazero , and configured with a 50MiB db cache target . This is the version we'll be releasing soonish, now we're happy with how we've tamed everything. Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`, - }, - } { - text := filterableText(testcase.status) - if text != testcase.expectedText { - t.Errorf( - "test %d expected text '%s' got '%s'", - i, testcase.expectedText, text, - ) - } - } -} - -func TestStatusHash(t *testing.T) { - type testcase struct { - status *gtsmodel.Status - expectedHash string - } - - for i, testcase := range []testcase{ - { - status: >smodel.Status{ - ContentWarning: "This is a test status", - Content: `

      Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.

      `, - }, - expectedHash: `8bbb5439dbe62ae0`, - }, - { - status: >smodel.Status{ - Content: `

      @zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)

      https://github.com/superseriousbusiness/gotosocial/pull/2863

      `, - }, - expectedHash: `d039dfb4d04752d5`, - }, - { - status: >smodel.Status{ - Content: `

      Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg, both running on Wazero, and configured with a 50MiB db cache target. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.

      `, - Attachments: []*gtsmodel.MediaAttachment{ { - ID: "01J7TYSH1V5V4DCTVPASH3K9PQ", - Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`, + Description: `Another media attachment`, + }, + }, + Poll: >smodel.Poll{ + Options: []string{ + "Poll option 1", + "Poll option 2", }, }, }, - expectedHash: `414d975b2ef9d112`, + expectedFields: []string{ + "Nerd stuff", + "Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg , both running on Wazero , and configured with a 50MiB db cache target . This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.", + "Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.", + "Another media attachment", + "Poll option 1", + "Poll option 2", + }, }, } { - hash := StatusHash(testcase.status) - if hash != testcase.expectedHash { - t.Errorf( - "test %d expected hash '%s' got '%s'", - i, testcase.expectedHash, hash, - ) - } + fields := filterableFields(testcase.status) + assert.Equal(t, testcase.expectedFields, fields) } } diff --git a/scripts/build.sh b/scripts/build.sh index 1781731e3c..5b10a5493b 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -6,7 +6,7 @@ set -e log_exec() { echo "$ ${*}"; "$@"; } # Grab environment variables and set defaults + requirements. -GO_BUILDTAGS="${GO_BUILDTAGS-} netgo osusergo static_build kvformat timetzdata purego" +GO_BUILDTAGS="${GO_BUILDTAGS-} netgo osusergo static_build kvformat timetzdata" GO_LDFLAGS="${GO_LDFLAGS-} -s -w -extldflags '-static' -X 'main.Version=${VERSION:-$(git describe --tags --abbrev=0)}'" GO_GCFLAGS=${GO_GCFLAGS-} @@ -17,7 +17,6 @@ GO_GCFLAGS=${GO_GCFLAGS-} # Available Go build tags, with explanation, followed by benefits of enabling it: # - kvformat: enables prettier output of log fields (slightly better performance) # - timetzdata: embed timezone database inside binary (allow setting local time inside Docker containers, at cost of 450KB) -# - purego: disable amd64/arm64 assembly implementation for xxhash (increase portability at marginal performance cost) # - notracing: disables compiling-in otel tracing support (reduced binary size, better performance) # - nometrics: disables compiling-in otel metrics support (reduced binary size, better performance) # - noerrcaller: disables caller function prefix in errors (slightly better performance, at cost of err readability) diff --git a/vendor/github.com/cespare/xxhash/LICENSE.txt b/vendor/github.com/cespare/xxhash/LICENSE.txt deleted file mode 100644 index 24b53065f4..0000000000 --- a/vendor/github.com/cespare/xxhash/LICENSE.txt +++ /dev/null @@ -1,22 +0,0 @@ -Copyright (c) 2016 Caleb Spare - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/cespare/xxhash/README.md b/vendor/github.com/cespare/xxhash/README.md deleted file mode 100644 index 0982fd25e5..0000000000 --- a/vendor/github.com/cespare/xxhash/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# xxhash - -[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash) - -xxhash is a Go implementation of the 64-bit -[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a -high-quality hashing algorithm that is much faster than anything in the Go -standard library. - -The API is very small, taking its cue from the other hashing packages in the -standard library: - - $ go doc github.com/cespare/xxhash ! - package xxhash // import "github.com/cespare/xxhash" - - Package xxhash implements the 64-bit variant of xxHash (XXH64) as described - at http://cyan4973.github.io/xxHash/. - - func New() hash.Hash64 - func Sum64(b []byte) uint64 - func Sum64String(s string) uint64 - -This implementation provides a fast pure-Go implementation and an even faster -assembly implementation for amd64. - -## Benchmarks - -Here are some quick benchmarks comparing the pure-Go and assembly -implementations of Sum64 against another popular Go XXH64 implementation, -[github.com/OneOfOne/xxhash](https://github.com/OneOfOne/xxhash): - -| input size | OneOfOne | cespare (purego) | cespare | -| --- | --- | --- | --- | -| 5 B | 416 MB/s | 720 MB/s | 872 MB/s | -| 100 B | 3980 MB/s | 5013 MB/s | 5252 MB/s | -| 4 KB | 12727 MB/s | 12999 MB/s | 13026 MB/s | -| 10 MB | 9879 MB/s | 10775 MB/s | 10913 MB/s | - -These numbers were generated with: - -``` -$ go test -benchtime 10s -bench '/OneOfOne,' -$ go test -tags purego -benchtime 10s -bench '/xxhash,' -$ go test -benchtime 10s -bench '/xxhash,' -``` - -## Projects using this package - -- [InfluxDB](https://github.com/influxdata/influxdb) -- [Prometheus](https://github.com/prometheus/prometheus) diff --git a/vendor/github.com/cespare/xxhash/rotate.go b/vendor/github.com/cespare/xxhash/rotate.go deleted file mode 100644 index f3eac5ebc0..0000000000 --- a/vendor/github.com/cespare/xxhash/rotate.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build !go1.9 - -package xxhash - -// TODO(caleb): After Go 1.10 comes out, remove this fallback code. - -func rol1(x uint64) uint64 { return (x << 1) | (x >> (64 - 1)) } -func rol7(x uint64) uint64 { return (x << 7) | (x >> (64 - 7)) } -func rol11(x uint64) uint64 { return (x << 11) | (x >> (64 - 11)) } -func rol12(x uint64) uint64 { return (x << 12) | (x >> (64 - 12)) } -func rol18(x uint64) uint64 { return (x << 18) | (x >> (64 - 18)) } -func rol23(x uint64) uint64 { return (x << 23) | (x >> (64 - 23)) } -func rol27(x uint64) uint64 { return (x << 27) | (x >> (64 - 27)) } -func rol31(x uint64) uint64 { return (x << 31) | (x >> (64 - 31)) } diff --git a/vendor/github.com/cespare/xxhash/rotate19.go b/vendor/github.com/cespare/xxhash/rotate19.go deleted file mode 100644 index b99612bab8..0000000000 --- a/vendor/github.com/cespare/xxhash/rotate19.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build go1.9 - -package xxhash - -import "math/bits" - -func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) } -func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) } -func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) } -func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) } -func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) } -func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) } -func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) } -func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) } diff --git a/vendor/github.com/cespare/xxhash/xxhash.go b/vendor/github.com/cespare/xxhash/xxhash.go deleted file mode 100644 index f896bd28f0..0000000000 --- a/vendor/github.com/cespare/xxhash/xxhash.go +++ /dev/null @@ -1,168 +0,0 @@ -// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described -// at http://cyan4973.github.io/xxHash/. -package xxhash - -import ( - "encoding/binary" - "hash" -) - -const ( - prime1 uint64 = 11400714785074694791 - prime2 uint64 = 14029467366897019727 - prime3 uint64 = 1609587929392839161 - prime4 uint64 = 9650029242287828579 - prime5 uint64 = 2870177450012600261 -) - -// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where -// possible in the Go code is worth a small (but measurable) performance boost -// by avoiding some MOVQs. Vars are needed for the asm and also are useful for -// convenience in the Go code in a few places where we need to intentionally -// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the -// result overflows a uint64). -var ( - prime1v = prime1 - prime2v = prime2 - prime3v = prime3 - prime4v = prime4 - prime5v = prime5 -) - -type xxh struct { - v1 uint64 - v2 uint64 - v3 uint64 - v4 uint64 - total int - mem [32]byte - n int // how much of mem is used -} - -// New creates a new hash.Hash64 that implements the 64-bit xxHash algorithm. -func New() hash.Hash64 { - var x xxh - x.Reset() - return &x -} - -func (x *xxh) Reset() { - x.n = 0 - x.total = 0 - x.v1 = prime1v + prime2 - x.v2 = prime2 - x.v3 = 0 - x.v4 = -prime1v -} - -func (x *xxh) Size() int { return 8 } -func (x *xxh) BlockSize() int { return 32 } - -// Write adds more data to x. It always returns len(b), nil. -func (x *xxh) Write(b []byte) (n int, err error) { - n = len(b) - x.total += len(b) - - if x.n+len(b) < 32 { - // This new data doesn't even fill the current block. - copy(x.mem[x.n:], b) - x.n += len(b) - return - } - - if x.n > 0 { - // Finish off the partial block. - copy(x.mem[x.n:], b) - x.v1 = round(x.v1, u64(x.mem[0:8])) - x.v2 = round(x.v2, u64(x.mem[8:16])) - x.v3 = round(x.v3, u64(x.mem[16:24])) - x.v4 = round(x.v4, u64(x.mem[24:32])) - b = b[32-x.n:] - x.n = 0 - } - - if len(b) >= 32 { - // One or more full blocks left. - b = writeBlocks(x, b) - } - - // Store any remaining partial block. - copy(x.mem[:], b) - x.n = len(b) - - return -} - -func (x *xxh) Sum(b []byte) []byte { - s := x.Sum64() - return append( - b, - byte(s>>56), - byte(s>>48), - byte(s>>40), - byte(s>>32), - byte(s>>24), - byte(s>>16), - byte(s>>8), - byte(s), - ) -} - -func (x *xxh) Sum64() uint64 { - var h uint64 - - if x.total >= 32 { - v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4 - h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) - h = mergeRound(h, v1) - h = mergeRound(h, v2) - h = mergeRound(h, v3) - h = mergeRound(h, v4) - } else { - h = x.v3 + prime5 - } - - h += uint64(x.total) - - i, end := 0, x.n - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(x.mem[i:i+8])) - h ^= k1 - h = rol27(h)*prime1 + prime4 - } - if i+4 <= end { - h ^= uint64(u32(x.mem[i:i+4])) * prime1 - h = rol23(h)*prime2 + prime3 - i += 4 - } - for i < end { - h ^= uint64(x.mem[i]) * prime5 - h = rol11(h) * prime1 - i++ - } - - h ^= h >> 33 - h *= prime2 - h ^= h >> 29 - h *= prime3 - h ^= h >> 32 - - return h -} - -func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) } -func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) } - -func round(acc, input uint64) uint64 { - acc += input * prime2 - acc = rol31(acc) - acc *= prime1 - return acc -} - -func mergeRound(acc, val uint64) uint64 { - val = round(0, val) - acc ^= val - acc = acc*prime1 + prime4 - return acc -} diff --git a/vendor/github.com/cespare/xxhash/xxhash_amd64.go b/vendor/github.com/cespare/xxhash/xxhash_amd64.go deleted file mode 100644 index d617652680..0000000000 --- a/vendor/github.com/cespare/xxhash/xxhash_amd64.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build !appengine -// +build gc -// +build !purego - -package xxhash - -// Sum64 computes the 64-bit xxHash digest of b. -// -//go:noescape -func Sum64(b []byte) uint64 - -func writeBlocks(x *xxh, b []byte) []byte diff --git a/vendor/github.com/cespare/xxhash/xxhash_amd64.s b/vendor/github.com/cespare/xxhash/xxhash_amd64.s deleted file mode 100644 index 757f2011f0..0000000000 --- a/vendor/github.com/cespare/xxhash/xxhash_amd64.s +++ /dev/null @@ -1,233 +0,0 @@ -// +build !appengine -// +build gc -// +build !purego - -#include "textflag.h" - -// Register allocation: -// AX h -// CX pointer to advance through b -// DX n -// BX loop end -// R8 v1, k1 -// R9 v2 -// R10 v3 -// R11 v4 -// R12 tmp -// R13 prime1v -// R14 prime2v -// R15 prime4v - -// round reads from and advances the buffer pointer in CX. -// It assumes that R13 has prime1v and R14 has prime2v. -#define round(r) \ - MOVQ (CX), R12 \ - ADDQ $8, CX \ - IMULQ R14, R12 \ - ADDQ R12, r \ - ROLQ $31, r \ - IMULQ R13, r - -// mergeRound applies a merge round on the two registers acc and val. -// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v. -#define mergeRound(acc, val) \ - IMULQ R14, val \ - ROLQ $31, val \ - IMULQ R13, val \ - XORQ val, acc \ - IMULQ R13, acc \ - ADDQ R15, acc - -// func Sum64(b []byte) uint64 -TEXT ·Sum64(SB), NOSPLIT, $0-32 - // Load fixed primes. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 - MOVQ ·prime4v(SB), R15 - - // Load slice. - MOVQ b_base+0(FP), CX - MOVQ b_len+8(FP), DX - LEAQ (CX)(DX*1), BX - - // The first loop limit will be len(b)-32. - SUBQ $32, BX - - // Check whether we have at least one block. - CMPQ DX, $32 - JLT noBlocks - - // Set up initial state (v1, v2, v3, v4). - MOVQ R13, R8 - ADDQ R14, R8 - MOVQ R14, R9 - XORQ R10, R10 - XORQ R11, R11 - SUBQ R13, R11 - - // Loop until CX > BX. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) - - CMPQ CX, BX - JLE blockLoop - - MOVQ R8, AX - ROLQ $1, AX - MOVQ R9, R12 - ROLQ $7, R12 - ADDQ R12, AX - MOVQ R10, R12 - ROLQ $12, R12 - ADDQ R12, AX - MOVQ R11, R12 - ROLQ $18, R12 - ADDQ R12, AX - - mergeRound(AX, R8) - mergeRound(AX, R9) - mergeRound(AX, R10) - mergeRound(AX, R11) - - JMP afterBlocks - -noBlocks: - MOVQ ·prime5v(SB), AX - -afterBlocks: - ADDQ DX, AX - - // Right now BX has len(b)-32, and we want to loop until CX > len(b)-8. - ADDQ $24, BX - - CMPQ CX, BX - JG fourByte - -wordLoop: - // Calculate k1. - MOVQ (CX), R8 - ADDQ $8, CX - IMULQ R14, R8 - ROLQ $31, R8 - IMULQ R13, R8 - - XORQ R8, AX - ROLQ $27, AX - IMULQ R13, AX - ADDQ R15, AX - - CMPQ CX, BX - JLE wordLoop - -fourByte: - ADDQ $4, BX - CMPQ CX, BX - JG singles - - MOVL (CX), R8 - ADDQ $4, CX - IMULQ R13, R8 - XORQ R8, AX - - ROLQ $23, AX - IMULQ R14, AX - ADDQ ·prime3v(SB), AX - -singles: - ADDQ $4, BX - CMPQ CX, BX - JGE finalize - -singlesLoop: - MOVBQZX (CX), R12 - ADDQ $1, CX - IMULQ ·prime5v(SB), R12 - XORQ R12, AX - - ROLQ $11, AX - IMULQ R13, AX - - CMPQ CX, BX - JL singlesLoop - -finalize: - MOVQ AX, R12 - SHRQ $33, R12 - XORQ R12, AX - IMULQ R14, AX - MOVQ AX, R12 - SHRQ $29, R12 - XORQ R12, AX - IMULQ ·prime3v(SB), AX - MOVQ AX, R12 - SHRQ $32, R12 - XORQ R12, AX - - MOVQ AX, ret+24(FP) - RET - -// writeBlocks uses the same registers as above except that it uses AX to store -// the x pointer. - -// func writeBlocks(x *xxh, b []byte) []byte -TEXT ·writeBlocks(SB), NOSPLIT, $0-56 - // Load fixed primes needed for round. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 - - // Load slice. - MOVQ b_base+8(FP), CX - MOVQ CX, ret_base+32(FP) // initialize return base pointer; see NOTE below - MOVQ b_len+16(FP), DX - LEAQ (CX)(DX*1), BX - SUBQ $32, BX - - // Load vN from x. - MOVQ x+0(FP), AX - MOVQ 0(AX), R8 // v1 - MOVQ 8(AX), R9 // v2 - MOVQ 16(AX), R10 // v3 - MOVQ 24(AX), R11 // v4 - - // We don't need to check the loop condition here; this function is - // always called with at least one block of data to process. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) - - CMPQ CX, BX - JLE blockLoop - - // Copy vN back to x. - MOVQ R8, 0(AX) - MOVQ R9, 8(AX) - MOVQ R10, 16(AX) - MOVQ R11, 24(AX) - - // Construct return slice. - // NOTE: It's important that we don't construct a slice that has a base - // pointer off the end of the original slice, as in Go 1.7+ this will - // cause runtime crashes. (See discussion in, for example, - // https://github.com/golang/go/issues/16772.) - // Therefore, we calculate the length/cap first, and if they're zero, we - // keep the old base. This is what the compiler does as well if you - // write code like - // b = b[len(b):] - - // New length is 32 - (CX - BX) -> BX+32 - CX. - ADDQ $32, BX - SUBQ CX, BX - JZ afterSetBase - - MOVQ CX, ret_base+32(FP) - -afterSetBase: - MOVQ BX, ret_len+40(FP) - MOVQ BX, ret_cap+48(FP) // set cap == len - - RET diff --git a/vendor/github.com/cespare/xxhash/xxhash_other.go b/vendor/github.com/cespare/xxhash/xxhash_other.go deleted file mode 100644 index c68d13f89e..0000000000 --- a/vendor/github.com/cespare/xxhash/xxhash_other.go +++ /dev/null @@ -1,75 +0,0 @@ -// +build !amd64 appengine !gc purego - -package xxhash - -// Sum64 computes the 64-bit xxHash digest of b. -func Sum64(b []byte) uint64 { - // A simpler version would be - // x := New() - // x.Write(b) - // return x.Sum64() - // but this is faster, particularly for small inputs. - - n := len(b) - var h uint64 - - if n >= 32 { - v1 := prime1v + prime2 - v2 := prime2 - v3 := uint64(0) - v4 := -prime1v - for len(b) >= 32 { - v1 = round(v1, u64(b[0:8:len(b)])) - v2 = round(v2, u64(b[8:16:len(b)])) - v3 = round(v3, u64(b[16:24:len(b)])) - v4 = round(v4, u64(b[24:32:len(b)])) - b = b[32:len(b):len(b)] - } - h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) - h = mergeRound(h, v1) - h = mergeRound(h, v2) - h = mergeRound(h, v3) - h = mergeRound(h, v4) - } else { - h = prime5 - } - - h += uint64(n) - - i, end := 0, len(b) - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(b[i:i+8:len(b)])) - h ^= k1 - h = rol27(h)*prime1 + prime4 - } - if i+4 <= end { - h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 - h = rol23(h)*prime2 + prime3 - i += 4 - } - for ; i < end; i++ { - h ^= uint64(b[i]) * prime5 - h = rol11(h) * prime1 - } - - h ^= h >> 33 - h *= prime2 - h ^= h >> 29 - h *= prime3 - h ^= h >> 32 - - return h -} - -func writeBlocks(x *xxh, b []byte) []byte { - v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4 - for len(b) >= 32 { - v1 = round(v1, u64(b[0:8:len(b)])) - v2 = round(v2, u64(b[8:16:len(b)])) - v3 = round(v3, u64(b[16:24:len(b)])) - v4 = round(v4, u64(b[24:32:len(b)])) - b = b[32:len(b):len(b)] - } - x.v1, x.v2, x.v3, x.v4 = v1, v2, v3, v4 - return b -} diff --git a/vendor/github.com/cespare/xxhash/xxhash_safe.go b/vendor/github.com/cespare/xxhash/xxhash_safe.go deleted file mode 100644 index dfa15ab7e2..0000000000 --- a/vendor/github.com/cespare/xxhash/xxhash_safe.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build appengine - -// This file contains the safe implementations of otherwise unsafe-using code. - -package xxhash - -// Sum64String computes the 64-bit xxHash digest of s. -func Sum64String(s string) uint64 { - return Sum64([]byte(s)) -} diff --git a/vendor/github.com/cespare/xxhash/xxhash_unsafe.go b/vendor/github.com/cespare/xxhash/xxhash_unsafe.go deleted file mode 100644 index d2b64e8bb0..0000000000 --- a/vendor/github.com/cespare/xxhash/xxhash_unsafe.go +++ /dev/null @@ -1,30 +0,0 @@ -// +build !appengine - -// This file encapsulates usage of unsafe. -// xxhash_safe.go contains the safe implementations. - -package xxhash - -import ( - "reflect" - "unsafe" -) - -// Sum64String computes the 64-bit xxHash digest of s. -// It may be faster than Sum64([]byte(s)) by avoiding a copy. -// -// TODO(caleb): Consider removing this if an optimization is ever added to make -// it unnecessary: https://golang.org/issue/2205. -// -// TODO(caleb): We still have a function call; we could instead write Go/asm -// copies of Sum64 for strings to squeeze out a bit more speed. -func Sum64String(s string) uint64 { - // See https://groups.google.com/d/msg/golang-nuts/dcjzJy-bSpw/tcZYBzQqAQAJ - // for some discussion about this unsafe conversion. - var b []byte - bh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data - bh.Len = len(s) - bh.Cap = len(s) - return Sum64(b) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index a0734eabb3..92905c133c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -140,9 +140,6 @@ github.com/bytedance/sonic/loader/internal/rt # github.com/cenkalti/backoff/v4 v4.3.0 ## explicit; go 1.18 github.com/cenkalti/backoff/v4 -# github.com/cespare/xxhash v1.1.0 -## explicit -github.com/cespare/xxhash # github.com/cespare/xxhash/v2 v2.3.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 From 33f10a56b980af2b8a1c9371b9966b99f0f04914 Mon Sep 17 00:00:00 2001 From: tobi Date: Mon, 16 Sep 2024 11:22:16 +0200 Subject: [PATCH 07/11] remove now unused xxhash --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 527344d84c..989b7c1fac 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,6 @@ For bugs and feature requests, please check to see if there's [already an issue] The following open source libraries, frameworks, and tools are used by GoToSocial, with gratitude 💕 - [buckket/go-blurhash](https://github.com/buckket/go-blurhash); used for generating image blurhashes. [GPL-3.0 License](https://spdx.org/licenses/GPL-3.0-only.html). -- [cespare/xxhash](https://github.com/cespare/xxhash); xxHash generation. [MIT License](https://spdx.org/licenses/MIT.html). - [coreos/go-oidc](https://github.com/coreos/go-oidc); OIDC client library. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html). - [DmitriyVTitov/size](https://github.com/DmitriyVTitov/size); runtime model memory size calculations. [MIT License](https://spdx.org/licenses/MIT.html). - Gin: From 286898b1d247797f087cac9d57fb424a6571acc4 Mon Sep 17 00:00:00 2001 From: tobi Date: Mon, 16 Sep 2024 11:23:18 +0200 Subject: [PATCH 08/11] whoops, wrong logger --- internal/typeutils/converter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 8284bda96c..97d661a5d5 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -18,13 +18,13 @@ package typeutils import ( - "log" "sync" "time" "codeberg.org/gruf/go-cache/v3" "github.com/superseriousbusiness/gotosocial/internal/filter/interaction" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" + "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/state" ) From b4a4c478b0c6c030a7b4c284f91f6dea39a7c8da Mon Sep 17 00:00:00 2001 From: tobi Date: Mon, 16 Sep 2024 11:26:40 +0200 Subject: [PATCH 09/11] Merge branch 'main' into status_filtering_bugfix --- go.mod | 8 +- go.sum | 16 +- .../jackc/pgservicefile/.travis.yml | 9 - .../github.com/jackc/pgservicefile/README.md | 5 +- .../jackc/pgservicefile/pgservicefile.go | 4 +- vendor/github.com/jackc/pgx/v5/CHANGELOG.md | 22 ++ vendor/github.com/jackc/pgx/v5/batch.go | 44 +-- vendor/github.com/jackc/pgx/v5/conn.go | 26 +- .../github.com/jackc/pgx/v5/derived_types.go | 262 ++++++++++++++++++ vendor/github.com/jackc/pgx/v5/doc.go | 2 +- .../github.com/jackc/pgx/v5/pgconn/config.go | 64 +++-- vendor/github.com/jackc/pgx/v5/pgtype/doc.go | 3 + .../jackc/pgx/v5/pgtype/interval.go | 32 +-- vendor/github.com/jackc/pgx/v5/pgtype/json.go | 9 +- .../github.com/jackc/pgx/v5/pgtype/pgtype.go | 44 ++- .../jackc/pgx/v5/pgtype/pgtype_default.go | 3 + vendor/github.com/jackc/pgx/v5/pgtype/time.go | 8 +- .../github.com/jackc/pgx/v5/pgtype/uint32.go | 22 ++ vendor/github.com/jackc/pgx/v5/pgtype/xml.go | 198 +++++++++++++ vendor/github.com/jackc/pgx/v5/pgxpool/tx.go | 13 +- vendor/github.com/jackc/pgx/v5/rows.go | 21 +- vendor/github.com/jackc/pgx/v5/stdlib/sql.go | 24 +- .../github.com/jackc/puddle/v2/CHANGELOG.md | 5 + vendor/github.com/jackc/puddle/v2/README.md | 2 +- vendor/github.com/jackc/puddle/v2/nanotime.go | 16 ++ .../jackc/puddle/v2/nanotime_time.go | 13 - .../jackc/puddle/v2/nanotime_unsafe.go | 12 - vendor/github.com/jackc/puddle/v2/pool.go | 20 +- vendor/modules.txt | 10 +- 29 files changed, 753 insertions(+), 164 deletions(-) delete mode 100644 vendor/github.com/jackc/pgservicefile/.travis.yml create mode 100644 vendor/github.com/jackc/pgx/v5/derived_types.go create mode 100644 vendor/github.com/jackc/pgx/v5/pgtype/xml.go create mode 100644 vendor/github.com/jackc/puddle/v2/nanotime.go delete mode 100644 vendor/github.com/jackc/puddle/v2/nanotime_time.go delete mode 100644 vendor/github.com/jackc/puddle/v2/nanotime_unsafe.go diff --git a/go.mod b/go.mod index 67fe84d221..9f7d715573 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/feeds v1.2.0 github.com/gorilla/websocket v1.5.2 - github.com/jackc/pgx/v5 v5.6.0 + github.com/jackc/pgx/v5 v5.7.1 github.com/k3a/html2text v1.2.1 github.com/microcosm-cc/bluemonday v1.0.27 github.com/miekg/dns v1.1.62 @@ -76,7 +76,7 @@ require ( go.uber.org/automaxprocs v1.5.3 golang.org/x/crypto v0.27.0 golang.org/x/image v0.20.0 - golang.org/x/net v0.28.0 + golang.org/x/net v0.29.0 golang.org/x/oauth2 v0.23.0 golang.org/x/text v0.18.0 gopkg.in/mcuadros/go-syslog.v2 v2.3.0 @@ -155,8 +155,8 @@ require ( github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jessevdk/go-flags v1.5.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/go.sum b/go.sum index 749487a685..bace0fe7f4 100644 --- a/go.sum +++ b/go.sum @@ -364,12 +364,12 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= @@ -747,8 +747,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/vendor/github.com/jackc/pgservicefile/.travis.yml b/vendor/github.com/jackc/pgservicefile/.travis.yml deleted file mode 100644 index e176228e8e..0000000000 --- a/vendor/github.com/jackc/pgservicefile/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: go - -go: - - 1.x - - tip - -matrix: - allow_failures: - - go: tip diff --git a/vendor/github.com/jackc/pgservicefile/README.md b/vendor/github.com/jackc/pgservicefile/README.md index e50ca12627..2fc7e012cb 100644 --- a/vendor/github.com/jackc/pgservicefile/README.md +++ b/vendor/github.com/jackc/pgservicefile/README.md @@ -1,5 +1,6 @@ -[![](https://godoc.org/github.com/jackc/pgservicefile?status.svg)](https://godoc.org/github.com/jackc/pgservicefile) -[![Build Status](https://travis-ci.org/jackc/pgservicefile.svg)](https://travis-ci.org/jackc/pgservicefile) +[![Go Reference](https://pkg.go.dev/badge/github.com/jackc/pgservicefile.svg)](https://pkg.go.dev/github.com/jackc/pgservicefile) +[![Build Status](https://github.com/jackc/pgservicefile/actions/workflows/ci.yml/badge.svg)](https://github.com/jackc/pgservicefile/actions/workflows/ci.yml) + # pgservicefile diff --git a/vendor/github.com/jackc/pgservicefile/pgservicefile.go b/vendor/github.com/jackc/pgservicefile/pgservicefile.go index 797bbab9e7..c62caa7fef 100644 --- a/vendor/github.com/jackc/pgservicefile/pgservicefile.go +++ b/vendor/github.com/jackc/pgservicefile/pgservicefile.go @@ -57,7 +57,7 @@ func ParseServicefile(r io.Reader) (*Servicefile, error) { } else if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { service = &Service{Name: line[1 : len(line)-1], Settings: make(map[string]string)} servicefile.Services = append(servicefile.Services, service) - } else { + } else if service != nil { parts := strings.SplitN(line, "=", 2) if len(parts) != 2 { return nil, fmt.Errorf("unable to parse line %d", lineNum) @@ -67,6 +67,8 @@ func ParseServicefile(r io.Reader) (*Servicefile, error) { value := strings.TrimSpace(parts[1]) service.Settings[key] = value + } else { + return nil, fmt.Errorf("line %d is not in a section", lineNum) } } diff --git a/vendor/github.com/jackc/pgx/v5/CHANGELOG.md b/vendor/github.com/jackc/pgx/v5/CHANGELOG.md index 61b4695fd5..a0ff9ba3b5 100644 --- a/vendor/github.com/jackc/pgx/v5/CHANGELOG.md +++ b/vendor/github.com/jackc/pgx/v5/CHANGELOG.md @@ -1,3 +1,25 @@ +# 5.7.1 (September 10, 2024) + +* Fix data race in tracelog.TraceLog +* Update puddle to v2.2.2. This removes the import of nanotime via linkname. +* Update golang.org/x/crypto and golang.org/x/text + +# 5.7.0 (September 7, 2024) + +* Add support for sslrootcert=system (Yann Soubeyrand) +* Add LoadTypes to load multiple types in a single SQL query (Nick Farrell) +* Add XMLCodec supports encoding + scanning XML column type like json (nickcruess-soda) +* Add MultiTrace (Stepan Rabotkin) +* Add TraceLogConfig with customizable TimeKey (stringintech) +* pgx.ErrNoRows wraps sql.ErrNoRows to aid in database/sql compatibility with native pgx functions (merlin) +* Support scanning binary formatted uint32 into string / TextScanner (jennifersp) +* Fix interval encoding to allow 0s and avoid extra spaces (Carlos Pérez-Aradros Herce) +* Update pgservicefile - fixes panic when parsing invalid file +* Better error message when reading past end of batch +* Don't print url when url.Parse returns an error (Kevin Biju) +* Fix snake case name normalization collision in RowToStructByName with db tag (nolandseigler) +* Fix: Scan and encode types with underlying types of arrays + # 5.6.0 (May 25, 2024) * Add StrictNamedArgs (Tomas Zahradnicek) diff --git a/vendor/github.com/jackc/pgx/v5/batch.go b/vendor/github.com/jackc/pgx/v5/batch.go index 3540f57f53..c3c2834f2d 100644 --- a/vendor/github.com/jackc/pgx/v5/batch.go +++ b/vendor/github.com/jackc/pgx/v5/batch.go @@ -60,9 +60,13 @@ type Batch struct { QueuedQueries []*QueuedQuery } -// Queue queues a query to batch b. query can be an SQL query or the name of a prepared statement. -// The only pgx option argument that is supported is QueryRewriter. Queries are executed using the -// connection's DefaultQueryExecMode. +// Queue queues a query to batch b. query can be an SQL query or the name of a prepared statement. The only pgx option +// argument that is supported is QueryRewriter. Queries are executed using the connection's DefaultQueryExecMode. +// +// While query can contain multiple statements if the connection's DefaultQueryExecMode is QueryModeSimple, this should +// be avoided. QueuedQuery.Fn must not be set as it will only be called for the first query. That is, QueuedQuery.Query, +// QueuedQuery.QueryRow, and QueuedQuery.Exec must not be called. In addition, any error messages or tracing that +// include the current query may reference the wrong query. func (b *Batch) Queue(query string, arguments ...any) *QueuedQuery { qq := &QueuedQuery{ SQL: query, @@ -128,7 +132,7 @@ func (br *batchResults) Exec() (pgconn.CommandTag, error) { if !br.mrr.NextResult() { err := br.mrr.Close() if err == nil { - err = errors.New("no result") + err = errors.New("no more results in batch") } if br.conn.batchTracer != nil { br.conn.batchTracer.TraceBatchQuery(br.ctx, br.conn, TraceBatchQueryData{ @@ -180,7 +184,7 @@ func (br *batchResults) Query() (Rows, error) { if !br.mrr.NextResult() { rows.err = br.mrr.Close() if rows.err == nil { - rows.err = errors.New("no result") + rows.err = errors.New("no more results in batch") } rows.closed = true @@ -287,7 +291,10 @@ func (br *pipelineBatchResults) Exec() (pgconn.CommandTag, error) { return pgconn.CommandTag{}, br.err } - query, arguments, _ := br.nextQueryAndArgs() + query, arguments, err := br.nextQueryAndArgs() + if err != nil { + return pgconn.CommandTag{}, err + } results, err := br.pipeline.GetResults() if err != nil { @@ -330,9 +337,9 @@ func (br *pipelineBatchResults) Query() (Rows, error) { return &baseRows{err: br.err, closed: true}, br.err } - query, arguments, ok := br.nextQueryAndArgs() - if !ok { - query = "batch query" + query, arguments, err := br.nextQueryAndArgs() + if err != nil { + return &baseRows{err: err, closed: true}, err } rows := br.conn.getRows(br.ctx, query, arguments) @@ -421,13 +428,16 @@ func (br *pipelineBatchResults) earlyError() error { return br.err } -func (br *pipelineBatchResults) nextQueryAndArgs() (query string, args []any, ok bool) { - if br.b != nil && br.qqIdx < len(br.b.QueuedQueries) { - bi := br.b.QueuedQueries[br.qqIdx] - query = bi.SQL - args = bi.Arguments - ok = true - br.qqIdx++ +func (br *pipelineBatchResults) nextQueryAndArgs() (query string, args []any, err error) { + if br.b == nil { + return "", nil, errors.New("no reference to batch") } - return + + if br.qqIdx >= len(br.b.QueuedQueries) { + return "", nil, errors.New("no more results in batch") + } + + bi := br.b.QueuedQueries[br.qqIdx] + br.qqIdx++ + return bi.SQL, bi.Arguments, nil } diff --git a/vendor/github.com/jackc/pgx/v5/conn.go b/vendor/github.com/jackc/pgx/v5/conn.go index 3117214599..187b3dd57b 100644 --- a/vendor/github.com/jackc/pgx/v5/conn.go +++ b/vendor/github.com/jackc/pgx/v5/conn.go @@ -3,6 +3,7 @@ package pgx import ( "context" "crypto/sha256" + "database/sql" "encoding/hex" "errors" "fmt" @@ -102,13 +103,31 @@ func (ident Identifier) Sanitize() string { var ( // ErrNoRows occurs when rows are expected but none are returned. - ErrNoRows = errors.New("no rows in result set") + ErrNoRows = newProxyErr(sql.ErrNoRows, "no rows in result set") // ErrTooManyRows occurs when more rows than expected are returned. ErrTooManyRows = errors.New("too many rows in result set") ) -var errDisabledStatementCache = fmt.Errorf("cannot use QueryExecModeCacheStatement with disabled statement cache") -var errDisabledDescriptionCache = fmt.Errorf("cannot use QueryExecModeCacheDescribe with disabled description cache") +func newProxyErr(background error, msg string) error { + return &proxyError{ + msg: msg, + background: background, + } +} + +type proxyError struct { + msg string + background error +} + +func (err *proxyError) Error() string { return err.msg } + +func (err *proxyError) Unwrap() error { return err.background } + +var ( + errDisabledStatementCache = fmt.Errorf("cannot use QueryExecModeCacheStatement with disabled statement cache") + errDisabledDescriptionCache = fmt.Errorf("cannot use QueryExecModeCacheDescribe with disabled description cache") +) // Connect establishes a connection with a PostgreSQL server with a connection string. See // pgconn.Connect for details. @@ -843,7 +862,6 @@ func (c *Conn) getStatementDescription( mode QueryExecMode, sql string, ) (sd *pgconn.StatementDescription, err error) { - switch mode { case QueryExecModeCacheStatement: if c.statementCache == nil { diff --git a/vendor/github.com/jackc/pgx/v5/derived_types.go b/vendor/github.com/jackc/pgx/v5/derived_types.go new file mode 100644 index 0000000000..22ab069cf3 --- /dev/null +++ b/vendor/github.com/jackc/pgx/v5/derived_types.go @@ -0,0 +1,262 @@ +package pgx + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/jackc/pgx/v5/pgtype" +) + +/* +buildLoadDerivedTypesSQL generates the correct query for retrieving type information. + + pgVersion: the major version of the PostgreSQL server + typeNames: the names of the types to load. If nil, load all types. +*/ +func buildLoadDerivedTypesSQL(pgVersion int64, typeNames []string) string { + supportsMultirange := (pgVersion >= 14) + var typeNamesClause string + + if typeNames == nil { + // This should not occur; this will not return any types + typeNamesClause = "= ''" + } else { + typeNamesClause = "= ANY($1)" + } + parts := make([]string, 0, 10) + + // Each of the type names provided might be found in pg_class or pg_type. + // Additionally, it may or may not include a schema portion. + parts = append(parts, ` +WITH RECURSIVE +-- find the OIDs in pg_class which match one of the provided type names +selected_classes(oid,reltype) AS ( + -- this query uses the namespace search path, so will match type names without a schema prefix + SELECT pg_class.oid, pg_class.reltype + FROM pg_catalog.pg_class + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = pg_class.relnamespace + WHERE pg_catalog.pg_table_is_visible(pg_class.oid) + AND relname `, typeNamesClause, ` +UNION ALL + -- this query will only match type names which include the schema prefix + SELECT pg_class.oid, pg_class.reltype + FROM pg_class + INNER JOIN pg_namespace ON (pg_class.relnamespace = pg_namespace.oid) + WHERE nspname || '.' || relname `, typeNamesClause, ` +), +selected_types(oid) AS ( + -- collect the OIDs from pg_types which correspond to the selected classes + SELECT reltype AS oid + FROM selected_classes +UNION ALL + -- as well as any other type names which match our criteria + SELECT pg_type.oid + FROM pg_type + LEFT OUTER JOIN pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE typname `, typeNamesClause, ` + OR nspname || '.' || typname `, typeNamesClause, ` +), +-- this builds a parent/child mapping of objects, allowing us to know +-- all the child (ie: dependent) types that a parent (type) requires +-- As can be seen, there are 3 ways this can occur (the last of which +-- is due to being a composite class, where the composite fields are children) +pc(parent, child) AS ( + SELECT parent.oid, parent.typelem + FROM pg_type parent + WHERE parent.typtype = 'b' AND parent.typelem != 0 +UNION ALL + SELECT parent.oid, parent.typbasetype + FROM pg_type parent + WHERE parent.typtypmod = -1 AND parent.typbasetype != 0 +UNION ALL + SELECT pg_type.oid, atttypid + FROM pg_attribute + INNER JOIN pg_class ON (pg_class.oid = pg_attribute.attrelid) + INNER JOIN pg_type ON (pg_type.oid = pg_class.reltype) + WHERE NOT attisdropped + AND attnum > 0 +), +-- Now construct a recursive query which includes a 'depth' element. +-- This is used to ensure that the "youngest" children are registered before +-- their parents. +relationships(parent, child, depth) AS ( + SELECT DISTINCT 0::OID, selected_types.oid, 0 + FROM selected_types +UNION ALL + SELECT pg_type.oid AS parent, pg_attribute.atttypid AS child, 1 + FROM selected_classes c + inner join pg_type ON (c.reltype = pg_type.oid) + inner join pg_attribute on (c.oid = pg_attribute.attrelid) +UNION ALL + SELECT pc.parent, pc.child, relationships.depth + 1 + FROM pc + INNER JOIN relationships ON (pc.parent = relationships.child) +), +-- composite fields need to be encapsulated as a couple of arrays to provide the required information for registration +composite AS ( + SELECT pg_type.oid, ARRAY_AGG(attname ORDER BY attnum) AS attnames, ARRAY_AGG(atttypid ORDER BY ATTNUM) AS atttypids + FROM pg_attribute + INNER JOIN pg_class ON (pg_class.oid = pg_attribute.attrelid) + INNER JOIN pg_type ON (pg_type.oid = pg_class.reltype) + WHERE NOT attisdropped + AND attnum > 0 + GROUP BY pg_type.oid +) +-- Bring together this information, showing all the information which might possibly be required +-- to complete the registration, applying filters to only show the items which relate to the selected +-- types/classes. +SELECT typname, + pg_namespace.nspname, + typtype, + typbasetype, + typelem, + pg_type.oid,`) + if supportsMultirange { + parts = append(parts, ` + COALESCE(multirange.rngtypid, 0) AS rngtypid,`) + } else { + parts = append(parts, ` + 0 AS rngtypid,`) + } + parts = append(parts, ` + COALESCE(pg_range.rngsubtype, 0) AS rngsubtype, + attnames, atttypids + FROM relationships + INNER JOIN pg_type ON (pg_type.oid = relationships.child) + LEFT OUTER JOIN pg_range ON (pg_type.oid = pg_range.rngtypid)`) + if supportsMultirange { + parts = append(parts, ` + LEFT OUTER JOIN pg_range multirange ON (pg_type.oid = multirange.rngmultitypid)`) + } + + parts = append(parts, ` + LEFT OUTER JOIN composite USING (oid) + LEFT OUTER JOIN pg_namespace ON (pg_type.typnamespace = pg_namespace.oid) + WHERE NOT (typtype = 'b' AND typelem = 0)`) + parts = append(parts, ` + GROUP BY typname, pg_namespace.nspname, typtype, typbasetype, typelem, pg_type.oid, pg_range.rngsubtype,`) + if supportsMultirange { + parts = append(parts, ` + multirange.rngtypid,`) + } + parts = append(parts, ` + attnames, atttypids + ORDER BY MAX(depth) desc, typname;`) + return strings.Join(parts, "") +} + +type derivedTypeInfo struct { + Oid, Typbasetype, Typelem, Rngsubtype, Rngtypid uint32 + TypeName, Typtype, NspName string + Attnames []string + Atttypids []uint32 +} + +// LoadTypes performs a single (complex) query, returning all the required +// information to register the named types, as well as any other types directly +// or indirectly required to complete the registration. +// The result of this call can be passed into RegisterTypes to complete the process. +func (c *Conn) LoadTypes(ctx context.Context, typeNames []string) ([]*pgtype.Type, error) { + m := c.TypeMap() + if typeNames == nil || len(typeNames) == 0 { + return nil, fmt.Errorf("No type names were supplied.") + } + + // Disregard server version errors. This will result in + // the SQL not support recent structures such as multirange + serverVersion, _ := serverVersion(c) + sql := buildLoadDerivedTypesSQL(serverVersion, typeNames) + var rows Rows + var err error + if typeNames == nil { + rows, err = c.Query(ctx, sql, QueryExecModeSimpleProtocol) + } else { + rows, err = c.Query(ctx, sql, QueryExecModeSimpleProtocol, typeNames) + } + if err != nil { + return nil, fmt.Errorf("While generating load types query: %w", err) + } + defer rows.Close() + result := make([]*pgtype.Type, 0, 100) + for rows.Next() { + ti := derivedTypeInfo{} + err = rows.Scan(&ti.TypeName, &ti.NspName, &ti.Typtype, &ti.Typbasetype, &ti.Typelem, &ti.Oid, &ti.Rngtypid, &ti.Rngsubtype, &ti.Attnames, &ti.Atttypids) + if err != nil { + return nil, fmt.Errorf("While scanning type information: %w", err) + } + var type_ *pgtype.Type + switch ti.Typtype { + case "b": // array + dt, ok := m.TypeForOID(ti.Typelem) + if !ok { + return nil, fmt.Errorf("Array element OID %v not registered while loading pgtype %q", ti.Typelem, ti.TypeName) + } + type_ = &pgtype.Type{Name: ti.TypeName, OID: ti.Oid, Codec: &pgtype.ArrayCodec{ElementType: dt}} + case "c": // composite + var fields []pgtype.CompositeCodecField + for i, fieldName := range ti.Attnames { + dt, ok := m.TypeForOID(ti.Atttypids[i]) + if !ok { + return nil, fmt.Errorf("Unknown field for composite type %q: field %q (OID %v) is not already registered.", ti.TypeName, fieldName, ti.Atttypids[i]) + } + fields = append(fields, pgtype.CompositeCodecField{Name: fieldName, Type: dt}) + } + + type_ = &pgtype.Type{Name: ti.TypeName, OID: ti.Oid, Codec: &pgtype.CompositeCodec{Fields: fields}} + case "d": // domain + dt, ok := m.TypeForOID(ti.Typbasetype) + if !ok { + return nil, fmt.Errorf("Domain base type OID %v was not already registered, needed for %q", ti.Typbasetype, ti.TypeName) + } + + type_ = &pgtype.Type{Name: ti.TypeName, OID: ti.Oid, Codec: dt.Codec} + case "e": // enum + type_ = &pgtype.Type{Name: ti.TypeName, OID: ti.Oid, Codec: &pgtype.EnumCodec{}} + case "r": // range + dt, ok := m.TypeForOID(ti.Rngsubtype) + if !ok { + return nil, fmt.Errorf("Range element OID %v was not already registered, needed for %q", ti.Rngsubtype, ti.TypeName) + } + + type_ = &pgtype.Type{Name: ti.TypeName, OID: ti.Oid, Codec: &pgtype.RangeCodec{ElementType: dt}} + case "m": // multirange + dt, ok := m.TypeForOID(ti.Rngtypid) + if !ok { + return nil, fmt.Errorf("Multirange element OID %v was not already registered, needed for %q", ti.Rngtypid, ti.TypeName) + } + + type_ = &pgtype.Type{Name: ti.TypeName, OID: ti.Oid, Codec: &pgtype.MultirangeCodec{ElementType: dt}} + default: + return nil, fmt.Errorf("Unknown typtype %q was found while registering %q", ti.Typtype, ti.TypeName) + } + if type_ != nil { + m.RegisterType(type_) + if ti.NspName != "" { + nspType := &pgtype.Type{Name: ti.NspName + "." + type_.Name, OID: type_.OID, Codec: type_.Codec} + m.RegisterType(nspType) + result = append(result, nspType) + } + result = append(result, type_) + } + } + return result, nil +} + +// serverVersion returns the postgresql server version. +func serverVersion(c *Conn) (int64, error) { + serverVersionStr := c.PgConn().ParameterStatus("server_version") + serverVersionStr = regexp.MustCompile(`^[0-9]+`).FindString(serverVersionStr) + // if not PostgreSQL do nothing + if serverVersionStr == "" { + return 0, fmt.Errorf("Cannot identify server version in %q", serverVersionStr) + } + + version, err := strconv.ParseInt(serverVersionStr, 10, 64) + if err != nil { + return 0, fmt.Errorf("postgres version parsing failed: %w", err) + } + return version, nil +} diff --git a/vendor/github.com/jackc/pgx/v5/doc.go b/vendor/github.com/jackc/pgx/v5/doc.go index bc0391dde3..0e91d64e89 100644 --- a/vendor/github.com/jackc/pgx/v5/doc.go +++ b/vendor/github.com/jackc/pgx/v5/doc.go @@ -175,7 +175,7 @@ notification is received or the context is canceled. Tracing and Logging -pgx supports tracing by setting ConnConfig.Tracer. +pgx supports tracing by setting ConnConfig.Tracer. To combine several tracers you can use the multitracer.Tracer. In addition, the tracelog package provides the TraceLog type which lets a traditional logger act as a Tracer. diff --git a/vendor/github.com/jackc/pgx/v5/pgconn/config.go b/vendor/github.com/jackc/pgx/v5/pgconn/config.go index 598917f554..6a198e6750 100644 --- a/vendor/github.com/jackc/pgx/v5/pgconn/config.go +++ b/vendor/github.com/jackc/pgx/v5/pgconn/config.go @@ -467,14 +467,17 @@ func parseEnvSettings() map[string]string { func parseURLSettings(connString string) (map[string]string, error) { settings := make(map[string]string) - url, err := url.Parse(connString) + parsedURL, err := url.Parse(connString) if err != nil { + if urlErr := new(url.Error); errors.As(err, &urlErr) { + return nil, urlErr.Err + } return nil, err } - if url.User != nil { - settings["user"] = url.User.Username() - if password, present := url.User.Password(); present { + if parsedURL.User != nil { + settings["user"] = parsedURL.User.Username() + if password, present := parsedURL.User.Password(); present { settings["password"] = password } } @@ -482,7 +485,7 @@ func parseURLSettings(connString string) (map[string]string, error) { // Handle multiple host:port's in url.Host by splitting them into host,host,host and port,port,port. var hosts []string var ports []string - for _, host := range strings.Split(url.Host, ",") { + for _, host := range strings.Split(parsedURL.Host, ",") { if host == "" { continue } @@ -508,7 +511,7 @@ func parseURLSettings(connString string) (map[string]string, error) { settings["port"] = strings.Join(ports, ",") } - database := strings.TrimLeft(url.Path, "/") + database := strings.TrimLeft(parsedURL.Path, "/") if database != "" { settings["database"] = database } @@ -517,7 +520,7 @@ func parseURLSettings(connString string) (map[string]string, error) { "dbname": "database", } - for k, v := range url.Query() { + for k, v := range parsedURL.Query() { if k2, present := nameMap[k]; present { k = k2 } @@ -654,6 +657,36 @@ func configTLS(settings map[string]string, thisHost string, parseConfigOptions P tlsConfig := &tls.Config{} + if sslrootcert != "" { + var caCertPool *x509.CertPool + + if sslrootcert == "system" { + var err error + + caCertPool, err = x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("unable to load system certificate pool: %w", err) + } + + sslmode = "verify-full" + } else { + caCertPool = x509.NewCertPool() + + caPath := sslrootcert + caCert, err := os.ReadFile(caPath) + if err != nil { + return nil, fmt.Errorf("unable to read CA file: %w", err) + } + + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, errors.New("unable to add CA to cert pool") + } + } + + tlsConfig.RootCAs = caCertPool + tlsConfig.ClientCAs = caCertPool + } + switch sslmode { case "disable": return []*tls.Config{nil}, nil @@ -711,23 +744,6 @@ func configTLS(settings map[string]string, thisHost string, parseConfigOptions P return nil, errors.New("sslmode is invalid") } - if sslrootcert != "" { - caCertPool := x509.NewCertPool() - - caPath := sslrootcert - caCert, err := os.ReadFile(caPath) - if err != nil { - return nil, fmt.Errorf("unable to read CA file: %w", err) - } - - if !caCertPool.AppendCertsFromPEM(caCert) { - return nil, errors.New("unable to add CA to cert pool") - } - - tlsConfig.RootCAs = caCertPool - tlsConfig.ClientCAs = caCertPool - } - if (sslcert != "" && sslkey == "") || (sslcert == "" && sslkey != "") { return nil, errors.New(`both "sslcert" and "sslkey" are required`) } diff --git a/vendor/github.com/jackc/pgx/v5/pgtype/doc.go b/vendor/github.com/jackc/pgx/v5/pgtype/doc.go index d56c1dc701..7687ea8fee 100644 --- a/vendor/github.com/jackc/pgx/v5/pgtype/doc.go +++ b/vendor/github.com/jackc/pgx/v5/pgtype/doc.go @@ -53,6 +53,9 @@ similar fashion to database/sql. The second is to use a pointer to a pointer. return err } +When using nullable pgtype types as parameters for queries, one has to remember +to explicitly set their Valid field to true, otherwise the parameter's value will be NULL. + JSON Support pgtype automatically marshals and unmarshals data from json and jsonb PostgreSQL types. diff --git a/vendor/github.com/jackc/pgx/v5/pgtype/interval.go b/vendor/github.com/jackc/pgx/v5/pgtype/interval.go index 06703d4dc1..4b51162953 100644 --- a/vendor/github.com/jackc/pgx/v5/pgtype/interval.go +++ b/vendor/github.com/jackc/pgx/v5/pgtype/interval.go @@ -132,29 +132,25 @@ func (encodePlanIntervalCodecText) Encode(value any, buf []byte) (newBuf []byte, if interval.Days != 0 { buf = append(buf, strconv.FormatInt(int64(interval.Days), 10)...) - buf = append(buf, " day"...) + buf = append(buf, " day "...) } - if interval.Microseconds != 0 { - buf = append(buf, " "...) - - absMicroseconds := interval.Microseconds - if absMicroseconds < 0 { - absMicroseconds = -absMicroseconds - buf = append(buf, '-') - } + absMicroseconds := interval.Microseconds + if absMicroseconds < 0 { + absMicroseconds = -absMicroseconds + buf = append(buf, '-') + } - hours := absMicroseconds / microsecondsPerHour - minutes := (absMicroseconds % microsecondsPerHour) / microsecondsPerMinute - seconds := (absMicroseconds % microsecondsPerMinute) / microsecondsPerSecond + hours := absMicroseconds / microsecondsPerHour + minutes := (absMicroseconds % microsecondsPerHour) / microsecondsPerMinute + seconds := (absMicroseconds % microsecondsPerMinute) / microsecondsPerSecond - timeStr := fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) - buf = append(buf, timeStr...) + timeStr := fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + buf = append(buf, timeStr...) - microseconds := absMicroseconds % microsecondsPerSecond - if microseconds != 0 { - buf = append(buf, fmt.Sprintf(".%06d", microseconds)...) - } + microseconds := absMicroseconds % microsecondsPerSecond + if microseconds != 0 { + buf = append(buf, fmt.Sprintf(".%06d", microseconds)...) } return buf, nil diff --git a/vendor/github.com/jackc/pgx/v5/pgtype/json.go b/vendor/github.com/jackc/pgx/v5/pgtype/json.go index e71dcb9bf9..c2aa0d3bf8 100644 --- a/vendor/github.com/jackc/pgx/v5/pgtype/json.go +++ b/vendor/github.com/jackc/pgx/v5/pgtype/json.go @@ -37,7 +37,7 @@ func (c *JSONCodec) PlanEncode(m *Map, oid uint32, format int16, value any) Enco // // https://github.com/jackc/pgx/issues/1430 // - // Check for driver.Valuer must come before json.Marshaler so that it is guaranteed to beused + // Check for driver.Valuer must come before json.Marshaler so that it is guaranteed to be used // when both are implemented https://github.com/jackc/pgx/issues/1805 case driver.Valuer: return &encodePlanDriverValuer{m: m, oid: oid, formatCode: format} @@ -177,13 +177,6 @@ func (scanPlanJSONToByteSlice) Scan(src []byte, dst any) error { return nil } -type scanPlanJSONToBytesScanner struct{} - -func (scanPlanJSONToBytesScanner) Scan(src []byte, dst any) error { - scanner := (dst).(BytesScanner) - return scanner.ScanBytes(src) -} - type scanPlanJSONToJSONUnmarshal struct { unmarshal func(data []byte, v any) error } diff --git a/vendor/github.com/jackc/pgx/v5/pgtype/pgtype.go b/vendor/github.com/jackc/pgx/v5/pgtype/pgtype.go index 4082956832..bdd9f05cad 100644 --- a/vendor/github.com/jackc/pgx/v5/pgtype/pgtype.go +++ b/vendor/github.com/jackc/pgx/v5/pgtype/pgtype.go @@ -26,6 +26,8 @@ const ( XIDOID = 28 CIDOID = 29 JSONOID = 114 + XMLOID = 142 + XMLArrayOID = 143 JSONArrayOID = 199 PointOID = 600 LsegOID = 601 @@ -214,6 +216,15 @@ type Map struct { TryWrapScanPlanFuncs []TryWrapScanPlanFunc } +// Copy returns a new Map containing the same registered types. +func (m *Map) Copy() *Map { + newMap := NewMap() + for _, type_ := range m.oidToType { + newMap.RegisterType(type_) + } + return newMap +} + func NewMap() *Map { defaultMapInitOnce.Do(initDefaultMap) @@ -248,6 +259,13 @@ func NewMap() *Map { } } +// RegisterTypes registers multiple data types in the sequence they are provided. +func (m *Map) RegisterTypes(types []*Type) { + for _, t := range types { + m.RegisterType(t) + } +} + // RegisterType registers a data type with the Map. t must not be mutated after it is registered. func (m *Map) RegisterType(t *Type) { m.oidToType[t.OID] = t @@ -555,17 +573,24 @@ func TryFindUnderlyingTypeScanPlan(dst any) (plan WrappedScanPlanNextSetter, nex elemValue = dstValue.Elem() } nextDstType := elemKindToPointerTypes[elemValue.Kind()] - if nextDstType == nil && elemValue.Kind() == reflect.Slice { - if elemValue.Type().Elem().Kind() == reflect.Uint8 { - var v *[]byte - nextDstType = reflect.TypeOf(v) + if nextDstType == nil { + if elemValue.Kind() == reflect.Slice { + if elemValue.Type().Elem().Kind() == reflect.Uint8 { + var v *[]byte + nextDstType = reflect.TypeOf(v) + } + } + + // Get underlying type of any array. + // https://github.com/jackc/pgx/issues/2107 + if elemValue.Kind() == reflect.Array { + nextDstType = reflect.PointerTo(reflect.ArrayOf(elemValue.Len(), elemValue.Type().Elem())) } } if nextDstType != nil && dstValue.Type() != nextDstType && dstValue.CanConvert(nextDstType) { return &underlyingTypeScanPlan{dstType: dstValue.Type(), nextDstType: nextDstType}, dstValue.Convert(nextDstType).Interface(), true } - } return nil, nil, false @@ -1405,6 +1430,15 @@ func TryWrapFindUnderlyingTypeEncodePlan(value any) (plan WrappedEncodePlanNextS return &underlyingTypeEncodePlan{nextValueType: byteSliceType}, refValue.Convert(byteSliceType).Interface(), true } + // Get underlying type of any array. + // https://github.com/jackc/pgx/issues/2107 + if refValue.Kind() == reflect.Array { + underlyingArrayType := reflect.ArrayOf(refValue.Len(), refValue.Type().Elem()) + if refValue.Type() != underlyingArrayType { + return &underlyingTypeEncodePlan{nextValueType: underlyingArrayType}, refValue.Convert(underlyingArrayType).Interface(), true + } + } + return nil, nil, false } diff --git a/vendor/github.com/jackc/pgx/v5/pgtype/pgtype_default.go b/vendor/github.com/jackc/pgx/v5/pgtype/pgtype_default.go index 9525f37c92..c812573112 100644 --- a/vendor/github.com/jackc/pgx/v5/pgtype/pgtype_default.go +++ b/vendor/github.com/jackc/pgx/v5/pgtype/pgtype_default.go @@ -2,6 +2,7 @@ package pgtype import ( "encoding/json" + "encoding/xml" "net" "net/netip" "reflect" @@ -89,6 +90,7 @@ func initDefaultMap() { defaultMap.RegisterType(&Type{Name: "varbit", OID: VarbitOID, Codec: BitsCodec{}}) defaultMap.RegisterType(&Type{Name: "varchar", OID: VarcharOID, Codec: TextCodec{}}) defaultMap.RegisterType(&Type{Name: "xid", OID: XIDOID, Codec: Uint32Codec{}}) + defaultMap.RegisterType(&Type{Name: "xml", OID: XMLOID, Codec: &XMLCodec{Marshal: xml.Marshal, Unmarshal: xml.Unmarshal}}) // Range types defaultMap.RegisterType(&Type{Name: "daterange", OID: DaterangeOID, Codec: &RangeCodec{ElementType: defaultMap.oidToType[DateOID]}}) @@ -153,6 +155,7 @@ func initDefaultMap() { defaultMap.RegisterType(&Type{Name: "_varbit", OID: VarbitArrayOID, Codec: &ArrayCodec{ElementType: defaultMap.oidToType[VarbitOID]}}) defaultMap.RegisterType(&Type{Name: "_varchar", OID: VarcharArrayOID, Codec: &ArrayCodec{ElementType: defaultMap.oidToType[VarcharOID]}}) defaultMap.RegisterType(&Type{Name: "_xid", OID: XIDArrayOID, Codec: &ArrayCodec{ElementType: defaultMap.oidToType[XIDOID]}}) + defaultMap.RegisterType(&Type{Name: "_xml", OID: XMLArrayOID, Codec: &ArrayCodec{ElementType: defaultMap.oidToType[XMLOID]}}) // Integer types that directly map to a PostgreSQL type registerDefaultPgTypeVariants[int16](defaultMap, "int2") diff --git a/vendor/github.com/jackc/pgx/v5/pgtype/time.go b/vendor/github.com/jackc/pgx/v5/pgtype/time.go index 61a3abdfdb..f8fd94891f 100644 --- a/vendor/github.com/jackc/pgx/v5/pgtype/time.go +++ b/vendor/github.com/jackc/pgx/v5/pgtype/time.go @@ -19,9 +19,11 @@ type TimeValuer interface { // Time represents the PostgreSQL time type. The PostgreSQL time is a time of day without time zone. // -// Time is represented as the number of microseconds since midnight in the same way that PostgreSQL does. Other time -// and date types in pgtype can use time.Time as the underlying representation. However, pgtype.Time type cannot due -// to needing to handle 24:00:00. time.Time converts that to 00:00:00 on the following day. +// Time is represented as the number of microseconds since midnight in the same way that PostgreSQL does. Other time and +// date types in pgtype can use time.Time as the underlying representation. However, pgtype.Time type cannot due to +// needing to handle 24:00:00. time.Time converts that to 00:00:00 on the following day. +// +// The time with time zone type is not supported. Use of time with time zone is discouraged by the PostgreSQL documentation. type Time struct { Microseconds int64 // Number of microseconds since midnight Valid bool diff --git a/vendor/github.com/jackc/pgx/v5/pgtype/uint32.go b/vendor/github.com/jackc/pgx/v5/pgtype/uint32.go index 098c516c12..f2b2fa6d42 100644 --- a/vendor/github.com/jackc/pgx/v5/pgtype/uint32.go +++ b/vendor/github.com/jackc/pgx/v5/pgtype/uint32.go @@ -205,6 +205,8 @@ func (Uint32Codec) PlanScan(m *Map, oid uint32, format int16, target any) ScanPl return scanPlanBinaryUint32ToUint32{} case Uint32Scanner: return scanPlanBinaryUint32ToUint32Scanner{} + case TextScanner: + return scanPlanBinaryUint32ToTextScanner{} } case TextFormatCode: switch target.(type) { @@ -282,6 +284,26 @@ func (scanPlanBinaryUint32ToUint32Scanner) Scan(src []byte, dst any) error { return s.ScanUint32(Uint32{Uint32: n, Valid: true}) } +type scanPlanBinaryUint32ToTextScanner struct{} + +func (scanPlanBinaryUint32ToTextScanner) Scan(src []byte, dst any) error { + s, ok := (dst).(TextScanner) + if !ok { + return ErrScanTargetTypeChanged + } + + if src == nil { + return s.ScanText(Text{}) + } + + if len(src) != 4 { + return fmt.Errorf("invalid length for uint32: %v", len(src)) + } + + n := uint64(binary.BigEndian.Uint32(src)) + return s.ScanText(Text{String: strconv.FormatUint(n, 10), Valid: true}) +} + type scanPlanTextAnyToUint32Scanner struct{} func (scanPlanTextAnyToUint32Scanner) Scan(src []byte, dst any) error { diff --git a/vendor/github.com/jackc/pgx/v5/pgtype/xml.go b/vendor/github.com/jackc/pgx/v5/pgtype/xml.go new file mode 100644 index 0000000000..fb4c49ad9e --- /dev/null +++ b/vendor/github.com/jackc/pgx/v5/pgtype/xml.go @@ -0,0 +1,198 @@ +package pgtype + +import ( + "database/sql" + "database/sql/driver" + "encoding/xml" + "fmt" + "reflect" +) + +type XMLCodec struct { + Marshal func(v any) ([]byte, error) + Unmarshal func(data []byte, v any) error +} + +func (*XMLCodec) FormatSupported(format int16) bool { + return format == TextFormatCode || format == BinaryFormatCode +} + +func (*XMLCodec) PreferredFormat() int16 { + return TextFormatCode +} + +func (c *XMLCodec) PlanEncode(m *Map, oid uint32, format int16, value any) EncodePlan { + switch value.(type) { + case string: + return encodePlanXMLCodecEitherFormatString{} + case []byte: + return encodePlanXMLCodecEitherFormatByteSlice{} + + // Cannot rely on driver.Valuer being handled later because anything can be marshalled. + // + // https://github.com/jackc/pgx/issues/1430 + // + // Check for driver.Valuer must come before xml.Marshaler so that it is guaranteed to be used + // when both are implemented https://github.com/jackc/pgx/issues/1805 + case driver.Valuer: + return &encodePlanDriverValuer{m: m, oid: oid, formatCode: format} + + // Must come before trying wrap encode plans because a pointer to a struct may be unwrapped to a struct that can be + // marshalled. + // + // https://github.com/jackc/pgx/issues/1681 + case xml.Marshaler: + return &encodePlanXMLCodecEitherFormatMarshal{ + marshal: c.Marshal, + } + } + + // Because anything can be marshalled the normal wrapping in Map.PlanScan doesn't get a chance to run. So try the + // appropriate wrappers here. + for _, f := range []TryWrapEncodePlanFunc{ + TryWrapDerefPointerEncodePlan, + TryWrapFindUnderlyingTypeEncodePlan, + } { + if wrapperPlan, nextValue, ok := f(value); ok { + if nextPlan := c.PlanEncode(m, oid, format, nextValue); nextPlan != nil { + wrapperPlan.SetNext(nextPlan) + return wrapperPlan + } + } + } + + return &encodePlanXMLCodecEitherFormatMarshal{ + marshal: c.Marshal, + } +} + +type encodePlanXMLCodecEitherFormatString struct{} + +func (encodePlanXMLCodecEitherFormatString) Encode(value any, buf []byte) (newBuf []byte, err error) { + xmlString := value.(string) + buf = append(buf, xmlString...) + return buf, nil +} + +type encodePlanXMLCodecEitherFormatByteSlice struct{} + +func (encodePlanXMLCodecEitherFormatByteSlice) Encode(value any, buf []byte) (newBuf []byte, err error) { + xmlBytes := value.([]byte) + if xmlBytes == nil { + return nil, nil + } + + buf = append(buf, xmlBytes...) + return buf, nil +} + +type encodePlanXMLCodecEitherFormatMarshal struct { + marshal func(v any) ([]byte, error) +} + +func (e *encodePlanXMLCodecEitherFormatMarshal) Encode(value any, buf []byte) (newBuf []byte, err error) { + xmlBytes, err := e.marshal(value) + if err != nil { + return nil, err + } + + buf = append(buf, xmlBytes...) + return buf, nil +} + +func (c *XMLCodec) PlanScan(m *Map, oid uint32, format int16, target any) ScanPlan { + switch target.(type) { + case *string: + return scanPlanAnyToString{} + + case **string: + // This is to fix **string scanning. It seems wrong to special case **string, but it's not clear what a better + // solution would be. + // + // https://github.com/jackc/pgx/issues/1470 -- **string + // https://github.com/jackc/pgx/issues/1691 -- ** anything else + + if wrapperPlan, nextDst, ok := TryPointerPointerScanPlan(target); ok { + if nextPlan := m.planScan(oid, format, nextDst); nextPlan != nil { + if _, failed := nextPlan.(*scanPlanFail); !failed { + wrapperPlan.SetNext(nextPlan) + return wrapperPlan + } + } + } + + case *[]byte: + return scanPlanXMLToByteSlice{} + case BytesScanner: + return scanPlanBinaryBytesToBytesScanner{} + + // Cannot rely on sql.Scanner being handled later because scanPlanXMLToXMLUnmarshal will take precedence. + // + // https://github.com/jackc/pgx/issues/1418 + case sql.Scanner: + return &scanPlanSQLScanner{formatCode: format} + } + + return &scanPlanXMLToXMLUnmarshal{ + unmarshal: c.Unmarshal, + } +} + +type scanPlanXMLToByteSlice struct{} + +func (scanPlanXMLToByteSlice) Scan(src []byte, dst any) error { + dstBuf := dst.(*[]byte) + if src == nil { + *dstBuf = nil + return nil + } + + *dstBuf = make([]byte, len(src)) + copy(*dstBuf, src) + return nil +} + +type scanPlanXMLToXMLUnmarshal struct { + unmarshal func(data []byte, v any) error +} + +func (s *scanPlanXMLToXMLUnmarshal) Scan(src []byte, dst any) error { + if src == nil { + dstValue := reflect.ValueOf(dst) + if dstValue.Kind() == reflect.Ptr { + el := dstValue.Elem() + switch el.Kind() { + case reflect.Ptr, reflect.Slice, reflect.Map, reflect.Interface, reflect.Struct: + el.Set(reflect.Zero(el.Type())) + return nil + } + } + + return fmt.Errorf("cannot scan NULL into %T", dst) + } + + elem := reflect.ValueOf(dst).Elem() + elem.Set(reflect.Zero(elem.Type())) + + return s.unmarshal(src, dst) +} + +func (c *XMLCodec) DecodeDatabaseSQLValue(m *Map, oid uint32, format int16, src []byte) (driver.Value, error) { + if src == nil { + return nil, nil + } + + dstBuf := make([]byte, len(src)) + copy(dstBuf, src) + return dstBuf, nil +} + +func (c *XMLCodec) DecodeValue(m *Map, oid uint32, format int16, src []byte) (any, error) { + if src == nil { + return nil, nil + } + + var dst any + err := c.Unmarshal(src, &dst) + return dst, err +} diff --git a/vendor/github.com/jackc/pgx/v5/pgxpool/tx.go b/vendor/github.com/jackc/pgx/v5/pgxpool/tx.go index 74df8593a9..b49e7f4d96 100644 --- a/vendor/github.com/jackc/pgx/v5/pgxpool/tx.go +++ b/vendor/github.com/jackc/pgx/v5/pgxpool/tx.go @@ -18,9 +18,10 @@ func (tx *Tx) Begin(ctx context.Context) (pgx.Tx, error) { return tx.t.Begin(ctx) } -// Commit commits the transaction and returns the associated connection back to the Pool. Commit will return ErrTxClosed -// if the Tx is already closed, but is otherwise safe to call multiple times. If the commit fails with a rollback status -// (e.g. the transaction was already in a broken state) then ErrTxCommitRollback will be returned. +// Commit commits the transaction and returns the associated connection back to the Pool. Commit will return an error +// where errors.Is(ErrTxClosed) is true if the Tx is already closed, but is otherwise safe to call multiple times. If +// the commit fails with a rollback status (e.g. the transaction was already in a broken state) then ErrTxCommitRollback +// will be returned. func (tx *Tx) Commit(ctx context.Context) error { err := tx.t.Commit(ctx) if tx.c != nil { @@ -30,9 +31,9 @@ func (tx *Tx) Commit(ctx context.Context) error { return err } -// Rollback rolls back the transaction and returns the associated connection back to the Pool. Rollback will return ErrTxClosed -// if the Tx is already closed, but is otherwise safe to call multiple times. Hence, defer tx.Rollback() is safe even if -// tx.Commit() will be called first in a non-error condition. +// Rollback rolls back the transaction and returns the associated connection back to the Pool. Rollback will return +// where an error where errors.Is(ErrTxClosed) is true if the Tx is already closed, but is otherwise safe to call +// multiple times. Hence, defer tx.Rollback() is safe even if tx.Commit() will be called first in a non-error condition. func (tx *Tx) Rollback(ctx context.Context) error { err := tx.t.Rollback(ctx) if tx.c != nil { diff --git a/vendor/github.com/jackc/pgx/v5/rows.go b/vendor/github.com/jackc/pgx/v5/rows.go index d4f7a9016c..f23625d4c5 100644 --- a/vendor/github.com/jackc/pgx/v5/rows.go +++ b/vendor/github.com/jackc/pgx/v5/rows.go @@ -797,7 +797,7 @@ func computeNamedStructFields( if !dbTagPresent { colName = sf.Name } - fpos := fieldPosByName(fldDescs, colName) + fpos := fieldPosByName(fldDescs, colName, !dbTagPresent) if fpos == -1 { if missingField == "" { missingField = colName @@ -816,16 +816,21 @@ func computeNamedStructFields( const structTagKey = "db" -func fieldPosByName(fldDescs []pgconn.FieldDescription, field string) (i int) { +func fieldPosByName(fldDescs []pgconn.FieldDescription, field string, normalize bool) (i int) { i = -1 - for i, desc := range fldDescs { - // Snake case support. + if normalize { field = strings.ReplaceAll(field, "_", "") - descName := strings.ReplaceAll(desc.Name, "_", "") - - if strings.EqualFold(descName, field) { - return i + } + for i, desc := range fldDescs { + if normalize { + if strings.EqualFold(strings.ReplaceAll(desc.Name, "_", ""), field) { + return i + } + } else { + if desc.Name == field { + return i + } } } return diff --git a/vendor/github.com/jackc/pgx/v5/stdlib/sql.go b/vendor/github.com/jackc/pgx/v5/stdlib/sql.go index 29cd3fbbff..c1d00ab407 100644 --- a/vendor/github.com/jackc/pgx/v5/stdlib/sql.go +++ b/vendor/github.com/jackc/pgx/v5/stdlib/sql.go @@ -75,6 +75,7 @@ import ( "math" "math/rand" "reflect" + "slices" "strconv" "strings" "sync" @@ -98,7 +99,7 @@ func init() { // if pgx driver was already registered by different pgx major version then we // skip registration under the default name. - if !contains(sql.Drivers(), "pgx") { + if !slices.Contains(sql.Drivers(), "pgx") { sql.Register("pgx", pgxDriver) } sql.Register("pgx/v5", pgxDriver) @@ -120,17 +121,6 @@ func init() { } } -// TODO replace by slices.Contains when experimental package will be merged to stdlib -// https://pkg.go.dev/golang.org/x/exp/slices#Contains -func contains(list []string, y string) bool { - for _, x := range list { - if x == y { - return true - } - } - return false -} - // OptionOpenDB options for configuring the driver when opening a new db pool. type OptionOpenDB func(*connector) @@ -805,6 +795,16 @@ func (r *Rows) Next(dest []driver.Value) error { } return d.Value() } + case pgtype.XMLOID: + var d []byte + scanPlan := m.PlanScan(dataTypeOID, format, &d) + r.valueFuncs[i] = func(src []byte) (driver.Value, error) { + err := scanPlan.Scan(src, &d) + if err != nil { + return nil, err + } + return d, nil + } default: var d string scanPlan := m.PlanScan(dataTypeOID, format, &d) diff --git a/vendor/github.com/jackc/puddle/v2/CHANGELOG.md b/vendor/github.com/jackc/puddle/v2/CHANGELOG.md index a15991c581..d0d202c74a 100644 --- a/vendor/github.com/jackc/puddle/v2/CHANGELOG.md +++ b/vendor/github.com/jackc/puddle/v2/CHANGELOG.md @@ -1,3 +1,8 @@ +# 2.2.2 (September 10, 2024) + +* Add empty acquire time to stats (Maxim Ivanov) +* Stop importing nanotime from runtime via linkname (maypok86) + # 2.2.1 (July 15, 2023) * Fix: CreateResource cannot overflow pool. This changes documented behavior of CreateResource. Previously, diff --git a/vendor/github.com/jackc/puddle/v2/README.md b/vendor/github.com/jackc/puddle/v2/README.md index 0ad07ec430..fa82a9d46f 100644 --- a/vendor/github.com/jackc/puddle/v2/README.md +++ b/vendor/github.com/jackc/puddle/v2/README.md @@ -1,4 +1,4 @@ -[![](https://godoc.org/github.com/jackc/puddle?status.svg)](https://godoc.org/github.com/jackc/puddle) +[![Go Reference](https://pkg.go.dev/badge/github.com/jackc/puddle/v2.svg)](https://pkg.go.dev/github.com/jackc/puddle/v2) ![Build Status](https://github.com/jackc/puddle/actions/workflows/ci.yml/badge.svg) # Puddle diff --git a/vendor/github.com/jackc/puddle/v2/nanotime.go b/vendor/github.com/jackc/puddle/v2/nanotime.go new file mode 100644 index 0000000000..8a5351a0df --- /dev/null +++ b/vendor/github.com/jackc/puddle/v2/nanotime.go @@ -0,0 +1,16 @@ +package puddle + +import "time" + +// nanotime returns the time in nanoseconds since process start. +// +// This approach, described at +// https://github.com/golang/go/issues/61765#issuecomment-1672090302, +// is fast, monotonic, and portable, and avoids the previous +// dependence on runtime.nanotime using the (unsafe) linkname hack. +// In particular, time.Since does less work than time.Now. +func nanotime() int64 { + return time.Since(globalStart).Nanoseconds() +} + +var globalStart = time.Now() diff --git a/vendor/github.com/jackc/puddle/v2/nanotime_time.go b/vendor/github.com/jackc/puddle/v2/nanotime_time.go deleted file mode 100644 index f8e759386b..0000000000 --- a/vendor/github.com/jackc/puddle/v2/nanotime_time.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build purego || appengine || js - -// This file contains the safe implementation of nanotime using time.Now(). - -package puddle - -import ( - "time" -) - -func nanotime() int64 { - return time.Now().UnixNano() -} diff --git a/vendor/github.com/jackc/puddle/v2/nanotime_unsafe.go b/vendor/github.com/jackc/puddle/v2/nanotime_unsafe.go deleted file mode 100644 index fc3b8a258d..0000000000 --- a/vendor/github.com/jackc/puddle/v2/nanotime_unsafe.go +++ /dev/null @@ -1,12 +0,0 @@ -//go:build !purego && !appengine && !js - -// This file contains the implementation of nanotime using runtime.nanotime. - -package puddle - -import "unsafe" - -var _ = unsafe.Sizeof(0) - -//go:linkname nanotime runtime.nanotime -func nanotime() int64 diff --git a/vendor/github.com/jackc/puddle/v2/pool.go b/vendor/github.com/jackc/puddle/v2/pool.go index c8edc0fb68..c411d2f6ef 100644 --- a/vendor/github.com/jackc/puddle/v2/pool.go +++ b/vendor/github.com/jackc/puddle/v2/pool.go @@ -139,6 +139,7 @@ type Pool[T any] struct { acquireCount int64 acquireDuration time.Duration emptyAcquireCount int64 + emptyAcquireWaitTime time.Duration canceledAcquireCount atomic.Int64 resetCount int @@ -154,7 +155,7 @@ type Config[T any] struct { MaxSize int32 } -// NewPool creates a new pool. Panics if maxSize is less than 1. +// NewPool creates a new pool. Returns an error iff MaxSize is less than 1. func NewPool[T any](config *Config[T]) (*Pool[T], error) { if config.MaxSize < 1 { return nil, errors.New("MaxSize must be >= 1") @@ -202,6 +203,7 @@ type Stat struct { acquireCount int64 acquireDuration time.Duration emptyAcquireCount int64 + emptyAcquireWaitTime time.Duration canceledAcquireCount int64 } @@ -251,6 +253,13 @@ func (s *Stat) EmptyAcquireCount() int64 { return s.emptyAcquireCount } +// EmptyAcquireWaitTime returns the cumulative time waited for successful acquires +// from the pool for a resource to be released or constructed because the pool was +// empty. +func (s *Stat) EmptyAcquireWaitTime() time.Duration { + return s.emptyAcquireWaitTime +} + // CanceledAcquireCount returns the cumulative count of acquires from the pool // that were canceled by a context. func (s *Stat) CanceledAcquireCount() int64 { @@ -266,6 +275,7 @@ func (p *Pool[T]) Stat() *Stat { maxResources: p.maxSize, acquireCount: p.acquireCount, emptyAcquireCount: p.emptyAcquireCount, + emptyAcquireWaitTime: p.emptyAcquireWaitTime, canceledAcquireCount: p.canceledAcquireCount.Load(), acquireDuration: p.acquireDuration, } @@ -363,11 +373,13 @@ func (p *Pool[T]) acquire(ctx context.Context) (*Resource[T], error) { // If a resource is available in the pool. if res := p.tryAcquireIdleResource(); res != nil { + waitTime := time.Duration(nanotime() - startNano) if waitedForLock { p.emptyAcquireCount += 1 + p.emptyAcquireWaitTime += waitTime } p.acquireCount += 1 - p.acquireDuration += time.Duration(nanotime() - startNano) + p.acquireDuration += waitTime p.mux.Unlock() return res, nil } @@ -391,7 +403,9 @@ func (p *Pool[T]) acquire(ctx context.Context) (*Resource[T], error) { p.emptyAcquireCount += 1 p.acquireCount += 1 - p.acquireDuration += time.Duration(nanotime() - startNano) + waitTime := time.Duration(nanotime() - startNano) + p.acquireDuration += waitTime + p.emptyAcquireWaitTime += waitTime return res, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 92905c133c..e17e3e6736 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -413,11 +413,11 @@ github.com/inconshreveable/mousetrap # github.com/jackc/pgpassfile v1.0.0 ## explicit; go 1.12 github.com/jackc/pgpassfile -# github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a +# github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 ## explicit; go 1.14 github.com/jackc/pgservicefile -# github.com/jackc/pgx/v5 v5.6.0 -## explicit; go 1.20 +# github.com/jackc/pgx/v5 v5.7.1 +## explicit; go 1.21 github.com/jackc/pgx/v5 github.com/jackc/pgx/v5/internal/iobufpool github.com/jackc/pgx/v5/internal/pgio @@ -430,7 +430,7 @@ github.com/jackc/pgx/v5/pgproto3 github.com/jackc/pgx/v5/pgtype github.com/jackc/pgx/v5/pgxpool github.com/jackc/pgx/v5/stdlib -# github.com/jackc/puddle/v2 v2.2.1 +# github.com/jackc/puddle/v2 v2.2.2 ## explicit; go 1.19 github.com/jackc/puddle/v2 github.com/jackc/puddle/v2/internal/genstack @@ -1104,7 +1104,7 @@ golang.org/x/image/webp golang.org/x/mod/internal/lazyregexp golang.org/x/mod/module golang.org/x/mod/semver -# golang.org/x/net v0.28.0 +# golang.org/x/net v0.29.0 ## explicit; go 1.18 golang.org/x/net/bpf golang.org/x/net/context From 6b7665c39128b44b864578090623a3c72c2754c5 Mon Sep 17 00:00:00 2001 From: tobi Date: Mon, 16 Sep 2024 13:15:19 +0200 Subject: [PATCH 10/11] put cache in caches struct --- internal/cache/cache.go | 15 +++++++++++++++ internal/typeutils/converter.go | 23 ++++------------------- internal/typeutils/internaltofrontend.go | 8 +++++--- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 5554445b2f..68a31f11ee 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -47,6 +47,11 @@ type Caches struct { // Webfinger provides access to the webfinger URL cache. Webfinger *ttl.Cache[string, string] // TTL=24hr, sweep=5min + // TTL cache of statuses -> filterable text fields. + // To ensure up-to-date fields, cache is keyed as: + // `[status.ID][status.UpdatedAt.Unix()]` + StatusesFilterableFields *ttl.Cache[string, []string] + // prevent pass-by-value. _ nocopy } @@ -109,6 +114,7 @@ func (c *Caches) Init() { c.initUserMuteIDs() c.initWebfinger() c.initVisibility() + c.initStatusesFilterableFields() } // Start will start any caches that require a background @@ -204,3 +210,12 @@ func (c *Caches) initWebfinger() { 24*time.Hour, ) } + +func (c *Caches) initStatusesFilterableFields() { + c.Webfinger = new(ttl.Cache[string, string]) + c.Webfinger.Init( + 0, + 512, + 1*time.Hour, + ) +} diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 97d661a5d5..311839dc0a 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -19,12 +19,9 @@ package typeutils import ( "sync" - "time" - "codeberg.org/gruf/go-cache/v3" "github.com/superseriousbusiness/gotosocial/internal/filter/interaction" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" - "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/state" ) @@ -34,25 +31,13 @@ type Converter struct { randAvatars sync.Map visFilter *visibility.Filter intFilter *interaction.Filter - - // TTL cache of statuses -> filterable text fields. - // To ensure up-to-date fields, cache is keyed as: - // [status.ID][status.UpdatedAt.Unix()]` - statusesFilterableFields cache.TTLCache[string, []string] } func NewConverter(state *state.State) *Converter { - statusHashesToFilterableText := cache.NewTTL[string, []string](0, 512, 0) - statusHashesToFilterableText.SetTTL(time.Hour, true) - if !statusHashesToFilterableText.Start(time.Minute) { - log.Panic(nil, "failed to start statusHashesToFilterableText cache") - } - return &Converter{ - state: state, - defaultAvatars: populateDefaultAvatars(), - visFilter: visibility.NewFilter(state), - intFilter: interaction.NewFilter(state), - statusesFilterableFields: statusHashesToFilterableText, + state: state, + defaultAvatars: populateDefaultAvatars(), + visFilter: visibility.NewFilter(state), + intFilter: interaction.NewFilter(state), } } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 58797e7901..fe49766fad 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "slices" + "strconv" "strings" "time" @@ -941,15 +942,16 @@ func (c *Converter) statusToAPIFilterResults( // Key this status based on ID + last updated time, // to ensure we always filter on latest version. - statusKey := fmt.Sprintf("%s%d", s.ID, s.UpdatedAt.Unix()) + statusKey := s.ID + strconv.FormatInt(s.UpdatedAt.Unix(), 10) // Check if we have filterable fields cached for this status. - fields, stored := c.statusesFilterableFields.Get(statusKey) + cache := c.state.Caches.StatusesFilterableFields + fields, stored := cache.Get(statusKey) if !stored { // We don't have filterable fields // cached, calculate + cache now. fields = filterableFields(s) - c.statusesFilterableFields.Set(statusKey, fields) + cache.Set(statusKey, fields) } // Record all matching warn filters and the reasons they matched. From 85261d8982310d7ce4bee82c30ab3ffd36b05131 Mon Sep 17 00:00:00 2001 From: tobi Date: Mon, 16 Sep 2024 13:36:44 +0200 Subject: [PATCH 11/11] pain --- internal/cache/cache.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 68a31f11ee..8291dec5ab 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -125,6 +125,10 @@ func (c *Caches) Start() { tryUntil("starting webfinger cache", 5, func() bool { return c.Webfinger.Start(5 * time.Minute) }) + + tryUntil("starting statusesFilterableFields cache", 5, func() bool { + return c.StatusesFilterableFields.Start(5 * time.Minute) + }) } // Stop will stop any caches that require a background @@ -133,6 +137,7 @@ func (c *Caches) Stop() { log.Infof(nil, "stop: %p", c) tryUntil("stopping webfinger cache", 5, c.Webfinger.Stop) + tryUntil("stopping statusesFilterableFields cache", 5, c.StatusesFilterableFields.Stop) } // Sweep will sweep all the available caches to ensure none @@ -212,8 +217,8 @@ func (c *Caches) initWebfinger() { } func (c *Caches) initStatusesFilterableFields() { - c.Webfinger = new(ttl.Cache[string, string]) - c.Webfinger.Init( + c.StatusesFilterableFields = new(ttl.Cache[string, []string]) + c.StatusesFilterableFields.Init( 0, 512, 1*time.Hour,