From 74daca6b3050d96cd3a08b13f45de24edb450163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 14 Sep 2022 11:58:45 +0200 Subject: [PATCH] Support PostProcess for all file types Not just HTML. Fixes #10269 --- commands/hugo.go | 13 +++-- common/hugio/hasBytesWriter.go | 57 ++++++++++++++++++ common/hugio/hasBytesWriter_test.go | 64 ++++++++++++++++++++ deps/deps.go | 20 ++++++- hugofs/hasbytes_fs.go | 90 +++++++++++++++++++++++++++++ hugolib/hugo_sites_build.go | 19 ++---- hugolib/resource_chain_test.go | 10 ++++ hugolib/testhelpers_test.go | 8 ++- resources/transform_test.go | 8 ++- 9 files changed, 265 insertions(+), 24 deletions(-) create mode 100644 common/hugio/hasBytesWriter.go create mode 100644 common/hugio/hasBytesWriter_test.go create mode 100644 hugofs/hasbytes_fs.go diff --git a/commands/hugo.go b/commands/hugo.go index 5169d65a52e..d127d37212f 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -511,12 +511,15 @@ func (c *commandeer) build() error { c.hugo().PrintProcessingStats(os.Stdout) fmt.Println() - if createCounter, ok := c.publishDirFs.(hugofs.DuplicatesReporter); ok { - dupes := createCounter.ReportDuplicates() - if dupes != "" { - c.logger.Warnln("Duplicate target paths:", dupes) + hugofs.WalkFilesystems(c.publishDirFs, func(fs afero.Fs) bool { + if dfs, ok := fs.(hugofs.DuplicatesReporter); ok { + dupes := dfs.ReportDuplicates() + if dupes != "" { + c.logger.Warnln("Duplicate target paths:", dupes) + } } - } + return false + }) unusedTemplates := c.hugo().Tmpl().(tpl.UnusedTemplatesProvider).UnusedTemplates() for _, unusedTemplate := range unusedTemplates { diff --git a/common/hugio/hasBytesWriter.go b/common/hugio/hasBytesWriter.go new file mode 100644 index 00000000000..7b7d7a5d756 --- /dev/null +++ b/common/hugio/hasBytesWriter.go @@ -0,0 +1,57 @@ +// Copyright 2022 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugio + +import ( + "bytes" +) + +// HasBytesWriter is a writer that will set Match to true if the given pattern +// is found in the stream. +type HasBytesWriter struct { + Match bool + Pattern []byte + + i int + done bool + buff []byte +} + +func (h *HasBytesWriter) Write(p []byte) (n int, err error) { + if h.done { + return len(p), nil + } + + if len(h.buff) == 0 { + h.buff = make([]byte, len(h.Pattern)*2) + } + + for i := range p { + h.buff[h.i] = p[i] + h.i++ + if h.i == len(h.buff) { + // Shift left. + copy(h.buff, h.buff[len(h.buff)/2:]) + h.i = len(h.buff) / 2 + } + + if bytes.Contains(h.buff, h.Pattern) { + h.Match = true + h.done = true + return len(p), nil + } + } + + return len(p), nil +} diff --git a/common/hugio/hasBytesWriter_test.go b/common/hugio/hasBytesWriter_test.go new file mode 100644 index 00000000000..b1b8011d5db --- /dev/null +++ b/common/hugio/hasBytesWriter_test.go @@ -0,0 +1,64 @@ +// Copyright 2022 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugio + +import ( + "bytes" + "fmt" + "io" + "math/rand" + "strings" + "testing" + "time" + + qt "github.com/frankban/quicktest" +) + +func TestHasBytesWriter(t *testing.T) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + c := qt.New((t)) + + neww := func() (*HasBytesWriter, io.Writer) { + var b bytes.Buffer + + h := &HasBytesWriter{ + Pattern: []byte("__foo"), + } + return h, io.MultiWriter(&b, h) + } + + rndStr := func() string { + return strings.Repeat("ab cfo", r.Intn(33)) + } + + for i := 0; i < 22; i++ { + h, w := neww() + fmt.Fprintf(w, rndStr()+"abc __foobar"+rndStr()) + c.Assert(h.Match, qt.Equals, true) + + h, w = neww() + fmt.Fprintf(w, rndStr()+"abc __f") + fmt.Fprintf(w, "oo bar"+rndStr()) + c.Assert(h.Match, qt.Equals, true) + + h, w = neww() + fmt.Fprintf(w, rndStr()+"abc __moo bar") + c.Assert(h.Match, qt.Equals, false) + } + + h, w := neww() + fmt.Fprintf(w, "__foo") + c.Assert(h.Match, qt.Equals, true) +} diff --git a/deps/deps.go b/deps/deps.go index ece4203024f..e1cbfce069e 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -16,6 +16,7 @@ import ( "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/postpub" "github.com/gohugoio/hugo/metrics" "github.com/gohugoio/hugo/output" @@ -78,6 +79,10 @@ type Deps struct { // All the output formats available for the current site. OutputFormatsConfig output.Formats + // FilenameHasPostProcessPrefix is a set of filenames in /public that + // contains a post-processing prefix. + FilenameHasPostProcessPrefix []string + templateProvider ResourceProvider WithTemplate func(templ tpl.TemplateManager) error `json:"-"` @@ -202,6 +207,7 @@ func New(cfg DepsCfg) (*Deps, error) { var ( logger = cfg.Logger fs = cfg.Fs + d *Deps ) if cfg.TemplateProvider == nil { @@ -239,6 +245,18 @@ func New(cfg DepsCfg) (*Deps, error) { } execHelper := hexec.New(securityConfig) + var filenameHasPostProcessPrefixMu sync.Mutex + cb := func(name string, match bool) { + if !match { + return + } + filenameHasPostProcessPrefixMu.Lock() + d.FilenameHasPostProcessPrefix = append(d.FilenameHasPostProcessPrefix, name) + filenameHasPostProcessPrefixMu.Unlock() + + } + fs.PublishDir = hugofs.NewHasBytesReceiver(fs.PublishDir, cb, []byte(postpub.PostProcessPrefix)) + ps, err := helpers.NewPathSpec(fs, cfg.Language, logger) if err != nil { return nil, fmt.Errorf("create PathSpec: %w", err) @@ -274,7 +292,7 @@ func New(cfg DepsCfg) (*Deps, error) { logDistinct := helpers.NewDistinctLogger(logger) - d := &Deps{ + d = &Deps{ Fs: fs, Log: ignorableLogger, LogDistinct: logDistinct, diff --git a/hugofs/hasbytes_fs.go b/hugofs/hasbytes_fs.go new file mode 100644 index 00000000000..b5f82877e8c --- /dev/null +++ b/hugofs/hasbytes_fs.go @@ -0,0 +1,90 @@ +// Copyright 2022 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + + "github.com/gohugoio/hugo/common/hugio" + "github.com/spf13/afero" +) + +var ( + _ afero.Fs = (*hasBytesFs)(nil) + _ FilesystemUnwrapper = (*hasBytesFs)(nil) +) + +type hasBytesFs struct { + afero.Fs + hasBytesCallback func(name string, match bool) + pattern []byte +} + +func NewHasBytesReceiver(delegate afero.Fs, hasBytesCallback func(name string, match bool), pattern []byte) afero.Fs { + return &hasBytesFs{Fs: delegate, hasBytesCallback: hasBytesCallback, pattern: pattern} +} + +func (fs *hasBytesFs) UnwrapFilesystem() afero.Fs { + return fs.Fs +} + +func (fs *hasBytesFs) Create(name string) (afero.File, error) { + f, err := fs.Fs.Create(name) + if err == nil { + f = fs.wrapFile(f) + } + return f, err +} + +func (fs *hasBytesFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + f, err := fs.Fs.OpenFile(name, flag, perm) + if err == nil && isWrite(flag) { + f = fs.wrapFile(f) + } + return f, err +} + +func (fs *hasBytesFs) wrapFile(f afero.File) afero.File { + return &hasBytesFile{ + File: f, + hbw: &hugio.HasBytesWriter{ + Pattern: fs.pattern, + }, + hasBytesCallback: fs.hasBytesCallback, + } + +} + +func (fs *hasBytesFs) Name() string { + return "hasBytesFs" +} + +type hasBytesFile struct { + hasBytesCallback func(name string, match bool) + hbw *hugio.HasBytesWriter + afero.File +} + +func (h *hasBytesFile) Write(p []byte) (n int, err error) { + n, err = h.File.Write(p) + if err != nil { + return + } + return h.hbw.Write(p) +} + +func (h *hasBytesFile) Close() error { + h.hasBytesCallback(h.Name(), h.hbw.Match) + return h.File.Close() +} diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 1a191257c18..3bebc5284e1 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "fmt" - "os" "path/filepath" "runtime/trace" "strings" @@ -439,23 +438,15 @@ func (h *HugoSites) postProcess() error { return nil } - _ = afero.Walk(h.BaseFs.PublishFs, "", func(path string, info os.FileInfo, err error) error { - if info == nil || info.IsDir() { - return nil - } - - if !strings.HasSuffix(path, "html") { - return nil - } - + for _, filename := range h.Deps.FilenameHasPostProcessPrefix { + filename := filename g.Run(func() error { - return handleFile(path) + return handleFile(filename) }) - - return nil - }) + } // Prepare for a new build. + h.Deps.FilenameHasPostProcessPrefix = nil for _, s := range h.Sites { s.ResourceSpec.PostProcessResources = make(map[string]postpub.PostPublishedResource) } diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index d94d389a75a..4edc2cb31a3 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -168,6 +168,11 @@ HELLO: {{ $hello.RelPermalink }} HELLO: {{ $hello.RelPermalink }}|Integrity: {{ $hello.Data.Integrity }}|MediaType: {{ $hello.MediaType.Type }} HELLO2: Name: {{ $hello.Name }}|Content: {{ $hello.Content }}|Title: {{ $hello.Title }}|ResourceType: {{ $hello.ResourceType }} +// Issue #10269 +{{ $m := dict "relPermalink" $hello.RelPermalink "integrity" $hello.Data.Integrity "mediaType" $hello.MediaType.Type }} +{{ $json := jsonify (dict "indent" " ") $m | resources.FromString "hello.json" -}} +JSON: {{ $json.RelPermalink }} + // Issue #8884 foo Hello @@ -188,6 +193,11 @@ End.`) b.AssertFileContent("public/page1/index.html", `HELLO: /hello.min.a2d1cb24f24b322a7dad520414c523e9.html`) b.AssertFileContent("public/page2/index.html", `HELLO: /hello.min.a2d1cb24f24b322a7dad520414c523e9.html`) + b.AssertFileContent("public/hello.json", ` +integrity": "md5-otHLJPJLMip9rVIEFMUj6Q== +mediaType": "text/html +relPermalink": "/hello.min.a2d1cb24f24b322a7dad520414c523e9.html" +`) } func BenchmarkResourceChainPostProcess(b *testing.B) { diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 46fa35debff..ca74e9340e2 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -764,8 +764,12 @@ func (s *sitesBuilder) AssertImage(width, height int, filename string) { func (s *sitesBuilder) AssertNoDuplicateWrites() { s.Helper() - d := s.Fs.PublishDir.(hugofs.DuplicatesReporter) - s.Assert(d.ReportDuplicates(), qt.Equals, "") + hugofs.WalkFilesystems(s.Fs.PublishDir, func(fs afero.Fs) bool { + if dfs, ok := fs.(hugofs.DuplicatesReporter); ok { + s.Assert(dfs.ReportDuplicates(), qt.Equals, "") + } + return false + }) } func (s *sitesBuilder) FileContent(filename string) string { diff --git a/resources/transform_test.go b/resources/transform_test.go index af8ccbc1fe1..1bd8302d29b 100644 --- a/resources/transform_test.go +++ b/resources/transform_test.go @@ -71,8 +71,12 @@ func TestTransform(t *testing.T) { // Verify that we publish the same file once only. assertNoDuplicateWrites := func(c *qt.C, spec *Spec) { c.Helper() - d := spec.Fs.PublishDir.(hugofs.DuplicatesReporter) - c.Assert(d.ReportDuplicates(), qt.Equals, "") + hugofs.WalkFilesystems(spec.Fs.PublishDir, func(fs afero.Fs) bool { + if dfs, ok := fs.(hugofs.DuplicatesReporter); ok { + c.Assert(dfs.ReportDuplicates(), qt.Equals, "") + } + return false + }) } assertShouldExist := func(c *qt.C, spec *Spec, filename string, should bool) {