cmd/golangorg: generate release history page from structured data

Previously, the release history page was a raw HTML file that was
manually edited whenever new Go releases were made. This change
converts release history entries into a structured format in the
new internal/history package, and generates release history entries
from that format.

For now, only Go 1.9 and newer releases are converted, but the
structured format is flexible enough to represent all releases
going back to the original Go 1 release.

Various English grammar rules and special cases are preserved,
so that the release history entries appear in a consistent way.

New release history entries need only to be added to the internal/
history package, making it so that English grammar rules and HTML
tags don't need to go through human code review for each release.
Future work may involve constructing that list from data already
available in the Go issue tracker.

This change makes minimal contributions to reducing the dependence
of x/website on the x/tools/godoc rendering engine for displaying
pages other than Go package documentation. The x/tools/godoc code
is in another module and does not provide flexibility desired for
the general purpose website needs of x/website.

Fixes golang/go#38488.
For golang/go#37090.
For golang/go#29206.

Change-Id: I80864e4f218782e6e3b5fcd5a1d63f3699314c81
Reviewed-on: https://go-review.googlesource.com/c/website/+/229081
Run-TryBot: Dmitri Shuralyov <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Alexander Rakoczy <[email protected]>

cmd/golangorg/godoc.go

@@ -0,0 +1,64 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"golang.org/x/tools/godoc"
+	"golang.org/x/website/internal/env"
+)
+
+// This file holds common code from the x/tools/godoc serving engine.
+// It's being used during the transition. See golang.org/issue/29206.
+
+// extractMetadata extracts the godoc.Metadata from a byte slice.
+// It returns the godoc.Metadata value and the remaining data.
+// If no metadata is present the original byte slice is returned.
+//
+func extractMetadata(b []byte) (meta godoc.Metadata, tail []byte, _ error) {
+	tail = b
+	if !bytes.HasPrefix(b, jsonStart) {
+		return godoc.Metadata{}, tail, nil
+	}
+	end := bytes.Index(b, jsonEnd)
+	if end < 0 {
+		return godoc.Metadata{}, tail, nil
+	}
+	b = b[len(jsonStart)-1 : end+1] // drop leading <!-- and include trailing }
+	if err := json.Unmarshal(b, &meta); err != nil {
+		return godoc.Metadata{}, nil, err
+	}
+	tail = tail[end+len(jsonEnd):]
+	return meta, tail, nil
+}
+
+var (
+	jsonStart = []byte("<!--{")
+	jsonEnd   = []byte("}-->")
+)
+
+// googleCN reports whether request r is considered
+// to be served from golang.google.cn.
+// TODO: This is duplicated within internal/proxy. Move to a common location.
+func googleCN(r *http.Request) bool {
+	if r.FormValue("googlecn") != "" {
+		return true
+	}
+	if strings.HasSuffix(r.Host, ".cn") {
+		return true
+	}
+	if !env.CheckCountry() {
+		return false
+	}
+	switch r.Header.Get("X-Appengine-Country") {
+	case "", "ZZ", "CN":
+		return true
+	}
+	return false
+}

cmd/golangorg/handlers.go

@@ -23,6 +23,7 @@ import (
 	"golang.org/x/tools/godoc"
 	"golang.org/x/tools/godoc/vfs"
 	"golang.org/x/website/internal/env"
+	"golang.org/x/website/internal/history"
 	"golang.org/x/website/internal/redirect"
 )
 
@@ -85,6 +86,7 @@ func registerHandlers(pres *godoc.Presentation) *http.ServeMux {
 	mux.Handle("/", pres)
 	mux.Handle("/pkg/C/", redirect.Handler("/cmd/cgo/"))
 	mux.HandleFunc("/fmt", fmtHandler)
+	mux.Handle("/doc/devel/release.html", releaseHandler{ReleaseHistory: sortReleases(history.Releases)})
 	redirect.Register(mux)
 
 	http.Handle("/", hostEnforcerHandler{mux})

cmd/golangorg/regtest_test.go

@@ -123,6 +123,11 @@ func TestLiveServer(t *testing.T) {
 			Substring:   `<meta name="go-import" content="golang.org/x/net git https://go.googlesource.com/net">`,
 			NoAnalytics: true,
 		},
+		{
+			Message: "release history page has an entry for Go 1.14.2",
+			Path:    "/doc/devel/release.html",
+			Regexp:  `go1\.14\.2\s+\(released 2020/04/08\)\s+includes\s+fixes to cgo, the go command, the runtime,`,
+		},
 	}
 
 	for _, tc := range substringTests {

cmd/golangorg/release.go

@@ -0,0 +1,269 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"html"
+	"html/template"
+	"log"
+	"net/http"
+	"sort"
+	"strings"
+
+	"golang.org/x/tools/godoc"
+	"golang.org/x/tools/godoc/vfs"
+	"golang.org/x/website/internal/history"
+)
+
+// releaseHandler serves the Release History page.
+type releaseHandler struct {
+	ReleaseHistory []Major // Pre-computed release history to display.
+}
+
+func (h releaseHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	const relPath = "doc/devel/release.html"
+
+	src, err := vfs.ReadFile(fs, "/doc/devel/release.html")
+	if err != nil {
+		log.Printf("reading template %s: %v", relPath, err)
+		pres.ServeError(w, req, relPath, err)
+		return
+	}
+
+	meta, src, err := extractMetadata(src)
+	if err != nil {
+		log.Printf("decoding metadata %s: %v", relPath, err)
+		pres.ServeError(w, req, relPath, err)
+		return
+	}
+	if !meta.Template {
+		err := fmt.Errorf("got non-template, want template")
+		log.Printf("unexpected metadata %s: %v", relPath, err)
+		pres.ServeError(w, req, relPath, err)
+		return
+	}
+
+	page := godoc.Page{
+		Title:    meta.Title,
+		Subtitle: meta.Subtitle,
+		GoogleCN: googleCN(req),
+	}
+	data := releaseTemplateData{
+		Major: h.ReleaseHistory,
+	}
+
+	// Evaluate as HTML template.
+	tmpl, err := template.New("").Parse(string(src))
+	if err != nil {
+		log.Printf("parsing template %s: %v", relPath, err)
+		pres.ServeError(w, req, relPath, err)
+		return
+	}
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, data); err != nil {
+		log.Printf("executing template %s: %v", relPath, err)
+		pres.ServeError(w, req, relPath, err)
+		return
+	}
+	src = buf.Bytes()
+
+	page.Body = src
+	pres.ServePage(w, page)
+}
+
+// sortReleases returns a sorted list of Go releases, suitable to be
+// displayed on the Release History page. Releases are arranged into
+// major releases, each with minor revisions.
+func sortReleases(rs map[history.Version]history.Release) []Major {
+	var major []Major
+	byMajorVersion := make(map[history.Version]Major)
+	for v, r := range rs {
+		switch {
+		case v.IsMajor():
+			m := byMajorVersion[v]
+			m.Release = Release{ver: v, rel: r}
+			byMajorVersion[v] = m
+		case v.IsMinor():
+			m := byMajorVersion[majorOf(v)]
+			m.Minor = append(m.Minor, Release{ver: v, rel: r})
+			byMajorVersion[majorOf(v)] = m
+		}
+	}
+	for _, m := range byMajorVersion {
+		sort.Slice(m.Minor, func(i, j int) bool { return m.Minor[i].ver.Z < m.Minor[j].ver.Z })
+		major = append(major, m)
+	}
+	sort.Slice(major, func(i, j int) bool {
+		if major[i].ver.X != major[j].ver.X {
+			return major[i].ver.X > major[j].ver.X
+		}
+		return major[i].ver.Y > major[j].ver.Y
+	})
+	return major
+}
+
+// majorOf takes a Go version like 1.5, 1.5.1, 1.5.2, etc.,
+// and returns the corresponding major version like 1.5.
+func majorOf(v history.Version) history.Version {
+	return history.Version{X: v.X, Y: v.Y, Z: 0}
+}
+
+type releaseTemplateData struct {
+	Major []Major
+}
+
+// Major represents a major Go release and its minor revisions
+// as displayed on the release history page.
+type Major struct {
+	Release
+	Minor []Release
+}
+
+// Release represents a Go release entry as displayed on the release history page.
+type Release struct {
+	ver history.Version
+	rel history.Release
+}
+
+// V returns the Go release version string, like "1.14", "1.14.1", "1.14.2", etc.
+func (r Release) V() string {
+	switch {
+	case r.ver.Z != 0:
+		return fmt.Sprintf("%d.%d.%d", r.ver.X, r.ver.Y, r.ver.Z)
+	case r.ver.Y != 0:
+		return fmt.Sprintf("%d.%d", r.ver.X, r.ver.Y)
+	default:
+		return fmt.Sprintf("%d", r.ver.X)
+	}
+}
+
+// Date returns the date of the release, formatted for display on the release history page.
+func (r Release) Date() string {
+	d := r.rel.Date
+	return fmt.Sprintf("%04d/%02d/%02d", d.Year, d.Month, d.Day)
+}
+
+// Released reports whether release r has been released.
+func (r Release) Released() bool {
+	return !r.rel.Future
+}
+
+func (r Release) Summary() (template.HTML, error) {
+	var buf bytes.Buffer
+	err := releaseSummaryHTML.Execute(&buf, releaseSummaryTemplateData{
+		V:                     r.V(),
+		Security:              r.rel.Security,
+		Released:              r.Released(),
+		Quantifier:            r.rel.Quantifier,
+		ComponentsAndPackages: joinComponentsAndPackages(r.rel),
+		More:                  r.rel.More,
+		CustomSummary:         r.rel.CustomSummary,
+	})
+	return template.HTML(buf.String()), err
+}
+
+type releaseSummaryTemplateData struct {
+	V                     string        // Go release version string, like "1.14", "1.14.1", "1.14.2", etc.
+	Security              bool          // Security release.
+	Released              bool          // Whether release has been released.
+	Quantifier            string        // Optional quantifier. Empty string for unspecified amount of fixes (typical), "a" for a single fix, "two", "three" for multiple fixes, etc.
+	ComponentsAndPackages template.HTML // Components and packages involved.
+	More                  template.HTML // Additional release content.
+	CustomSummary         template.HTML // CustomSummary, if non-empty, replaces the entire release content summary with custom HTML.
+}
+
+var releaseSummaryHTML = template.Must(template.New("").Parse(`
+{{if not .CustomSummary}}
+	{{if .Released}}includes{{else}}will include{{end}}
+	{{.Quantifier}}
+	{{if .Security}}security{{end}}
+	{{if eq .Quantifier "a"}}fix{{else}}fixes{{end -}}
+	{{with .ComponentsAndPackages}} to {{.}}{{end}}.
+	{{.More}}
+
+	See the
+	<a href="https://github.com/golang/go/issues?q=milestone%3AGo{{.V}}+label%3ACherryPickApproved">Go
+	{{.V}} milestone</a> on our issue tracker for details.
+{{else}}
+	{{.CustomSummary}}
+{{end}}
+`))
+
+// joinComponentsAndPackages joins components and packages involved
+// in a Go release for the purposes of being displayed on the
+// release history page, keeping English grammar rules in mind.
+//
+// The different special cases are:
+//
+// 	c1
+// 	c1 and c2
+// 	c1, c2, and c3
+//
+// 	the p1 package
+// 	the p1 and p2 packages
+// 	the p1, p2, and p3 packages
+//
+// 	c1 and [1 package]
+// 	c1, and [2 or more packages]
+// 	c1, c2, and [1 or more packages]
+//
+func joinComponentsAndPackages(r history.Release) template.HTML {
+	var buf strings.Builder
+
+	// List components, if any.
+	for i, comp := range r.Components {
+		if len(r.Packages) == 0 {
+			// No packages, so components are joined with more rules.
+			switch {
+			case i != 0 && len(r.Components) == 2:
+				buf.WriteString(" and ")
+			case i != 0 && len(r.Components) >= 3 && i != len(r.Components)-1:
+				buf.WriteString(", ")
+			case i != 0 && len(r.Components) >= 3 && i == len(r.Components)-1:
+				buf.WriteString(", and ")
+			}
+		} else {
+			// When there are packages, all components are comma-separated.
+			if i != 0 {
+				buf.WriteString(", ")
+			}
+		}
+		buf.WriteString(string(comp))
+	}
+
+	// Join components and packages using a comma and/or "and" as needed.
+	if len(r.Components) > 0 && len(r.Packages) > 0 {
+		if len(r.Components)+len(r.Packages) >= 3 {
+			buf.WriteString(",")
+		}
+		buf.WriteString(" and ")
+	}
+
+	// List packages, if any.
+	if len(r.Packages) > 0 {
+		buf.WriteString("the ")
+	}
+	for i, pkg := range r.Packages {
+		switch {
+		case i != 0 && len(r.Packages) == 2:
+			buf.WriteString(" and ")
+		case i != 0 && len(r.Packages) >= 3 && i != len(r.Packages)-1:
+			buf.WriteString(", ")
+		case i != 0 && len(r.Packages) >= 3 && i == len(r.Packages)-1:
+			buf.WriteString(", and ")
+		}
+		buf.WriteString("<code>" + html.EscapeString(pkg) + "</code>")
+	}
+	switch {
+	case len(r.Packages) == 1:
+		buf.WriteString(" package")
+	case len(r.Packages) >= 2:
+		buf.WriteString(" packages")
+	}
+
+	return template.HTML(buf.String())
+}