From d8504a5e8804ed8f6205f857835336f5a754de84 Mon Sep 17 00:00:00 2001 From: bakape Date: Sat, 4 May 2019 11:59:58 +0300 Subject: [PATCH 01/10] cli: Remove inbuilt daemonization and config CLI addresses #992 --- auth/auth.go | 48 +++--- auth/auth_test.go | 26 ++-- auth/captcha.go | 2 +- cache/getters_test.go | 2 + cache/main.go | 7 +- cache/main_test.go | 9 +- config/config.go | 17 --- config/server.go | 87 +++++++++++ db/init.go | 44 +++--- db/init_test.go | 6 + db/upkeep.go | 8 +- db/util.go | 2 +- docs/config.json | 26 ++-- imager/main_test.go | 9 +- main_test.go | 17 --- parser/main_test.go | 9 +- server/daemon.go | 103 ------------- server/init.go | 272 ++++++--------------------------- server/router.go | 46 ++---- server/router_test.go | 21 +-- websockets/feeds/feeds_test.go | 10 +- websockets/websockets_test.go | 13 +- 22 files changed, 273 insertions(+), 511 deletions(-) create mode 100644 config/server.go delete mode 100644 main_test.go delete mode 100644 server/daemon.go diff --git a/auth/auth.go b/auth/auth.go index dba16e881..95c01690c 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -6,22 +6,13 @@ import ( "crypto/rand" "encoding/base64" "fmt" - "github.com/bakape/meguca/config" "net" "net/http" "strings" - "golang.org/x/crypto/bcrypt" -) - -var ( - // IsReverseProxied specifies, if the server is deployed behind a reverse - // proxy. - IsReverseProxied bool + "github.com/bakape/meguca/config" - // ReverseProxyIP specifies the IP of a non-localhost reverse proxy. Used - // for filtering in XFF IP determination. - ReverseProxyIP string + "golang.org/x/crypto/bcrypt" ) // IsBoard confirms the string is a valid board @@ -45,31 +36,26 @@ func GetIP(r *http.Request) (string, error) { } func getIP(req *http.Request) string { - if IsReverseProxied { - for _, h := range [...]string{"X-Forwarded-For", "X-Real-Ip"} { - addresses := strings.Split(req.Header.Get(h), ",") - - // March from right to left until we get a public address. - // That will be the address right before our reverse proxy. - for i := len(addresses) - 1; i >= 0; i-- { - // Header can contain padding spaces - ip := strings.TrimSpace(addresses[i]) - - // Filter the reverse proxy IPs - switch { - case ip == ReverseProxyIP: - case !net.ParseIP(ip).IsGlobalUnicast(): - default: - return ip - } + var ip string + if config.Server.Server.ReverseProxied { + h := req.Header.Get("X-Forwarded-For") + if h != "" { + if i := strings.LastIndexByte(h, ','); i != -1 { + h = h[i+1:] } + + ip = strings.TrimSpace(h) // Header can contain padding spaces } } - ip, _, err := net.SplitHostPort(req.RemoteAddr) + if ip == "" { + ip = req.RemoteAddr + } + + split, _, err := net.SplitHostPort(ip) if err != nil { - return req.RemoteAddr // No port in address + return ip // No port in address } - return ip + return split } // RandomID generates a randomID of base64 characters of desired byte length diff --git a/auth/auth_test.go b/auth/auth_test.go index 1f44ffc7a..9721f7eea 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -48,40 +48,38 @@ func TestGetIP(t *testing.T) { ip = "207.178.71.93" reverseProxyIP = "162.30.251.246" ) - IsReverseProxied = true - ReverseProxyIP = reverseProxyIP + config.Server.Server.ReverseProxied = true cases := [...]struct { name, xff, out string }{ - {"valid XFF", "10.121.169.19", "10.121.169.19"}, - {"no XFF", "", ip}, - {"invalid XFF", "notip, nope", ip}, { - "hosted on localhost", - "105.124.243.122, 10.168.239.157, 127.0.0.1, ::1", - "10.168.239.157", + name: "valid XFF", + xff: "10.121.169.19", + out: "10.121.169.19", }, { - "behind reverse proxy", - "105.124.243.122," + reverseProxyIP, - "105.124.243.122", + name: "no XFF", + out: ip, }, } for i := range cases { c := cases[i] t.Run(c.name, func(t *testing.T) { - t.Parallel() + // t.Parallel() req := httptest.NewRequest("GET", "/", nil) req.RemoteAddr = ip if c.xff != "" { req.Header.Set("X-Forwarded-For", c.xff) } - if i, _ := GetIP(req); i != c.out { - LogUnexpected(t, c.out, ip) + + res, err := GetIP(req) + if err != nil { + t.Fatal(err) } + AssertEquals(t, res, c.out) }) } } diff --git a/auth/captcha.go b/auth/captcha.go index 25d8b4e41..8d6f00009 100644 --- a/auth/captcha.go +++ b/auth/captcha.go @@ -112,7 +112,7 @@ func CaptchaService(board string) *captchouli.Service { // This function blocks until all services are initialized. func LoadCaptchaServices() (err error) { conf := config.Get() - if !conf.Captcha || config.ImagerMode == config.NoImager { + if !conf.Captcha || config.Server.ImagerMode == config.NoImager { return } diff --git a/cache/getters_test.go b/cache/getters_test.go index e55dd6176..9ee7e3955 100644 --- a/cache/getters_test.go +++ b/cache/getters_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/bakape/meguca/config" . "github.com/bakape/meguca/test" ) @@ -94,6 +95,7 @@ func TestGetHTML(t *testing.T) { func TestCounterExpiry(t *testing.T) { Clear() + config.Server.CacheSize = 1 << 7 var counterChecks, fetches int f := FrontEnd{ diff --git a/cache/main.go b/cache/main.go index 4faf3ada3..6957da349 100644 --- a/cache/main.go +++ b/cache/main.go @@ -6,6 +6,8 @@ import ( "container/list" "sync" "time" + + "github.com/bakape/meguca/config" ) // Time for the cache to expire and need counter comparison @@ -16,9 +18,6 @@ var ( ll = list.New() totalUsed int mu sync.Mutex - - // Size sets the maximum size of the cache before evicting unread data in MB - Size float64 = 1 << 7 ) // Key stores the ID of either a thread or board page @@ -82,7 +81,7 @@ func updateUsedSize(k Key, delta int) { } totalUsed += delta - for totalUsed > int(Size)*(1<<20) { + for totalUsed > int(config.Server.CacheSize)*(1<<20) { if last := ll.Back(); last != nil { removeEntry(last) } diff --git a/cache/main_test.go b/cache/main_test.go index be4ab579d..1ab18b6d9 100644 --- a/cache/main_test.go +++ b/cache/main_test.go @@ -1,17 +1,18 @@ package cache import ( - "github.com/bakape/meguca/test" "sync" "testing" - "time" + + "github.com/bakape/meguca/config" + "github.com/bakape/meguca/test" ) // Basic test for deadlocks func TestConcurrency(t *testing.T) { Clear() - Size = 1 + config.Server.CacheSize = 1 f := FrontEnd{ GetCounter: func(k Key) (uint64, error) { @@ -43,7 +44,7 @@ func TestConcurrency(t *testing.T) { func TestCacheEviction(t *testing.T) { Clear() - Size = 0.005 + config.Server.CacheSize = 0.005 f := FrontEnd{ GetCounter: func(k Key) (uint64, error) { return 1, nil diff --git a/config/config.go b/config/config.go index 761e19344..71a9bf178 100644 --- a/config/config.go +++ b/config/config.go @@ -12,23 +12,6 @@ import ( "github.com/bakape/meguca/util" ) -// ImagerModeType is the imager functionality setting for this meguca process -type ImagerModeType int - -const ( - // IntegratedImager is regular and imager functionality both handled by this process - IntegratedImager ImagerModeType = iota - - // NoImager is imager functionality not handled by this process - NoImager - - // ImagerOnly is only imager functionality handled by this process - ImagerOnly -) - -// ImagerMode is imager functionality setting for this meguca process -var ImagerMode ImagerModeType - var ( // Ensures no reads happen, while the configuration is reloading globalMu, boardMu sync.RWMutex diff --git a/config/server.go b/config/server.go new file mode 100644 index 000000000..4c913a0aa --- /dev/null +++ b/config/server.go @@ -0,0 +1,87 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" +) + +// Configurations of this specific instance passed from config file. +// Immutable after loading. +var Server ServerConfigs + +// ImagerModeType is the imager functionality setting for this meguca process +type ImagerModeType int + +const ( + // IntegratedImager is regular and imager functionality both handled by this process + IntegratedImager ImagerModeType = iota + + // NoImager is imager functionality not handled by this process + NoImager + + // ImagerOnly is only imager functionality handled by this process + ImagerOnly +) + +// Configurations of this specific instance passed from config file +type ServerConfigs struct { + Debug bool + ImagerMode ImagerModeType `json:"imager_mode"` + Database string + CacheSize float64 `json:"cache_size"` + Server struct { + ReverseProxied bool `json:"reverse_proxied"` + Address string + TLS struct { + Enabled bool + CertPath string `json:"cert_path"` + KeyPath string `json:"key_path"` + } + } + Test struct { + Database string + } +} + +// Load configs from JSON or defaults, if none present +func (c *ServerConfigs) Load() (err error) { + path := "config.json" + prefix := "" +try: + f, err := os.Open(filepath.Join(prefix, path)) + if err != nil { + if os.IsNotExist(err) { + _, err = os.Stat(filepath.Join(prefix, "go.mod")) + switch { + case err == nil: + // Reached the root dir + c.setDefaults() + return + case os.IsNotExist(err): + // Go up one dir + prefix = filepath.Join("..", prefix) + goto try + default: + return + } + + } + return + } + defer func() { + if f != nil { + f.Close() + } + }() + + return json.NewDecoder(f).Decode(c) +} + +func (c *ServerConfigs) setDefaults() { + c.Debug = true + c.Database = "postgres://meguca:meguca@localhost:5432/meguca?sslmode=disable" + c.CacheSize = 128 + c.Server.Address = ":8000" + c.Test.Database = "postgres://meguca:meguca@localhost:5432/meguca_test?sslmode=disable" +} diff --git a/db/init.go b/db/init.go index aa6cee398..39c675bb0 100644 --- a/db/init.go +++ b/db/init.go @@ -3,9 +3,11 @@ package db import ( "database/sql" "fmt" + "net/url" "os" "os/exec" "os/user" + "strings" "time" "github.com/Masterminds/squirrel" @@ -18,14 +20,9 @@ import ( _ "github.com/lib/pq" // Postgres driver ) -const ( - // DefaultConnArgs specifies the default PostgreSQL connection arguments - DefaultConnArgs = "user=meguca password=meguca dbname=meguca sslmode=disable binary_parameters=yes" -) - var ( - // ConnArgs specifies the PostgreSQL connection arguments - ConnArgs string + // ConnArgs specifies the PostgreSQL connection URL + connectionURL string // Stores the postgres database instance db *sql.DB @@ -39,7 +36,7 @@ var ( // Connects to PostgreSQL database and performs schema upgrades func LoadDB() error { - return loadDB("") + return loadDB(config.Server.Database, "") } // Create and load testing database. Call close() to clean up temporary @@ -47,6 +44,11 @@ func LoadDB() error { func LoadTestDB(suffix string) (close func() error, err error) { common.IsTest = true + connURL, err := url.Parse(config.Server.Test.Database) + if err != nil { + return + } + // If running as root (CI like Travis or something), authenticate as the // postgres user var u *user.User @@ -55,7 +57,7 @@ func LoadTestDB(suffix string) (close func() error, err error) { return } var sudo []string - user := "meguca" + user := connURL.User.Username() if u.Name == "root" { sudo = append(sudo, "sudo", "-u", "postgres") user = "postgres" @@ -69,12 +71,11 @@ func LoadTestDB(suffix string) (close func() error, err error) { return c.Run() } - suffix = "_" + suffix - name := "meguca_test" + suffix + dbName := fmt.Sprintf("%s_%s", strings.Trim(connURL.Path, "/"), suffix) err = run("psql", "-U", user, - "-c", "drop database if exists "+name) + "-c", "drop database if exists "+dbName) if err != nil { return } @@ -87,25 +88,26 @@ func LoadTestDB(suffix string) (close func() error, err error) { return os.Remove(fmt.Sprintf("db%s.db", suffix)) } - fmt.Println("creating test database:", name) + fmt.Println("creating test database:", dbName) err = run("createdb", "-O", "meguca", "-U", user, "-E", "UTF8", - name) + dbName) if err != nil { return } - ConnArgs = fmt.Sprintf( - "postgres://meguca:meguca@localhost:5432/%s?sslmode=disable&binary_parameters=yes", - name) - err = loadDB(suffix) + connURL.Path = "/" + dbName + err = loadDB(connURL.String(), suffix) return } -func loadDB(dbSuffix string) (err error) { - db, err = sql.Open("postgres", ConnArgs) +func loadDB(connURL, dbSuffix string) (err error) { + // Set, for creating extra connections using Listen() + connectionURL = connURL + + db, err = sql.Open("postgres", connURL) if err != nil { return } @@ -136,7 +138,7 @@ func loadDB(dbSuffix string) (err error) { tasks, func() error { tasks := []func() error{loadConfigs, loadBans, handleSpamScores} - if config.ImagerMode != config.ImagerOnly { + if config.Server.ImagerMode != config.ImagerOnly { tasks = append(tasks, openBoltDB(dbSuffix), loadBanners, loadLoadingAnimations, loadThreadPostCounts) } diff --git a/db/init_test.go b/db/init_test.go index 8ddf99a35..9f28830ad 100644 --- a/db/init_test.go +++ b/db/init_test.go @@ -3,9 +3,15 @@ package db import ( "os" "testing" + + "github.com/bakape/meguca/config" ) func TestMain(m *testing.M) { + err := config.Server.Load() + if err != nil { + panic(err) + } close, err := LoadTestDB("db") if err != nil { panic(err) diff --git a/db/upkeep.go b/db/upkeep.go index ab2bf7cc3..61cef0d5b 100644 --- a/db/upkeep.go +++ b/db/upkeep.go @@ -39,14 +39,14 @@ func runCleanupTasks() { } func runMinuteTasks() { - if config.ImagerMode != config.ImagerOnly { + if config.Server.ImagerMode != config.ImagerOnly { logError("open post cleanup", closeDanglingPosts()) expireRows("image_tokens", "bans", "failed_captchas") } } func runHalfTasks() { - if config.ImagerMode != config.ImagerOnly { + if config.Server.ImagerMode != config.ImagerOnly { logError("unrestrict pyu_limit", FreePyuLimit()) logError("expire spam scores", expireSpamScores()) logError("expire last solved captcha times", expireLastSolvedCaptchas()) @@ -54,7 +54,7 @@ func runHalfTasks() { } func runHourTasks() { - if config.ImagerMode != config.ImagerOnly { + if config.Server.ImagerMode != config.ImagerOnly { expireRows("sessions") expireBy("created < now() at time zone 'utc' + '-7 days'", "mod_log", "reports") @@ -65,7 +65,7 @@ func runHourTasks() { _, err := db.Exec(`vacuum`) logError("vaccum database", err) } - if config.ImagerMode != config.NoImager { + if config.Server.ImagerMode != config.NoImager { logError("image cleanup", deleteUnusedImages()) } } diff --git a/db/util.go b/db/util.go index 0a2f03988..21d69c63d 100644 --- a/db/util.go +++ b/db/util.go @@ -83,7 +83,7 @@ func ListenCancelable(event string, canceller <-chan struct{}, fn func(msg string) error, ) (err error) { l := pq.NewListener( - ConnArgs, + connectionURL, time.Second, time.Second*10, func(_ pq.ListenerEventType, _ error) {}, diff --git a/docs/config.json b/docs/config.json index b956fc560..a90aea881 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1,12 +1,18 @@ { - "ssl": false, - "reverseProxied": false, - "gzip": false, - "imagerMode": 0, - "cacheSize": 128, - "address": "127.0.0.1:8000", - "database": "user=meguca password=meguca dbname=meguca sslmode=disable", - "certPath": "", - "keyPath": "", - "reverseProxyIP": "" + "debug": true, + "imager_mode": 0, + "database": "postgres://meguca:meguca@localhost:5432/meguca?sslmode=disable", + "cache_size": 128, + "server": { + "address": "127.0.0.1:8000", + "reverse_proxied": false, + "tls": { + "enabled": false, + "cert_path": "", + "key_path": "" + } + }, + "test": { + "database": "postgres://meguca:meguca@localhost:5432/meguca_test?sslmode=disable" + } } diff --git a/imager/main_test.go b/imager/main_test.go index 29368f5b4..cedc05b0a 100644 --- a/imager/main_test.go +++ b/imager/main_test.go @@ -1,15 +1,20 @@ package imager import ( + "os" + "testing" + "github.com/bakape/meguca/config" "github.com/bakape/meguca/db" "github.com/bakape/meguca/imager/assets" . "github.com/bakape/meguca/test" - "os" - "testing" ) func TestMain(m *testing.M) { + err := config.Server.Load() + if err != nil { + panic(err) + } close, err := db.LoadTestDB("imager") if err != nil { panic(err) diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 028518003..000000000 --- a/main_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - "os" - "testing" -) - -// Simple test, to see if the server starts -func TestServerStart(t *testing.T) { - os.Args = []string{os.Args[0], "init"} - defer func() { - if err := recover(); err != nil { - t.Fatal(err) - } - }() - main() -} diff --git a/parser/main_test.go b/parser/main_test.go index 168de8d39..0b554bbdb 100644 --- a/parser/main_test.go +++ b/parser/main_test.go @@ -1,13 +1,18 @@ package parser import ( - "github.com/bakape/meguca/config" - "github.com/bakape/meguca/db" "os" "testing" + + "github.com/bakape/meguca/config" + "github.com/bakape/meguca/db" ) func TestMain(m *testing.M) { + err := config.Server.Load() + if err != nil { + panic(err) + } close, err := db.LoadTestDB("parser") if err != nil { panic(err) diff --git a/server/daemon.go b/server/daemon.go deleted file mode 100644 index 21054772f..000000000 --- a/server/daemon.go +++ /dev/null @@ -1,103 +0,0 @@ -// +build linux darwin - -// Daemonization logic for the server - -package server - -import ( - "os" - "syscall" - "time" - - "github.com/go-playground/log" - "github.com/sevlyar/go-daemon" - - "github.com/bakape/meguca/config" - "github.com/bakape/meguca/log" -) - -func init() { - handleDaemon = func(arg string) { - if config.ImagerMode == config.ImagerOnly { - daemonContext.PidFileName = ".imager.pid" - daemonContext.LogFileName = "imager_error.log" - } - - switch arg { - case "debug": - mlog.Init(mlog.Console) - mlog.ConsoleHandler.SetDisplayColor(true) - startServer() - case "stop": - killDaemon() - fallthrough - case "init": // For internal use only - os.Exit(0) - case "restart": - killDaemon() - fallthrough - case "start": - mlog.Init(mlog.Console) - mlog.ConsoleHandler.SetDisplayColor(false) - daemonize() - default: - printUsage() - } - } -} - -// Configuration variables for handling daemons -var daemonContext = &daemon.Context{ - PidFileName: ".pid", - LogFileName: "error.log", -} - -// Spawn a detached process to work in the background -func daemonize() { - child, err := daemonContext.Reborn() - if err != nil && err.Error() == "resource temporarily unavailable" { - log.Fatal("Error: Server already running") - } - if child != nil { - return - } - daemonized = true - defer daemonContext.Release() - log.Info("Server started ------------------------------------") - - go startServer() - if err := daemon.ServeSignals(); err != nil { - log.Fatalf("daemon runtime error: %s\n", err) - } - log.Info("server terminated") -} - -// Terminate the running meguca server daemon -func killDaemon() { - proc := findDaemon() - if proc != nil { - if err := proc.Signal(syscall.SIGTERM); err != nil { - log.Fatalf("error killing running daemon: %s\n", err) - } - - // Ascertain process has exited - for { - if err := proc.Signal(syscall.Signal(0)); err != nil { - if err.Error() == "os: process already finished" { - break - } - log.Fatalf("error ascertaining daemon exited: %s\n", err) - } - time.Sleep(100 * time.Millisecond) - } - } -} - -// Find the running daemonized meguca server process -func findDaemon() *os.Process { - proc, err := daemonContext.Search() - if err != nil && (!os.IsNotExist(err) && err.Error() != "EOF") { - log.Fatalf("error locating running daemon: %s\n", err) - } - return proc -} diff --git a/server/init.go b/server/init.go index 30faa9f01..97020655c 100644 --- a/server/init.go +++ b/server/init.go @@ -3,234 +3,87 @@ package server import ( - "bytes" - "encoding/json" - "errors" "flag" - "fmt" - "io/ioutil" "os" - "runtime" - "strings" + "strconv" - "github.com/ErikDubbelboer/gspt" ass "github.com/bakape/meguca/assets" "github.com/bakape/meguca/auth" - "github.com/bakape/meguca/cache" "github.com/bakape/meguca/config" "github.com/bakape/meguca/db" "github.com/bakape/meguca/imager/assets" "github.com/bakape/meguca/lang" + mlog "github.com/bakape/meguca/log" "github.com/bakape/meguca/templates" "github.com/bakape/meguca/util" "github.com/bakape/meguca/websockets/feeds" - "github.com/go-playground/log" ) -var ( - // debugMode denotes the server has been started with the `debug` parameter. - // This will cause not to spawn a daemon and stay attached to the launching - // shell. - daemonized bool - isWindows = runtime.GOOS == "windows" - - // Is assigned in ./daemon.go to control/spawn a daemon process. That file - // is never compiled on Windows and this function is never called. - handleDaemon func(string) - - // CLI mode arguments and descriptions - arguments = map[string]string{ - "start": "start the meguca server", - "stop": "stop a running daemonized meguca server", - "restart": "combination of stop + start", - "debug": "start server in debug mode without daemonizing (default)", - "help": "print this help text", +// Start parses command line arguments and initializes the server. +func Start() (err error) { + err = config.Server.Load() + if err != nil { + return } -) -// Configs, that can be optionally passed through a JSON configuration file. -// Flags override this. All fields are optional. -type serverConfigs struct { - SSL, ReverseProxied, Gzip *bool - ImagerMode *uint - CacheSize *float64 - Address, Database, CertPath, KeyPath, ReverseProxyIP *string -} - -func validateImagerMode(m *uint) { - if *m > 2 { - panic(fmt.Errorf("invalid imager mode: %d", *m)) - } -} - -// Iterate struct fields and assign defaults to missing fields -func setConfigDefaults(c *serverConfigs) { - if c.SSL == nil { - c.SSL = new(bool) - } - if c.ReverseProxied == nil { - c.ReverseProxied = new(bool) - } - if c.Gzip == nil { - c.Gzip = new(bool) - } - if c.ImagerMode == nil { - c.ImagerMode = new(uint) - } else { - validateImagerMode(c.ImagerMode) - } - if c.CacheSize == nil { - c.CacheSize = new(float64) - *c.CacheSize = 128 - } - if c.Address == nil { - c.Address = new(string) - *c.Address = "127.0.0.1:8000" - } - if c.Database == nil { - c.Database = new(string) - *c.Database = db.DefaultConnArgs + // Write PID file + f, err := os.Create(".pid") + if err != nil { + return } - if c.CertPath == nil { - c.CertPath = new(string) + _, err = f.Write(strconv.AppendInt(nil, int64(os.Getpid()), 10)) + if err != nil { + return } - if c.KeyPath == nil { - c.KeyPath = new(string) + err = f.Close() + if err != nil { + return } - if c.ReverseProxyIP == nil { - c.ReverseProxyIP = new(string) - } -} - -// Start parses command line arguments and initializes the server. -func Start() error { - // Read config file, if any - var conf serverConfigs - buf, err := ioutil.ReadFile("config.json") - switch { - case os.IsNotExist(err): - err = nil - case err == nil: - err = json.Unmarshal(buf, &conf) + if !config.Server.Debug { + var f *os.File + f, err = os.OpenFile("errors.log", + os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) if err != nil { - return err + return } - default: - return err + defer f.Close() + os.Stdout = f + os.Stderr = f } + mlog.Init(mlog.Console) + mlog.ConsoleHandler.SetDisplayColor(config.Server.Debug) - setConfigDefaults(&conf) - - // Define flags - flag.StringVar( - &address, - "a", - *conf.Address, - "address to listen on", - ) - flag.Float64Var(&cache.Size, "c", *conf.CacheSize, "cache size in MB") - flag.StringVar( - &db.ConnArgs, - "d", - *conf.Database, - "PostgreSQL connection arguments", - ) - flag.BoolVar( - &ssl, - "s", - *conf.SSL, - "serve and listen only through HTTPS. Requires -S and -K to be set.", - ) - flag.StringVar(&sslCert, "S", *conf.CertPath, "path to SSL certificate") - flag.StringVar(&sslKey, "K", *conf.KeyPath, "path to SSL key") - flag.BoolVar( - &auth.IsReverseProxied, - "r", - *conf.ReverseProxied, - "assume server is behind reverse proxy, when resolving client IPs", - ) - flag.StringVar( - &auth.ReverseProxyIP, - "R", - *conf.ReverseProxyIP, - "IP of the reverse proxy. Only needed, when reverse proxy is not on localhost.", - ) - flag.BoolVar(&enableGzip, "g", *conf.Gzip, "compress all traffic with gzip") - flag.UintVar(conf.ImagerMode, "i", *conf.ImagerMode, - `image processing and serving mode for this instance -0 handle image processing and serving and all other functionality (default) -1 handle all functionality except for image processing and serving -2 only handle image processing and serving`) - flag.Usage = printUsage - - // Parse command line arguments - flag.Parse() - if cache.Size < 0 { - return errors.New("cache size must be a positive number") + err = util.Parallel(db.LoadDB, assets.CreateDirs) + if err != nil { + return } - validateImagerMode(conf.ImagerMode) - config.ImagerMode = config.ImagerModeType(*conf.ImagerMode) - arg := flag.Arg(0) - if arg == "" { - arg = "debug" + err = util.Parallel(lang.Load) + if err != nil { + return } - // Can't daemonize in windows, so only args they have is "start" and "help" - if isWindows { - switch arg { - case "debug", "start": - startServer() - case "init": // For internal use only - os.Exit(0) - default: - printUsage() - } - } else { - // Censor DB connection string, if any - args := make([]string, 0, len(os.Args)) - for i := 0; i < len(os.Args); i++ { - arg := os.Args[i] - if strings.HasSuffix(arg, "-d") { // To match both -d and --d - args = append(args, arg, "****") - i++ // Jump to args after password - } else { - args = append(args, arg) - } - } - switch config.ImagerMode { - case config.ImagerOnly: - args = append(args, "(imager only)") - case config.NoImager: - args = append(args, "(no imager)") - } - gspt.SetProcTitle(strings.Join(args, " ")) - - handleDaemon(arg) + // Depend on configs + var tasks []func() error + if config.Server.ImagerMode != config.ImagerOnly { + tasks = append(tasks, templates.Compile, listenToThreadDeletion) + go ass.WatchVideoDir() + } + if config.Server.ImagerMode != config.NoImager { + tasks = append(tasks, auth.LoadCaptchaServices) + } + tasks = append(tasks, feeds.Init) + err = util.Parallel(tasks...) + if err != nil { + return } - return nil + return startWebServer() } // Constructs and prints the CLI help text func printUsage() { - os.Stderr.WriteString("Usage: meguca [OPTIONS]... [MODE]\n\nMODES:\n") - - toPrint := []string{"start"} - if !isWindows { - toPrint = append(toPrint, []string{"stop", "restart"}...) - } else { - arguments["debug"] = `alias of "start"` - } - toPrint = append(toPrint, []string{"debug", "help"}...) - - help := new(bytes.Buffer) - for _, arg := range toPrint { - fmt.Fprintf(help, " %s\n \t%s\n", arg, arguments[arg]) - } - - help.WriteString("\nOPTIONS:\n") - os.Stderr.Write(help.Bytes()) + os.Stderr.WriteString("Usage: meguca [OPTIONS]...\n") flag.PrintDefaults() os.Stderr.WriteString( "\nConsult the bundled README.md for more information\n", @@ -238,32 +91,3 @@ func printUsage() { os.Exit(1) } - -func startServer() { - load := func(fns ...func() error) { - if err := util.Parallel(fns...); err != nil { - log.Fatal(err) - } - } - - load(db.LoadDB, assets.CreateDirs) - load(lang.Load) - - // Depend on configs - var ( - tasks []func() error - ) - if config.ImagerMode != config.ImagerOnly { - tasks = append(tasks, templates.Compile, listenToThreadDeletion) - go ass.WatchVideoDir() - } - if config.ImagerMode != config.NoImager { - tasks = append(tasks, auth.LoadCaptchaServices) - } - tasks = append(tasks, feeds.Init) - load(tasks...) - - if err := startWebServer(); err != nil { - log.Fatal(err) - } -} diff --git a/server/router.go b/server/router.go index fa4e07b1b..9b0b05b64 100644 --- a/server/router.go +++ b/server/router.go @@ -2,11 +2,11 @@ package server import ( "bytes" - "compress/gzip" "fmt" "net/http" "runtime/debug" "strconv" + "strings" "github.com/bakape/meguca/auth" "github.com/bakape/meguca/config" @@ -20,23 +20,6 @@ import ( ) var ( - // Address is the listening address of the HTTP web server - address string - - // Defines if HTTPS should be used for listening for incoming connections. - // Requires sslCert and sslKey to be set. - ssl bool - - // Path to SSL certificate - sslCert string - - // Path to SSL key - sslKey string - - // Defines, if all traffic should be piped through a gzip compression - // -decompression handler - enableGzip bool - healthCheckMsg = []byte("God's in His heaven, all's right with the world") ) @@ -45,23 +28,25 @@ var webRoot = "www" func startWebServer() (err error) { r := createRouter() + c := config.Server.Server - var w bytes.Buffer + var w strings.Builder w.WriteString("listening on http") - if ssl { + if c.TLS.Enabled { w.WriteByte('s') } - prettyAddr := address - if len(address) != 0 && address[0] == ':' { + prettyAddr := c.Address + if len(c.Address) != 0 && c.Address[0] == ':' { prettyAddr = "127.0.0.1" + prettyAddr } fmt.Fprintf(&w, "://%s", prettyAddr) log.Info(w.String()) - if ssl { - err = http.ListenAndServeTLS(address, sslCert, sslKey, r) + if config.Server.Server.TLS.Enabled { + err = http.ListenAndServeTLS(c.Address, c.TLS.CertPath, c.TLS.KeyPath, + r) } else { - err = http.ListenAndServe(address, r) + err = http.ListenAndServe(c.Address, r) } if err != nil { return util.WrapError("error starting web server", err) @@ -92,7 +77,7 @@ func createRouter() http.Handler { api := r.NewGroup("/api") api.GET("/health-check", healthCheck) assets := r.NewGroup("/assets") - if config.ImagerMode != config.NoImager { + if config.Server.ImagerMode != config.NoImager { // All upload images api.POST("/upload", imager.NewImageUpload) api.POST("/upload-hash", imager.UploadImageHash) @@ -107,7 +92,7 @@ func createRouter() http.Handler { captcha.POST("/:board", authenticateCaptcha) captcha.GET("/confirmation", renderCaptchaConfirmation) } - if config.ImagerMode != config.ImagerOnly { + if config.Server.ImagerMode != config.ImagerOnly { // HTML r.GET("/", redirectToDefault) r.GET("/:board/", func(w http.ResponseWriter, r *http.Request) { @@ -205,12 +190,7 @@ func createRouter() http.Handler { assets.GET("/*path", serveAssets) } - h := http.Handler(r) - if enableGzip { - h = handlers.CompressHandlerLevel(h, gzip.DefaultCompression) - } - - return h + return handlers.CompressHandler(http.Handler(r)) } // Redirects to / requests to /all/ board diff --git a/server/router_test.go b/server/router_test.go index 4c6f8d155..2a8815c27 100644 --- a/server/router_test.go +++ b/server/router_test.go @@ -22,6 +22,10 @@ var router http.Handler var con = console.New(true) func TestMain(m *testing.M) { + err := config.Server.Load() + if err != nil { + panic(err) + } close, err := db.LoadTestDB("server") if err != nil { panic(err) @@ -79,20 +83,3 @@ func TestPanicHandler(t *testing.T) { assertCode(t, rec, 500) assertBody(t, rec, "500 foo\n") } - -func TestGzip(t *testing.T) { - enableGzip = true - defer func() { - enableGzip = false - }() - - r := createRouter() - rec, req := newPair("/json/config") - req.Header.Set("Accept-Encoding", "gzip") - - r.ServeHTTP(rec, req) - - if rec.Header().Get("Content-Encoding") != "gzip" { - t.Fatal("response not gzipped") - } -} diff --git a/websockets/feeds/feeds_test.go b/websockets/feeds/feeds_test.go index fec6793d7..004108d8b 100644 --- a/websockets/feeds/feeds_test.go +++ b/websockets/feeds/feeds_test.go @@ -1,14 +1,20 @@ package feeds import ( + "os" + "testing" + + "github.com/bakape/meguca/config" "github.com/bakape/meguca/db" "github.com/bakape/meguca/test" "github.com/bakape/meguca/test/test_db" - "os" - "testing" ) func TestMain(m *testing.M) { + err := config.Server.Load() + if err != nil { + panic(err) + } close, err := db.LoadTestDB("feeds") if err != nil { panic(err) diff --git a/websockets/websockets_test.go b/websockets/websockets_test.go index d4e160c9f..2089b1618 100644 --- a/websockets/websockets_test.go +++ b/websockets/websockets_test.go @@ -4,10 +4,6 @@ import ( "bytes" "errors" "fmt" - "github.com/bakape/meguca/auth" - "github.com/bakape/meguca/common" - "github.com/bakape/meguca/db" - . "github.com/bakape/meguca/test" "net/http" "net/http/httptest" "os" @@ -16,6 +12,11 @@ import ( "testing" "time" + "github.com/bakape/meguca/auth" + "github.com/bakape/meguca/common" + "github.com/bakape/meguca/config" + "github.com/bakape/meguca/db" + . "github.com/bakape/meguca/test" "github.com/go-playground/log" "github.com/go-playground/log/handlers/console" "github.com/gorilla/websocket" @@ -41,6 +42,10 @@ type mockWSServer struct { } func TestMain(m *testing.M) { + err := config.Server.Load() + if err != nil { + panic(err) + } close, err := db.LoadTestDB("websockets") if err != nil { panic(err) From 50448877e705d0598b8abd1ad8b99f25bba2d790 Mon Sep 17 00:00:00 2001 From: bakape Date: Sat, 4 May 2019 12:01:15 +0300 Subject: [PATCH 02/10] server: Remove dead code --- server/dbserver.db | Bin 0 -> 32768 bytes server/init.go | 12 ------------ 2 files changed, 12 deletions(-) create mode 100644 server/dbserver.db diff --git a/server/dbserver.db b/server/dbserver.db new file mode 100644 index 0000000000000000000000000000000000000000..a300ce427aac8171c6f54c9b48f6c31a99e8c117 GIT binary patch literal 32768 zcmeI(J&FP`6ae7(vo>0Y?Sgm!vGFo?f@oo(E^Ox+JfY39osIT(&Nz96;DY7Iet}Ff zGZ}dKx|c~(s!BVz=VR~OEc^Y(=lON|$em7Z?_-PpcKSKqZ?2aZrzJpu009C72oNAZ zfB*pk1nMKu$f3NN|3|(b`F|^ae;UqyeYnEeEjE+aasZd2oNAZfB*pk1PBlyK;V4^BHxd7fcKq=$O#Z2K!5-N0t5&UAV7cs zfxiS|-+wbzCjrW9dGx Date: Sat, 4 May 2019 12:05:40 +0300 Subject: [PATCH 03/10] tests: Fix Bolt db naming --- db/init.go | 2 +- server/dbserver.db | Bin 32768 -> 0 bytes 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 server/dbserver.db diff --git a/db/init.go b/db/init.go index 39c675bb0..4e0ed20bf 100644 --- a/db/init.go +++ b/db/init.go @@ -167,7 +167,7 @@ func loadDB(connURL, dbSuffix string) (err error) { func openBoltDB(dbSuffix string) func() error { return func() (err error) { boltDB, err = bolt.Open( - fmt.Sprintf("db%s.db", dbSuffix), + fmt.Sprintf("db_%s.db", dbSuffix), 0600, &bolt.Options{ Timeout: time.Second, diff --git a/server/dbserver.db b/server/dbserver.db deleted file mode 100644 index a300ce427aac8171c6f54c9b48f6c31a99e8c117..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32768 zcmeI(J&FP`6ae7(vo>0Y?Sgm!vGFo?f@oo(E^Ox+JfY39osIT(&Nz96;DY7Iet}Ff zGZ}dKx|c~(s!BVz=VR~OEc^Y(=lON|$em7Z?_-PpcKSKqZ?2aZrzJpu009C72oNAZ zfB*pk1nMKu$f3NN|3|(b`F|^ae;UqyeYnEeEjE+aasZd2oNAZfB*pk1PBlyK;V4^BHxd7fcKq=$O#Z2K!5-N0t5&UAV7cs zfxiS|-+wbzCjrW9dGx Date: Sat, 4 May 2019 12:11:14 +0300 Subject: [PATCH 04/10] db: Fix test Bolt DB removal --- db/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/init.go b/db/init.go index 4e0ed20bf..30b593227 100644 --- a/db/init.go +++ b/db/init.go @@ -85,7 +85,7 @@ func LoadTestDB(suffix string) (close func() error, err error) { if err != nil { return } - return os.Remove(fmt.Sprintf("db%s.db", suffix)) + return os.Remove(fmt.Sprintf("db_%s.db", suffix)) } fmt.Println("creating test database:", dbName) From 8631e0eb1f2c94ba089f641bd61889c93e60f310 Mon Sep 17 00:00:00 2001 From: bakape Date: Sat, 4 May 2019 14:04:52 +0300 Subject: [PATCH 05/10] server: Graceful restarts --- config/server.go | 5 --- db/init.go | 61 +++++++++++--------------- db/open_posts.go | 112 ++++++++++++++++++++++++++++++++++++++++++++++- db/reader.go | 19 -------- docs/config.json | 7 +-- go.mod | 15 +++---- go.sum | 56 +++++------------------- server/init.go | 2 +- server/router.go | 19 +++----- 9 files changed, 160 insertions(+), 136 deletions(-) diff --git a/config/server.go b/config/server.go index 4c913a0aa..9517f371d 100644 --- a/config/server.go +++ b/config/server.go @@ -33,11 +33,6 @@ type ServerConfigs struct { Server struct { ReverseProxied bool `json:"reverse_proxied"` Address string - TLS struct { - Enabled bool - CertPath string `json:"cert_path"` - KeyPath string `json:"key_path"` - } } Test struct { Database string diff --git a/db/init.go b/db/init.go index 30b593227..ed9d57200 100644 --- a/db/init.go +++ b/db/init.go @@ -8,7 +8,6 @@ import ( "os/exec" "os/user" "strings" - "time" "github.com/Masterminds/squirrel" "github.com/bakape/meguca/auth" @@ -81,11 +80,19 @@ func LoadTestDB(suffix string) (close func() error, err error) { } close = func() (err error) { + _, err = os.Stat("db.db") + if err != nil { + if os.IsNotExist(err) { + return nil + } + return + } + err = boltDB.Close() if err != nil { return } - return os.Remove(fmt.Sprintf("db_%s.db", suffix)) + return os.Remove("db.db") } fmt.Println("creating test database:", dbName) @@ -139,8 +146,8 @@ func loadDB(connURL, dbSuffix string) (err error) { func() error { tasks := []func() error{loadConfigs, loadBans, handleSpamScores} if config.Server.ImagerMode != config.ImagerOnly { - tasks = append(tasks, openBoltDB(dbSuffix), loadBanners, - loadLoadingAnimations, loadThreadPostCounts) + tasks = append(tasks, loadBanners, loadLoadingAnimations, + loadThreadPostCounts) } if err := util.Parallel(tasks...); err != nil { return err @@ -164,24 +171,6 @@ func loadDB(connURL, dbSuffix string) (err error) { return nil } -func openBoltDB(dbSuffix string) func() error { - return func() (err error) { - boltDB, err = bolt.Open( - fmt.Sprintf("db_%s.db", dbSuffix), - 0600, - &bolt.Options{ - Timeout: time.Second, - }) - if err != nil { - return - } - return boltDB.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte("open_bodies")) - return err - }) - } -} - // initDB initializes a database func initDB() (err error) { log.Info("initializing database") @@ -233,21 +222,23 @@ func CreateSystemAccount(tx *sql.Tx) (err error) { func ClearTables(tables ...string) error { for _, t := range tables { // Clear open post body bucket - switch t { - case "boards", "threads", "posts": - err := boltDB.Update(func(tx *bolt.Tx) error { - buc := tx.Bucket([]byte("open_bodies")) - c := buc.Cursor() - for k, _ := c.First(); k != nil; k, _ = c.Next() { - err := buc.Delete(k) - if err != nil { - return err + if boltDBisOpen() { + switch t { + case "boards", "threads", "posts": + err := boltDB.Update(func(tx *bolt.Tx) error { + buc := tx.Bucket([]byte("open_bodies")) + c := buc.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + err := buc.Delete(k) + if err != nil { + return err + } } + return nil + }) + if err != nil { + return err } - return nil - }) - if err != nil { - return err } } diff --git a/db/open_posts.go b/db/open_posts.go index bbd309e72..09f907a37 100644 --- a/db/open_posts.go +++ b/db/open_posts.go @@ -3,12 +3,81 @@ package db import ( "database/sql" "encoding/binary" + "sync" + "sync/atomic" + "time" + "github.com/bakape/meguca/common" "github.com/boltdb/bolt" ) +const ( + boltNotOpened = iota // Not opened yet in this server instance + boltDBOpen // Opened and ready fort operation + boltDBClosed // Closed for graceful restart +) + +var ( + // Current state of BoltDB database. + // Should only be accessed using atomic operations. + boltDBState uint32 + + // Ensures boltdb is opened only once + boltDBOnce sync.Once +) + +// Close DB and release resources +func Close() (err error) { + atomic.StoreUint32(&boltDBState, boltDBClosed) + return boltDB.Close() +} + +// Need to drop any incoming requests, when Db is closed during graceful restart +func boltDBisOpen() bool { + return atomic.LoadUint32(&boltDBState) == boltDBOpen +} + +// Open boltdb, only when needed. This helps preventing conflicts on swapping +// the database accessing process during graceful restarts. +// If boltdb has already been closed, return open=false. +func tryOpenBoltDB() (open bool, err error) { + boltDBOnce.Do(func() { + boltDB, err = bolt.Open( + "db.db", + 0600, + &bolt.Options{ + Timeout: time.Second, + }) + if err != nil { + return + } + + err = boltDB.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte("open_bodies")) + return err + }) + if err != nil { + return + } + + atomic.StoreUint32(&boltDBState, boltDBOpen) + return + }) + if err != nil { + return + } + + open = boltDBisOpen() + return +} + // SetOpenBody sets the open body of a post -func SetOpenBody(id uint64, body []byte) error { +func SetOpenBody(id uint64, body []byte) (err error) { + ok, err := tryOpenBoltDB() + if err != nil || !ok { + return + } + buf := encodeUint64(id) return boltDB.Batch(func(tx *bolt.Tx) error { return bodyBucket(tx).Put(buf[:], body) @@ -36,6 +105,11 @@ func encodeUint64Heap(i uint64) []byte { // GetOpenBody retrieves an open body of a post func GetOpenBody(id uint64) (body string, err error) { + ok, err := tryOpenBoltDB() + if err != nil || !ok { + return + } + buf := encodeUint64(id) err = boltDB.View(func(tx *bolt.Tx) error { body = string(bodyBucket(tx).Get(buf[:])) @@ -44,16 +118,50 @@ func GetOpenBody(id uint64) (body string, err error) { return } -func deleteOpenPostBody(id uint64) error { +func deleteOpenPostBody(id uint64) (err error) { + ok, err := tryOpenBoltDB() + if err != nil || !ok { + return + } + buf := encodeUint64(id) return boltDB.Batch(func(tx *bolt.Tx) error { return bodyBucket(tx).Delete(buf[:]) }) } +// Inject open post bodies from the embedded database into the posts +func injectOpenBodies(posts []*common.Post) (err error) { + if len(posts) == 0 { + return + } + + ok, err := tryOpenBoltDB() + if err != nil || !ok { + return + } + + tx, err := boltDB.Begin(false) + if err != nil { + return + } + + buc := tx.Bucket([]byte("open_bodies")) + for _, p := range posts { + p.Body = string(buc.Get(encodeUint64Heap(p.ID))) + } + + return tx.Rollback() +} + // Delete orphaned post bodies, that refer to posts already closed or deleted. // This can happen on server restarts, board deletion, etc. func cleanUpOpenPostBodies() (err error) { + ok, err := tryOpenBoltDB() + if err != nil || !ok { + return + } + // Read IDs of all post bodies var ids []uint64 err = boltDB.View(func(tx *bolt.Tx) error { diff --git a/db/reader.go b/db/reader.go index 07c87f25a..cadca55f9 100644 --- a/db/reader.go +++ b/db/reader.go @@ -405,25 +405,6 @@ func filterModerated(moderated *[]*common.Post, p *common.Post) { } } -// Inject open post bodies from the embedded database into the posts -func injectOpenBodies(posts []*common.Post) error { - if len(posts) == 0 { - return nil - } - - tx, err := boltDB.Begin(false) - if err != nil { - return err - } - - buc := tx.Bucket([]byte("open_bodies")) - for _, p := range posts { - p.Body = string(buc.Get(encodeUint64Heap(p.ID))) - } - - return tx.Rollback() -} - // Inject moderation information into affected post structs. // tx is optional. func injectModeration(posts []*common.Post, tx *sql.Tx) (err error) { diff --git a/docs/config.json b/docs/config.json index a90aea881..352ef36ab 100644 --- a/docs/config.json +++ b/docs/config.json @@ -5,12 +5,7 @@ "cache_size": 128, "server": { "address": "127.0.0.1:8000", - "reverse_proxied": false, - "tls": { - "enabled": false, - "cert_path": "", - "key_path": "" - } + "reverse_proxied": false }, "test": { "database": "postgres://meguca:meguca@localhost:5432/meguca_test?sslmode=disable" diff --git a/go.mod b/go.mod index 0329101fc..1546492d9 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/ErikDubbelboer/gspt v0.0.0-20190125194910-e68493906b83 github.com/Masterminds/squirrel v1.1.0 github.com/PuerkitoBio/goquery v1.5.0 // indirect - github.com/Sirupsen/logrus v1.4.1 // indirect + github.com/Sirupsen/logrus v0.0.0-00010101000000-000000000000 // indirect github.com/abh/geoip v0.0.0-20160510155516-07cea4480daa github.com/aquilax/tripcode v1.0.0 github.com/badoux/goscraper v0.0.0-20181207103713-9b4686c4b62c @@ -16,27 +16,22 @@ require ( github.com/boltdb/bolt v1.3.1 github.com/chai2010/webp v1.1.0 github.com/dimfeld/httptreemux v5.0.1+incompatible - github.com/dsnet/compress v0.0.1 // indirect + github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect + github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434 + github.com/facebookgo/httpdown v0.0.0-20180706035922-5979d39b15c2 // indirect + github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 // indirect github.com/fsnotify/fsnotify v1.4.7 github.com/go-playground/ansi v2.1.0+incompatible // indirect github.com/go-playground/errors v3.3.0+incompatible // indirect github.com/go-playground/log v6.3.0+incompatible - github.com/golang/snappy v0.0.1 // indirect - github.com/gorilla/handlers v1.4.0 github.com/gorilla/websocket v1.4.0 - github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/lib/pq v1.0.1-0.20190326042056-d6156e141ac6 github.com/otium/ytdl v0.5.1 - github.com/pierrec/lz4 v2.0.5+incompatible // indirect github.com/rakyll/statik v0.1.6 github.com/sevlyar/go-daemon v0.1.4 github.com/ulikunitz/xz v0.5.6 - github.com/valyala/fasthttp v1.2.0 // indirect github.com/valyala/quicktemplate v1.0.2 golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 - golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 // indirect - golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67 // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/mholt/archiver.v2 v2.1.0 ) diff --git a/go.sum b/go.sum index 80f77583a..a265ab596 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/ErikDubbelboer/gspt v0.0.0-20190125194910-e68493906b83 h1:AUxmql2hgJxZTbfYZlRlXvCakS55HMd4BDsu6Hsqomo= github.com/ErikDubbelboer/gspt v0.0.0-20190125194910-e68493906b83/go.mod h1:01yGrV5aDnYieX2hSBoKGMaRiqHvnLoUy4ngWGZ/owg= github.com/Masterminds/squirrel v0.0.0-20190107164353-fa735ea14f09/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA= github.com/Masterminds/squirrel v1.1.0 h1:baP1qLdoQCeTw3ifCdOq2dkYc6vGcmRdaociKLbEJXs= @@ -15,30 +14,10 @@ github.com/badoux/goscraper v0.0.0-20181207103713-9b4686c4b62c h1:eacIMsthlVgjQ8 github.com/badoux/goscraper v0.0.0-20181207103713-9b4686c4b62c/go.mod h1:5iU5AiceCVP7wmrAIn/9YhJzvmErX/GihV/T2o5QUpM= github.com/bakape/boorufetch v1.0.1 h1:lTgxhYNQazFB/0SElkkcH3xX6URlHxvTiPgtbt4hIQ0= github.com/bakape/boorufetch v1.0.1/go.mod h1:5w+CugEba6no6uHTyaDMjZeYq5Xir8C6A9xtZr/wsbs= -github.com/bakape/captchouli v1.0.1 h1:UfG351N0qSF9/e3X3QaKTXmumn7q8edN1O5OmlOAzrA= -github.com/bakape/captchouli v1.0.1/go.mod h1:5lk1FfadAtzGtNKktjNHPvpNEAwKuadBnBilY4F8lkw= -github.com/bakape/captchouli v1.1.0 h1:piIp8oww1R3bIQQCB9SSml3lhWv4bPZudLv6i1yHuVg= -github.com/bakape/captchouli v1.1.0/go.mod h1:5lk1FfadAtzGtNKktjNHPvpNEAwKuadBnBilY4F8lkw= -github.com/bakape/captchouli v1.1.1 h1:l/l+x4AY2IQ2CBTKLvzgTQZWpT/TlkgUmt1WrxJbpOA= -github.com/bakape/captchouli v1.1.1/go.mod h1:5lk1FfadAtzGtNKktjNHPvpNEAwKuadBnBilY4F8lkw= -github.com/bakape/captchouli v1.1.2 h1:hCPfvgidQefSAO8IilF9RzB4TuZ8OrsokCgF2YaVMY0= -github.com/bakape/captchouli v1.1.2/go.mod h1:5lk1FfadAtzGtNKktjNHPvpNEAwKuadBnBilY4F8lkw= -github.com/bakape/captchouli v1.1.3 h1:MIE27PvWg+TD/Cp5/4k78WRuGWnT72/mys6SQuDvjm8= -github.com/bakape/captchouli v1.1.3/go.mod h1:h6OVnTaKH/JeKTZimRmC008ddem3Yj/ZC9J+RK1R0vc= -github.com/bakape/captchouli v1.1.4 h1:GqoDUI0QQ6xGluDhpaEuE2OymBV571SKa6clueyQ2t4= -github.com/bakape/captchouli v1.1.4/go.mod h1:h6OVnTaKH/JeKTZimRmC008ddem3Yj/ZC9J+RK1R0vc= github.com/bakape/captchouli v1.1.5 h1:fekFc6jmkMeg2IHn00arrAU354+eMA9uAgC0CizWems= github.com/bakape/captchouli v1.1.5/go.mod h1:h6OVnTaKH/JeKTZimRmC008ddem3Yj/ZC9J+RK1R0vc= github.com/bakape/mnemonics v0.0.0-20170918165711-056d8d325992 h1:n89d5g4t1+7ROrwElvy5W/knDo3bUKWOZ77acIdHA04= github.com/bakape/mnemonics v0.0.0-20170918165711-056d8d325992/go.mod h1:+x6GUyBL1gkeUoBPZAEzF9u04TGIFEljjrqMpGT0KVE= -github.com/bakape/thumbnailer v0.0.0-20190313202626-e737add357f7 h1:DHX8jbvH0n4Hvo1vjfZ3qb9syGtlC5/Gkmcb9l/0C8A= -github.com/bakape/thumbnailer v0.0.0-20190313202626-e737add357f7/go.mod h1:s7Sm20txJX+qAc11XFnLPqDPXnYShoscD3++dZKTxIc= -github.com/bakape/thumbnailer v0.0.0-20190419231732-9bd6fcd43d22 h1:42HP+b3orjoOFq7rsg2iJNRXYmy40x/Zg5E/FX3pLHU= -github.com/bakape/thumbnailer v0.0.0-20190419231732-9bd6fcd43d22/go.mod h1:s7Sm20txJX+qAc11XFnLPqDPXnYShoscD3++dZKTxIc= -github.com/bakape/thumbnailer v0.0.0-20190421122149-f9b845baba30 h1:CtTJ9QIscPOLV2tOJAct3j5SWBg6wvFqv81VgL0QBfI= -github.com/bakape/thumbnailer v0.0.0-20190421122149-f9b845baba30/go.mod h1:s7Sm20txJX+qAc11XFnLPqDPXnYShoscD3++dZKTxIc= -github.com/bakape/thumbnailer v0.0.0-20190424201625-d663001341e2 h1:usjszayEgphPgotIQuzxHmm0b0Wqxwg2+ItlOkDhylU= -github.com/bakape/thumbnailer v0.0.0-20190424201625-d663001341e2/go.mod h1:s7Sm20txJX+qAc11XFnLPqDPXnYShoscD3++dZKTxIc= github.com/bakape/thumbnailer v0.0.0-20190501133407-c89db19cab54 h1:gJiZ/+RG8gXyZ+OCaXAr6c3isJyiWOrsoPpN/HPiyDc= github.com/bakape/thumbnailer v0.0.0-20190501133407-c89db19cab54/go.mod h1:s7Sm20txJX+qAc11XFnLPqDPXnYShoscD3++dZKTxIc= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= @@ -49,9 +28,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434 h1:mOp33BLbcbJ8fvTAmZacbBiOASfxN+MLcLxymZCIrGE= +github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434/go.mod h1:KigFdumBXUPSwzLDbeuzyt0elrL7+CP7TKuhrhT4bcU= +github.com/facebookgo/httpdown v0.0.0-20180706035922-5979d39b15c2 h1:nXeeRHmgNgjLxi+7dY9l9aDvSS1uwVlNLqUWIY4Ath0= +github.com/facebookgo/httpdown v0.0.0-20180706035922-5979d39b15c2/go.mod h1:TUV/fX3XrTtBQb5+ttSUJzcFgLNpILONFTKmBuk5RSw= +github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 h1:0YtRCqIZs2+Tz49QuH6cJVw/IFqzo39gEqZ0iYLxD2M= +github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4/go.mod h1:vsJz7uE339KUCpBXx3JAJzSRH7Uk4iGGyJzR529qDIA= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-playground/ansi v2.1.0+incompatible h1:f9ldskdk1seTFmYjbmPaYB+WYsDKWc4UXcGb+e9JrN8= @@ -60,22 +44,13 @@ github.com/go-playground/errors v3.3.0+incompatible h1:w7qP6bdFXNmI86aV8VEfhXrGx github.com/go-playground/errors v3.3.0+incompatible/go.mod h1:n+RcthKmtLxDczVHKkhqiUSOGtTjvRl+HB4Gga0vWSI= github.com/go-playground/log v6.3.0+incompatible h1:CVT3y82/iLS65WJ4xfF8+SI6dxRdMiXpX+9surI/R2U= github.com/go-playground/log v6.3.0+incompatible/go.mod h1:3M1OvdKL8KYwOjJa3XM42iqzpvde2LHla8Ys0oz7Ma0= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA= -github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= -github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -90,12 +65,9 @@ github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWk github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/otium/ytdl v0.5.1 h1:aeqoca6n38vjWIN1qeRB5WTnY7uiQkWpyKRflC5eyIQ= github.com/otium/ytdl v0.5.1/go.mod h1:592mOGF/cH7ud/s969KgRp28KKiMMIXkZcK2498KaPw= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rakyll/statik v0.1.6 h1:uICcfUXpgqtw2VopbIncslhAmE5hwc4g20TEyEENBNs= github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs= -github.com/sevlyar/go-daemon v0.1.4 h1:Ayxp/9SNHwPBjV+kKbnHl2ch6rhxTu08jfkGkoxgULQ= github.com/sevlyar/go-daemon v0.1.4/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= github.com/sirupsen/logrus v1.4.0 h1:yKenngtzGh+cUSSh6GWbxW2abRqhYUSR/t/6+2QqNvE= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -108,32 +80,24 @@ github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4A github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v0.0.0-20180905170723-c6fd90e432cc/go.mod h1:+g/po7GqyG5E+1CNgquiIxJnsXEi5vwFn5weFujbO78= -github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/quicktemplate v1.0.0/go.mod h1:uX8yrAzr955BRr+rAerWX4V1NHb8bFfsePffuN5lFso= github.com/valyala/quicktemplate v1.0.2 h1:ZeVRKan1W/25u5f9ilDo5HWtdxD+QxR10WrZwxqLM54= github.com/valyala/quicktemplate v1.0.2/go.mod h1:KZAB+RlYlfNtBUGQMzIrnE8uuNgD2SbUn5CpZyod0sk= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190110200230-915654e7eabc h1:Yx9JGxI1SBhVLFjpAkWMaO1TF+xyqtHLjZpvQboJGiM= golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67 h1:1Fzlr8kkDLQwqMP8GxrhptBLqZG/EDpiATneiZHY998= -golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= -gopkg.in/mholt/archiver.v2 v2.1.0 h1:XdJA2b5a3ZucLaJkkZF25NAzgMFPiMWQTp1NfTTUcSI= gopkg.in/mholt/archiver.v2 v2.1.0/go.mod h1:WxDxKgbnqCjBfcDqukl78rDwZ7SkxagiW/lWW/mMnYY= diff --git a/server/init.go b/server/init.go index 711b0a4ee..a15e917ac 100644 --- a/server/init.go +++ b/server/init.go @@ -57,7 +57,7 @@ func Start() (err error) { if err != nil { return } - err = util.Parallel(lang.Load) + err = lang.Load() if err != nil { return } diff --git a/server/router.go b/server/router.go index 9b0b05b64..c66e4f268 100644 --- a/server/router.go +++ b/server/router.go @@ -15,8 +15,8 @@ import ( "github.com/bakape/meguca/util" "github.com/bakape/meguca/websockets" "github.com/dimfeld/httptreemux" + "github.com/facebookgo/grace/gracehttp" "github.com/go-playground/log" - "github.com/gorilla/handlers" ) var ( @@ -27,14 +27,10 @@ var ( var webRoot = "www" func startWebServer() (err error) { - r := createRouter() c := config.Server.Server var w strings.Builder w.WriteString("listening on http") - if c.TLS.Enabled { - w.WriteByte('s') - } prettyAddr := c.Address if len(c.Address) != 0 && c.Address[0] == ':' { prettyAddr = "127.0.0.1" + prettyAddr @@ -42,12 +38,11 @@ func startWebServer() (err error) { fmt.Fprintf(&w, "://%s", prettyAddr) log.Info(w.String()) - if config.Server.Server.TLS.Enabled { - err = http.ListenAndServeTLS(c.Address, c.TLS.CertPath, c.TLS.KeyPath, - r) - } else { - err = http.ListenAndServe(c.Address, r) - } + gracehttp.PreStartProcess(db.Close) + err = gracehttp.Serve(&http.Server{ + Addr: c.Address, + Handler: createRouter(), + }) if err != nil { return util.WrapError("error starting web server", err) } @@ -190,7 +185,7 @@ func createRouter() http.Handler { assets.GET("/*path", serveAssets) } - return handlers.CompressHandler(http.Handler(r)) + return r } // Redirects to / requests to /all/ board From 3ab428d2eac41ab3534197d5ba511c51a71aeda0 Mon Sep 17 00:00:00 2001 From: bakape Date: Sat, 4 May 2019 14:26:58 +0300 Subject: [PATCH 06/10] docs: Add sample sytemd unit file --- docs/meguca.service | 19 +++++++++++++++++++ docs/{nginx.conf.example => nginx.conf} | 0 2 files changed, 19 insertions(+) create mode 100644 docs/meguca.service rename docs/{nginx.conf.example => nginx.conf} (100%) diff --git a/docs/meguca.service b/docs/meguca.service new file mode 100644 index 000000000..c910695fd --- /dev/null +++ b/docs/meguca.service @@ -0,0 +1,19 @@ +[Unit] +Description=meguca imageboard server + +[Service] +Type=simple +Restart=always +RestartSec=5s +User=meguca +Group=meguca +WorkingDirectory=/home/meguca/server +ExecStart=/home/meguca/server/meguca +PIDFile=/home/meguca/server/.pid +ExecReload=/bin/kill -USR2 $MAINPID +ExecStop=/bin/kill -s TERM $MAINPID + +[Install] +WantedBy=multi-user.target +Requires=postgresql.service +After=postgresql.service diff --git a/docs/nginx.conf.example b/docs/nginx.conf similarity index 100% rename from docs/nginx.conf.example rename to docs/nginx.conf From 03e5d2c15a48f937408a959f8fbb492d8fd882c4 Mon Sep 17 00:00:00 2001 From: bakape Date: Sat, 4 May 2019 15:37:59 +0300 Subject: [PATCH 07/10] docs: Document changes in CLI and configs --- README.md | 29 +++++++++++++++++----- docs/config.json | 2 +- docs/config.jsonc | 59 +++++++++++++++++++++++++++++++++++++++++++++ docs/meguca.service | 4 ++- 4 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 docs/config.jsonc diff --git a/README.md b/README.md index 34fce93d9..2822196e7 100644 --- a/README.md +++ b/README.md @@ -9,17 +9,18 @@ Platforms: Linux, OSX, Win64 License: GNU AGPL ## Runtime dependencies + * [PostgresSQL](https://www.postgresql.org/download/) >= 10.0 ## Building from source - ### Native installation. For installing meguca directly onto a server follow the steps bellow. A reference list of commands can be found in `./docs/installation.md` #### Build dependencies + * [Go](https://golang.org/doc/install) >=1.11 (for building server) * [Node.js](https://nodejs.org) >=5.0 (for building client) * C11 compiler @@ -39,9 +40,11 @@ compiled with: * git #### Linux and OSX + * Run `make` #### Windows + * Install [MSYS2](https://sourceforge.net/projects/msys2/) * Open MSYS2 shell * Install dependencies listed above with the `mingw-w64-x86_64-` prefix with @@ -59,20 +62,34 @@ docker run -t -p 8000:8000 meguca ``` ## Setup -* See `./meguca help` for server operation + +### Deployment + +meguca can be started in debug mode simply with `./meguca`. +Configurations are split between meguca instance configurations +and server instance configurations, which are required to start +the server and connect to the database. +The meguca instance configurations are stored in the database, but +server instance configurations are optionally loaded from a `config.json` +file on server start. +A sample configuration file can be found under `docs/config.json`. +Documentation for this file is available under `docs/config.jsonc`. + +It is recommended to serve meguca behind a reverse proxy like NGINX or Apache +with properly configured TLS settings. A sample NGINX configuration file can be +found in `docs/`. + +### Initial instance configuration + * Login into the "admin" account via the infinity symbol in the top banner with the password "password" * Change the default password * Create a board from the administration panel * Configure server from the administration panel -* To enable country flags on posts download and place `GeoLite2-Country.mmdb` -into the root directory -* To avoid having to always type in CLI flags on server start you can specify them in `config.json` file in the project root. A sample file with all the default settings can be found in `docs/`. ## Development * See `./docs` for more documentation -* `./meguca` or `./meguca debug` run the server in development mode * `make server` and `make client` build the server and client separately * `make watch` watches the file system for changes and incrementally rebuilds the client diff --git a/docs/config.json b/docs/config.json index 352ef36ab..d4f31c98a 100644 --- a/docs/config.json +++ b/docs/config.json @@ -2,7 +2,7 @@ "debug": true, "imager_mode": 0, "database": "postgres://meguca:meguca@localhost:5432/meguca?sslmode=disable", - "cache_size": 128, + "cache_size": 128.0, "server": { "address": "127.0.0.1:8000", "reverse_proxied": false diff --git a/docs/config.jsonc b/docs/config.jsonc new file mode 100644 index 000000000..8ac7f8a75 --- /dev/null +++ b/docs/config.jsonc @@ -0,0 +1,59 @@ +{ + /* + Should the server be running in debug mode. Debug mode has more verbose + logging and prints all logs to stdout. If disabled, error logs will be + printed to the `errors.log` file. + */ + "debug": true, + /* + Sets what functionality this instance of the application server will + handle. The functionality is divided around handling image-related + processing. + + 0: handle image processing and serving and all other functionality + 1: handle all functionality except for image processing and serving + 2: only handle image processing and serving + */ + "imager_mode": 0, + /* + Database URL to connect to on server start + */ + "database": "postgres://meguca:meguca@localhost:5432/meguca?sslmode=disable", + /* + Size limit of internal cache in MB. Once limit is exceeded, the least + recently used records from the cache will be evicted. + */ + "cache_size": 128.0, + "server": { + /* + Address to listen on for incoming connections. + */ + "address": "127.0.0.1:8000", + /* + The server can only be accessed by clients through a reverse proxy + like NGINX and thus can safely honour "X-Forwarded-For" headers + for client IP resolution. + */ + "reverse_proxied": false + }, + "test": { + /* + Database URL to use during tests. + + This URL only serves as the base. The actual databases are created + and dropped during testing automatically with prefixes for each + database according to the submodule using this test database for + running unit tests. This division allows database-related for each + submodule to be run concurrently, overall reducing the runtime of + unit tests. + + To allow database creation during tests the role used must have the + necessary PostgreSQL permissions. These can be granted by running + + ALTER USER WITH CREATEDB; + + as the administrator postgres user in the psql shell. + */ + "database": "postgres://meguca:meguca@localhost:5432/meguca_test?sslmode=disable" + } +} diff --git a/docs/meguca.service b/docs/meguca.service index c910695fd..9316cf688 100644 --- a/docs/meguca.service +++ b/docs/meguca.service @@ -4,7 +4,9 @@ Description=meguca imageboard server [Service] Type=simple Restart=always -RestartSec=5s +RestartSec=5 +StartLimitIntervalSec=6 +StartLimitBurst=1 User=meguca Group=meguca WorkingDirectory=/home/meguca/server From 3136d28296444563ded9dd1592ba81b215f9d347 Mon Sep 17 00:00:00 2001 From: bakape Date: Sun, 5 May 2019 00:16:27 +0300 Subject: [PATCH 08/10] db: Force binary_parameters DB query string --- auth/captcha.go | 5 ++--- db/init.go | 8 ++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/auth/captcha.go b/auth/captcha.go index 8d6f00009..8624e3646 100644 --- a/auth/captcha.go +++ b/auth/captcha.go @@ -6,11 +6,10 @@ import ( "net/http/httptest" "sync" - "github.com/bakape/meguca/common" - "github.com/bakape/meguca/config" - "github.com/bakape/captchouli" captchouli_common "github.com/bakape/captchouli/common" + "github.com/bakape/meguca/common" + "github.com/bakape/meguca/config" ) var ( diff --git a/db/init.go b/db/init.go index ed9d57200..08a8a95e8 100644 --- a/db/init.go +++ b/db/init.go @@ -111,6 +111,14 @@ func LoadTestDB(suffix string) (close func() error, err error) { } func loadDB(connURL, dbSuffix string) (err error) { + // Enable binary parameters for more efficient encoding of []byte + u, err := url.Parse(connURL) + if err != nil { + return + } + u.Query().Set("binary_parameters", "yes") + connURL = u.String() + // Set, for creating extra connections using Listen() connectionURL = connURL From 3d21e99584707fa0a8171191f7aafe40b7c7d572 Mon Sep 17 00:00:00 2001 From: bakape Date: Sun, 5 May 2019 00:18:47 +0300 Subject: [PATCH 09/10] cache: Fix race condition in test --- cache/getters_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache/getters_test.go b/cache/getters_test.go index 9ee7e3955..1be1cdbbe 100644 --- a/cache/getters_test.go +++ b/cache/getters_test.go @@ -95,7 +95,7 @@ func TestGetHTML(t *testing.T) { func TestCounterExpiry(t *testing.T) { Clear() - config.Server.CacheSize = 1 << 7 + config.Server.CacheSize = 128 * (1 << 20) var counterChecks, fetches int f := FrontEnd{ From 3360821398eeefded5f3910dfa0db9b50ed372c9 Mon Sep 17 00:00:00 2001 From: bakape Date: Sun, 5 May 2019 00:36:50 +0300 Subject: [PATCH 10/10] docs: Fix systemd unit file --- docs/meguca.service | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/meguca.service b/docs/meguca.service index 9316cf688..c8857118b 100644 --- a/docs/meguca.service +++ b/docs/meguca.service @@ -1,11 +1,13 @@ [Unit] Description=meguca imageboard server +Requires=postgresql.service +After=postgresql.service [Service] Type=simple Restart=always RestartSec=5 -StartLimitIntervalSec=6 +StartLimitInterval=6s StartLimitBurst=1 User=meguca Group=meguca @@ -17,5 +19,3 @@ ExecStop=/bin/kill -s TERM $MAINPID [Install] WantedBy=multi-user.target -Requires=postgresql.service -After=postgresql.service