From da72fdf6ca4a50c2ebf7058fd46c39a57dae1b15 Mon Sep 17 00:00:00 2001 From: Wayback Archiver <66856220+waybackarchiver@users.noreply.github.com> Date: Sun, 30 Apr 2023 14:40:50 +0000 Subject: [PATCH] Add support for XMPP (#380) --- README.md | 9 +- cmd/wayback/serve.go | 15 +- config/config_test.go | 69 ++++++++ config/options.go | 39 +++++ config/parser.go | 8 + docs/index.md | 3 +- docs/index.zh.md | 3 +- docs/integrations/xmpp.md | 25 ++- docs/integrations/xmpp.zh.md | 26 ++- go.mod | 8 +- go.sum | 16 +- metrics/metrics.go | 1 + publish/publish.go | 1 + service/service.go | 4 + service/xmpp/doc.go | 8 + service/xmpp/xmpp.go | 326 +++++++++++++++++++++++++++++++++++ service/xmpp/xmpp_test.go | 253 +++++++++++++++++++++++++++ template/render/xmpp.go | 57 ++++++ template/render/xmpp_test.go | 28 +++ wayback.1 | 12 ++ wayback.conf | 4 + 21 files changed, 905 insertions(+), 10 deletions(-) create mode 100644 service/xmpp/doc.go create mode 100644 service/xmpp/xmpp.go create mode 100644 service/xmpp/xmpp_test.go create mode 100644 template/render/xmpp.go create mode 100644 template/render/xmpp_test.go diff --git a/README.md b/README.md index 418af83f..ac12cc59 100644 --- a/README.md +++ b/README.md @@ -21,12 +21,13 @@ Supported Golang version: See [.github/workflows/testing.yml](./.github/workflow ## Features - Free and open-source +- Expose prometheus metrics - Cross-platform compatibility - Batch wayback URLs for faster archiving - Built-in CLI (`wayback`) for convenient use - Serve as a Tor Hidden Service or local web entry for added privacy and accessibility - Easier wayback to Internet Archive, archive.today, IPFS and Telegraph integration -- Interactive with IRC, Matrix, Telegram bot, Discord bot, Mastodon, and Twitter as a daemon service for convenient use +- Interactive with IRC, Matrix, Telegram bot, Discord bot, Mastodon, Twitter, and XMPP as a daemon service for convenient use - Supports publishing wayback results to Telegram channel, Mastodon, and GitHub Issues for sharing - Supports storing archived files to disk for offline use - Download streaming media (requires [FFmpeg](https://ffmpeg.org/)) for convenient media archiving. @@ -114,7 +115,7 @@ Examples: Flags: --chatid string Telegram channel id -c, --config string Configuration file path, defaults: ./wayback.conf, ~/wayback.conf, /etc/wayback.conf - -d, --daemon strings Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc + -d, --daemon strings Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc, xmpp --debug Enable debug mode (default mode is false) -h, --help help for wayback --ia Wayback webpages to Internet Archive @@ -283,6 +284,10 @@ You can also specify configuration options either via command flags or via envir | - | `WAYBACK_SLACK_HELPTEXT` | - | The help text for Slack slash command | | - | `WAYBACK_NOSTR_RELAY_URL` | `wss://nostr.developer.li` | Nostr relay server url, multiple separated by comma | | - | `WAYBACK_NOSTR_PRIVATE_KEY` | - | The private key of a Nostr account | +| - | `WAYBACK_XMPP_JID` | - | The JID of a XMPP account | +| - | `WAYBACK_XMPP_PASSWORD` | - | The password of a XMPP account | +| - | `WAYBACK_XMPP_NOTLS` | - | Connect to XMPP server without TLS | +| - | `WAYBACK_XMPP_HELPTEXT` | - | The help text for XMPP command | | `--tor` | `WAYBACK_USE_TOR` | `false` | Snapshot webpage via Tor anonymity network | | `--tor-key` | `WAYBACK_ONION_PRIVKEY` | - | The private key for Tor Hidden Service | | - | `WAYBACK_ONION_LOCAL_PORT` | `8964` | Local port for Tor Hidden Service, also support for a **reverse proxy**. This is ignored if `WAYBACK_LISTEN_ADDR` is set. | diff --git a/cmd/wayback/serve.go b/cmd/wayback/serve.go index 32ce8a0f..b4651510 100644 --- a/cmd/wayback/serve.go +++ b/cmd/wayback/serve.go @@ -7,6 +7,7 @@ import ( "context" "os" "os/signal" + "strings" "syscall" "github.com/spf13/cobra" @@ -23,6 +24,7 @@ import ( "github.com/wabarc/wayback/service/slack" "github.com/wabarc/wayback/service/telegram" "github.com/wabarc/wayback/service/twitter" + "github.com/wabarc/wayback/service/xmpp" "github.com/wabarc/wayback/storage" "github.com/wabarc/wayback/systemd" @@ -91,7 +93,7 @@ func (srv *services) run(ctx context.Context, opts service.Options) *services { srv.targets = make([]target, 0, size) for _, s := range daemon { s := s - switch s { + switch strings.ToLower(s) { case "irc": irc := relaychat.New(ctx, opts) go func() { @@ -180,6 +182,17 @@ func (srv *services) run(ctx context.Context, opts service.Options) *services { call: func() { h.Shutdown() }, // nolint:errcheck name: s, }) + case "jabber", "xmpp": + h := xmpp.New(ctx, opts) + go func() { + if err := h.Serve(); err != xmpp.ErrServiceClosed { + logger.Error("start %s service failed: %v", s, err) + } + }() + srv.targets = append(srv.targets, target{ + call: func() { h.Shutdown() }, // nolint:errcheck + name: s, + }) default: logger.Fatal("unrecognize %s in `--daemon`", s) } diff --git a/config/config_test.go b/config/config_test.go index 227cffda..3768d88f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1295,6 +1295,75 @@ func TestPublishToSlackChannel(t *testing.T) { } } +func TestXMPPUsername(t *testing.T) { + expected := "foo@example.com" + + os.Clearenv() + os.Setenv("WAYBACK_XMPP_USERNAME", expected) + + opts, err := NewParser().ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing environment variables failed: %v`, err) + } + + got := opts.XMPPUsername() + if got != expected { + t.Fatalf(`Unexpected XMPP username got %v instead of %v`, got, expected) + } +} + +func TestXMPPPassword(t *testing.T) { + expected := "bar" + + os.Clearenv() + os.Setenv("WAYBACK_XMPP_PASSWORD", expected) + + opts, err := NewParser().ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing environment variables failed: %v`, err) + } + + got := opts.XMPPPassword() + if got != expected { + t.Fatalf(`Unexpected XMPP password got %v instead of %v`, got, expected) + } +} + +func TestXMPPNoTLS(t *testing.T) { + expected := true + + os.Clearenv() + os.Setenv("WAYBACK_XMPP_NOTLS", strconv.FormatBool(expected)) + + opts, err := NewParser().ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing environment variables failed: %v`, err) + } + + got := opts.XMPPNoTLS() + if got != expected { + t.Fatalf(`Unexpected disable XMPP TLS got %v instead of %v`, got, expected) + } +} + +func TestXMPPHelptext(t *testing.T) { + expected := "some text" + + os.Clearenv() + os.Setenv("WAYBACK_XMPP_HELPTEXT", expected) + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing environment variables failed: %v`, err) + } + + got := opts.XMPPHelptext() + if got != expected { + t.Fatalf(`Unexpected XMPP help text got %v instead of %v`, got, expected) + } +} + func TestNostrRelayURL(t *testing.T) { var tests = []struct { url string diff --git a/config/options.go b/config/options.go index 06f9005b..d57726db 100644 --- a/config/options.go +++ b/config/options.go @@ -64,6 +64,11 @@ const ( defIRCChannel = "" defIRCServer = "irc.libera.chat:6697" + defXMPPUsername = "" + defXMPPPassword = "" + defXMPPNoTLS = false + defXMPPHelptext = "Hi there." + defMatrixHomeserver = "https://matrix.org" defMatrixUserID = "" defMatrixRoomID = "" @@ -125,6 +130,7 @@ type Options struct { nostr *nostr irc *irc onion *onion + xmpp *xmpp listenAddr string chromeRemoteAddr string @@ -226,6 +232,13 @@ type onion struct { disabled bool } +type xmpp struct { + username string + password string + noTLS bool + helptext string +} + // NewOptions returns Options with default values. func NewOptions() *Options { opts := &Options{ @@ -320,6 +333,12 @@ func NewOptions() *Options { localPort: defOnionLocalPort, remotePorts: defOnionRemotePorts, }, + xmpp: &xmpp{ + username: defXMPPUsername, + password: defXMPPPassword, + noTLS: defXMPPNoTLS, + helptext: defXMPPHelptext, + }, } return opts @@ -636,6 +655,26 @@ func (o *Options) PublishToSlackChannel() bool { return o.SlackBotToken() != "" && o.SlackChannel() != "" } +// XMPPUsername returns the XMPP username (JID). +func (o *Options) XMPPUsername() string { + return o.xmpp.username +} + +// XMPPPassword returns the XMPP password. +func (o *Options) XMPPPassword() string { + return o.xmpp.password +} + +// XMPPNoTLS returns whether disable TLS. +func (o *Options) XMPPNoTLS() bool { + return o.xmpp.noTLS +} + +// XMPPHelptext returns the help text for XMPP. +func (o *Options) XMPPHelptext() string { + return breakLine(o.xmpp.helptext) +} + // NotionToken returns the Notion integration token. func (o *Options) NotionToken() string { return o.notion.token diff --git a/config/parser.go b/config/parser.go index df8a8b5e..1e07c766 100644 --- a/config/parser.go +++ b/config/parser.go @@ -177,6 +177,14 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.slack.channel = parseString(val, defSlackChannel) case "WAYBACK_SLACK_HELPTEXT": p.opts.slack.helptext = parseString(val, defSlackHelptext) + case "WAYBACK_XMPP_JID", "WAYBACK_XMPP_USERNAME": + p.opts.xmpp.username = parseString(val, defXMPPUsername) + case "WAYBACK_XMPP_PASSWORD": + p.opts.xmpp.password = parseString(val, defXMPPPassword) + case "WAYBACK_XMPP_NOTLS": + p.opts.xmpp.noTLS = parseBool(val, defXMPPNoTLS) + case "WAYBACK_XMPP_HELPTEXT": + p.opts.xmpp.helptext = parseString(val, defXMPPHelptext) case "WAYBACK_NOSTR_RELAY_URL": p.opts.nostr.url = parseString(val, defNostrRelayURL) case "WAYBACK_NOSTR_PRIVATE_KEY": diff --git a/docs/index.md b/docs/index.md index 24a9776f..2836ddc3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,12 +7,13 @@ Whether you need to archive a single web page or a large collection of websites, ## Features - Free and open-source +- Expose prometheus metrics - Cross-platform compatibility - Batch wayback URLs for faster archiving - Built-in CLI (`wayback`) for convenient use - Serve as a Tor Hidden Service or local web entry for added privacy and accessibility - Easier wayback to Internet Archive, archive.today, IPFS and Telegraph integration -- Interactive with IRC, Matrix, Telegram bot, Discord bot, Mastodon, and Twitter as a daemon service for convenient use +- Interactive with IRC, Matrix, Telegram bot, Discord bot, Mastodon, Twitter, and XMPP as a daemon service for convenient use - Supports publishing wayback results to Telegram channel, Mastodon, and GitHub Issues for sharing - Supports storing archived files to disk for offline use - Download streaming media (requires [FFmpeg](https://ffmpeg.org/)) for convenient media archiving. diff --git a/docs/index.zh.md b/docs/index.zh.md index 46f7e3cf..194e2a46 100644 --- a/docs/index.zh.md +++ b/docs/index.zh.md @@ -12,11 +12,12 @@ Wayback是用Go编写的开源网络存档应用程序。具有模块化和可 - 完全开源 - 跨平台兼容 +- 输出Prometheus度量指标 - 批量存档URL以加快存档速度 - 内置CLI工具（`wayback`）以便于使用 - 可作为Tor隐藏服务或本地Web入口，增加隐私和可访问性 - 更容易地集成到Internet Archive、archive.today、IPFS和Telegraph中 -- 可与IRC、Matrix、Telegram机器人、Discord机器人、Mastodon和Twitter进行交互，作为守护进程服务，以便于使用 +- 可与IRC、Matrix、Telegram机器人、Discord机器人、Mastodon、Twitter和XMPP进行交互，作为守护进程服务，以便于使用 - 支持将存档结果发布到Telegram频道、Mastodon和GitHub Issues中进行共享 - 支持将存档文件存储到磁盘中以供离线使用 - 下载流媒体（需要[FFmpeg](https://ffmpeg.org/)）以便于媒体存档。 diff --git a/docs/integrations/xmpp.md b/docs/integrations/xmpp.md index 00d7bdd4..e67204c0 100644 --- a/docs/integrations/xmpp.md +++ b/docs/integrations/xmpp.md @@ -1 +1,24 @@ -WIP +--- +title: Interactive with XMPP +--- + +## How to build a XMPP Service + +To create an XMPP account, you need to find a client and a public server. Here are some recommended collections of XMPP servers to help you get started. + +- [Directory 404](https://xmpp.404.city/) +- [Public XMPP servers](https://list.jabber.at/) +- [Tracking the progress of OMEMO integration in XMPP clients](https://omemo.top) + +## Configuration + +To use the XMPP service, you will need to set the following environment variables or configuration file: + +- `WAYBACK_IRC_JID`: The JID for the XMPP client (required). +- `WAYBACK_IRC_PASSWORD`: The password for the XMPP client (required). +- `WAYBACK_IRC_NOTLS`: Connect to XMPP server without TLS (optional). +- `WAYBACK_IRC_HELPTEXT`: The help text for XMPP command (optional). + +## Further reading +- [XMPP | The universal messaging standard](https://xmpp.org/) + diff --git a/docs/integrations/xmpp.zh.md b/docs/integrations/xmpp.zh.md index 00d7bdd4..71c8f5f0 100644 --- a/docs/integrations/xmpp.zh.md +++ b/docs/integrations/xmpp.zh.md @@ -1 +1,25 @@ -WIP +--- +title: XMPP +--- + +## 如何构建XMPP服务 + +创建 XMPP 账户，您需要找到一个客户端和一个公共服务器。以下是一些推荐的 XMPP 服务器集合，可帮助您入门。 + +- [Directory 404](https://xmpp.404.city/) +- [Public XMPP servers](https://list.jabber.at/) +- [Tracking the progress of OMEMO integration in XMPP clients](https://omemo.top) + +## 配置 + +为了使用XMPP服务，您需要设置以下环境变量或配置文件： + +- `WAYBACK_IRC_JID`: XMPP客户端的JID（必填）。 +- `WAYBACK_IRC_PASSWORD`: XMPP客户端的密码（必填）。 +- `WAYBACK_IRC_NOTLS`: 连接到XMPP服务器时不使用TLS加密（可选）。 +- `WAYBACK_IRC_HELPTEXT`: XMPP命令的帮助文本（可选）。 + +## 相关资料 + +- [XMPP | The universal messaging standard](https://xmpp.org/) + diff --git a/go.mod b/go.mod index f27bc7de..57d4116e 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,9 @@ require ( golang.org/x/sync v0.1.0 gopkg.in/telebot.v3 v3.0.0-20220130115853-f0291132d3c3 maunium.net/go/mautrix v0.12.0 + mellium.im/sasl v0.3.1 + mellium.im/xmlstream v0.15.4 + mellium.im/xmpp v0.21.4 ) require ( @@ -146,13 +149,16 @@ require ( github.com/wabarc/memento v0.0.0-20210703205719-adc2f8ab8bae // indirect github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect github.com/ybbus/httpretry v1.0.1 // indirect - golang.org/x/crypto v0.4.0 // indirect + golang.org/x/crypto v0.5.0 // indirect golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect + golang.org/x/mod v0.8.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect lukechampine.com/blake3 v1.1.7 // indirect + mellium.im/reader v0.1.0 // indirect mvdan.cc/xurls/v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 2b0975d8..ce3401b3 100644 --- a/go.sum +++ b/go.sum @@ -681,8 +681,8 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -716,6 +716,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -919,6 +921,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1040,6 +1044,14 @@ lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= maunium.net/go/mautrix v0.12.0 h1:jyT1TkJBIRJ7+OW7NhmMHmnEEBLsQe9ml+FYwSLhlaU= maunium.net/go/mautrix v0.12.0/go.mod h1:hHvNi5iKVAiI2MAdAeXHtP4g9BvNEX2rsQpSF/x6Kx4= +mellium.im/reader v0.1.0 h1:UUEMev16gdvaxxZC7fC08j7IzuDKh310nB6BlwnxTww= +mellium.im/reader v0.1.0/go.mod h1:F+X5HXpkIfJ9EE1zHQG9lM/hO946iYAmU7xjg5dsQHI= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= +mellium.im/xmlstream v0.15.4 h1:gLKxcWl4rLMUpKgtzrTBvr4OexPeO/edYus+uK3F6ZI= +mellium.im/xmlstream v0.15.4/go.mod h1:yXaCW2++fmVO4L9piKVkyLDqnCmictVYF7FDQW8prb4= +mellium.im/xmpp v0.21.4 h1:hhAGFC/mGt2Bbmx46vPn+kQT0pJec7uaq+9xckkr9uI= +mellium.im/xmpp v0.21.4/go.mod h1:Emo7bXXyEEgH2hdTO9zp9eGJoc9yK5dAlG0/YVJlh+U= mvdan.cc/xurls/v2 v2.2.0/go.mod h1:EV1RMtya9D6G5DMYPGD8zTQzaHet6Jh8gFlRgGRJeO8= mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc= mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg= diff --git a/metrics/metrics.go b/metrics/metrics.go index 670b036c..49743a8e 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -24,6 +24,7 @@ const ( ServiceMastodon = "mastodon" ServiceTelegram = "telegram" ServiceTwitter = "twitter" + ServiceXMPP = "xmpp" PublishIRC = "irc" // IRC channel PublishGithub = "github" // GitHub issues diff --git a/publish/publish.go b/publish/publish.go index d39b41d6..b7c922dc 100644 --- a/publish/publish.go +++ b/publish/publish.go @@ -28,6 +28,7 @@ const ( FlagSlack // FlagSlack publish from slack service FlagNostr // FlagSlack publish from nostr FlagIRC // FlagIRC publish from relaychat service + FlagXMPP // FlagXMPP publish from XMPP service FlagNotion // FlagNotion is a flag for notion publish service FlagGitHub // FlagGitHub is a flag for github publish service FlagMeili // FlagMeili is a flag for meilisearch publish service diff --git a/service/service.go b/service/service.go index 80f8f913..7d19819b 100644 --- a/service/service.go +++ b/service/service.go @@ -15,6 +15,10 @@ import ( ) const ( + CommandHelp = "help" + CommandMetrics = "metrics" + CommandPlayback = "playback" + MsgWaybackRetrying = "wayback timeout, retrying." MsgWaybackTimeout = "wayback timeout, please try later." ) diff --git a/service/xmpp/doc.go b/service/xmpp/doc.go new file mode 100644 index 00000000..7142a879 --- /dev/null +++ b/service/xmpp/doc.go @@ -0,0 +1,8 @@ +// Copyright 2023 Wayback Archiver. All rights reserved. +// Use of this source code is governed by the GNU GPL v3 +// license that can be found in the LICENSE file. + +/* +Package xmpp implements the xmpp daemon service. +*/ +package xmpp // import "github.com/wabarc/wayback/service/xmpp" diff --git a/service/xmpp/xmpp.go b/service/xmpp/xmpp.go new file mode 100644 index 00000000..3385b31a --- /dev/null +++ b/service/xmpp/xmpp.go @@ -0,0 +1,326 @@ +// Copyright 2023 Wayback Archiver. All rights reserved. +// Use of this source code is governed by the GNU GPL v3 +// license that can be found in the LICENSE file. + +package xmpp // import "github.com/wabarc/wayback/service/xmpp" + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/xml" + "fmt" + "io" + "strings" + "time" + + "github.com/wabarc/logger" + "github.com/wabarc/wayback" + "github.com/wabarc/wayback/config" + "github.com/wabarc/wayback/errors" + "github.com/wabarc/wayback/metrics" + "github.com/wabarc/wayback/pooling" + "github.com/wabarc/wayback/publish" + "github.com/wabarc/wayback/reduxer" + "github.com/wabarc/wayback/service" + "github.com/wabarc/wayback/storage" + "github.com/wabarc/wayback/template/render" + "mellium.im/sasl" + "mellium.im/xmlstream" + "mellium.im/xmpp" + "mellium.im/xmpp/dial" + "mellium.im/xmpp/jid" + "mellium.im/xmpp/stanza" +) + +// ErrServiceClosed is returned by the Service's Serve method after a call to Shutdown. +var ErrServiceClosed = errors.New("xmpp: Service closed") + +// XMPP represents an XMPP service in the application. +type XMPP struct { + ctx context.Context + bot *xmpp.Session + opts *config.Options + pool *pooling.Pool + store *storage.Storage + pub *publish.Publish +} + +// messageBody is a message stanza that contains a body. It is normally used for +// chat messages. +type messageBody struct { + stanza.Message + Body string `xml:"body"` +} + +// New XMPP struct. +func New(ctx context.Context, opts service.Options) *XMPP { + if opts.Config.XMPPUsername() == "" || opts.Config.XMPPPassword() == "" { + logger.Fatal("missing required environment variable") + } + if ctx == nil { + ctx = context.Background() + } + + // Parse and set up JID. + id, err := jid.Parse(opts.Config.XMPPUsername()) + if err != nil { + logger.Fatal("parsing JID failed: %v", err) + } + + // Enable optional features and initialize client session, according to configuration. + features := []xmpp.StreamFeature{xmpp.BindResource()} + if opts.Config.XMPPNoTLS() { + features = append(features, xmpp.StartTLS(&tls.Config{ + ServerName: id.Domain().String(), + MinVersion: tls.VersionTLS12, + })) + } + var defaultAuthMechanisms = []sasl.Mechanism{ + sasl.Plain, + sasl.ScramSha1, + sasl.ScramSha1Plus, + } + + if opts.Config.XMPPPassword() != "" { + features = append(features, xmpp.SASL("", opts.Config.XMPPPassword(), defaultAuthMechanisms...)) + } + + dialCtx, dialCtxCancel := context.WithTimeout(ctx, 30*time.Second) + defer dialCtxCancel() + + // Initialze connection according to configuration. + dialer := &dial.Dialer{NoTLS: opts.Config.XMPPNoTLS(), NoLookup: opts.Config.XMPPNoTLS()} + conn, err := dialer.Dial(dialCtx, "tcp", id) + if err != nil { + logger.Fatal("establishing connection failed: %v", err) + } + + bot, err := xmpp.NewClientSession(dialCtx, id, conn, features...) + if err != nil { + logger.Fatal("new xmpp client failed: %v", err) + } + + // Send initial presence to let the server know we want to receive messages. + err = bot.Send(ctx, stanza.Presence{Type: stanza.AvailablePresence}.Wrap(nil)) + if err != nil { + logger.Fatal("Error sending initial presence: %v", err) + } + + return &XMPP{ + ctx: ctx, + bot: bot, + store: opts.Storage, + opts: opts.Config, + pool: opts.Pool, + pub: opts.Publish, + } +} + +// Serve loop request direct messages from the XMPP server. +// Serve returns an error. +func (x *XMPP) Serve() error { + addr := x.bot.LocalAddr() + logger.Info("Serving XMPP JID: %s", addr) + + // Handle incoming messages. + go func() { + err := x.bot.Serve(xmpp.HandlerFunc(func(t xmlstream.TokenReadEncoder, start *xml.StartElement) error { + // This is a workaround for https://mellium.im/issue/196 + // until a cleaner permanent fix is devised (see https://mellium/issue/197) + d := xml.NewTokenDecoder(xmlstream.MultiReader(xmlstream.Token(*start), t)) + if _, err := d.Token(); err != nil { + return err + } + + // Ignore anything that's not a message. In a real system we'd want to at + // least respond to IQs. + if start.Name.Local != "message" { + return nil + } + + msg := messageBody{} + err := d.DecodeElement(&msg, start) + if err != nil && err != io.EOF { + logger.Error("Error decoding message: %q", err) + return nil + } + + // Don't reflect messages unless they are chat messages and actually have a + // body. + // In a real world situation we'd probably want to respond to IQs, at least. + if msg.Body == "" || msg.Type != stanza.ChatMessage { + return nil + } + + err = x.process(msg) + if err != nil { + logger.Error("process failed: %v", err) + } + + return err + })) + if err != nil { + logger.Error("serve xmpp error: %v", err) + } + }() + + // Block until context cone + <-x.ctx.Done() + + return ErrServiceClosed +} + +// Shutdown shuts down the XMPP service, it always retuan a nil error. +func (x *XMPP) Shutdown() error { + if err := x.bot.Close(); err != nil { + return err + } + + if err := x.bot.Conn().Close(); err != nil { + return err + } + + return nil +} + +func (x *XMPP) process(msg messageBody) error { + cmdctx, cancel := context.WithTimeout(x.ctx, 5*time.Second) + defer cancel() + + cmd := command(msg) + switch cmd { + case service.CommandHelp: + return x.reply(cmdctx, msg, x.opts.XMPPHelptext()) + case service.CommandMetrics: + stats := metrics.Gather.Export("wayback") + if x.opts.EnabledMetrics() && stats != "" { + if err := x.reply(cmdctx, msg, stats); err != nil { + return err + } + } + return nil + case service.CommandPlayback: + return x.playback(cmdctx, msg) + default: + metrics.IncrementWayback(metrics.ServiceXMPP, metrics.StatusRequest) + bucket := pooling.Bucket{ + Request: func(ctx context.Context) error { + if err := x.wayback(ctx, msg); err != nil { + logger.Error("process failure, message: %s, error: %v", msg.Body, err) + return err + } + metrics.IncrementWayback(metrics.ServiceXMPP, metrics.StatusSuccess) + return nil + }, + Fallback: func(ctx context.Context) error { + if err := x.reply(ctx, msg, service.MsgWaybackTimeout); err != nil { + logger.Error("process failure: %v", err) + } + metrics.IncrementWayback(metrics.ServiceXMPP, metrics.StatusFailure) + return nil + }, + } + x.pool.Put(bucket) + } + return nil +} + +func (x *XMPP) wayback(ctx context.Context, msg messageBody) error { + text := msg.Body + logger.Debug("received message: %s", text) + + urls := service.MatchURL(x.opts, text) + if len(urls) == 0 { + return x.reply(ctx, msg, "URL no found") + } + + do := func(cols []wayback.Collect, rdx reduxer.Reduxer) error { + logger.Debug("reduxer: %#v", rdx) + + text := render.ForReply(&render.XMPP{Cols: cols}).String() + err := x.reply(ctx, msg, text) + if err != nil { + return err + } + + x.pub.Spread(ctx, rdx, cols, publish.FlagXMPP) + + return nil + } + + return service.Wayback(ctx, x.opts, urls, do) +} + +func (x *XMPP) playback(ctx context.Context, msg messageBody) error { + metrics.IncrementPlayback(metrics.ServiceXMPP, metrics.StatusRequest) + + urls := service.MatchURL(x.opts, msg.Body) + if len(urls) == 0 { + return x.reply(ctx, msg, "URL no found") + } + + cols, err := wayback.Playback(ctx, x.opts, urls...) + if err != nil { + metrics.IncrementPlayback(metrics.ServiceXMPP, metrics.StatusFailure) + return errors.Wrap(err, "xmpp: playback failed") + } + logger.Debug("playback collections: %#v", cols) + + text := render.ForReply(&render.XMPP{Cols: cols}).String() + if err := x.reply(ctx, msg, text); err != nil { + metrics.IncrementPlayback(metrics.ServiceXMPP, metrics.StatusFailure) + logger.Error("send playback results failed: %v", err) + return err + } + metrics.IncrementPlayback(metrics.ServiceXMPP, metrics.StatusSuccess) + return nil +} + +func (x *XMPP) reply(ctx context.Context, msg messageBody, s string) error { + message := stanza.Message{ + To: msg.From, + Type: msg.Type, + } + body := messageBody{ + Message: message, + Body: fmt.Sprintf("%s\n%s", quote(msg.Body), s), + } + + if err := x.bot.Encode(ctx, body); err != nil { + return err + } + + return nil +} + +func quote(s string) string { + sb := strings.Builder{} + scanner := bufio.NewScanner(strings.NewReader(s)) + for scanner.Scan() { + _, _ = sb.WriteString("> " + scanner.Text() + "\n") + } + if err := scanner.Err(); err != nil { + return s + } + return sb.String() +} + +func command(msg messageBody) string { + body := strings.TrimSpace(msg.Body) + switch { + case strings.HasPrefix(body, service.CommandHelp), + strings.HasPrefix(body, "/"+service.CommandHelp), + strings.HasPrefix(body, service.CommandHelp+":"): + return service.CommandHelp + case strings.HasPrefix(body, service.CommandMetrics), + strings.HasPrefix(body, "/"+service.CommandMetrics), + strings.HasPrefix(body, service.CommandMetrics+":"): + return service.CommandMetrics + case strings.HasPrefix(body, service.CommandPlayback), + strings.HasPrefix(body, "/"+service.CommandPlayback), + strings.HasPrefix(body, service.CommandPlayback+":"): + return service.CommandPlayback + } + return "unknown" +} diff --git a/service/xmpp/xmpp_test.go b/service/xmpp/xmpp_test.go new file mode 100644 index 00000000..7dc377b9 --- /dev/null +++ b/service/xmpp/xmpp_test.go @@ -0,0 +1,253 @@ +// Copyright 2023 Wayback Archiver. All rights reserved. +// Use of this source code is governed by the GNU GPL v3 +// license that can be found in the LICENSE file. + +package xmpp // import "github.com/wabarc/wayback/service/xmpp" + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "reflect" + "strings" + "testing" + "time" + + "github.com/wabarc/helper" + "github.com/wabarc/wayback/config" + "github.com/wabarc/wayback/pooling" + "github.com/wabarc/wayback/publish" + "github.com/wabarc/wayback/service" + "mellium.im/xmpp" + "mellium.im/xmpp/jid" + "mellium.im/xmpp/stanza" +) + +var ( + pass = "foobar" + notls = "true" + + readyFeature = xmpp.StreamFeature{ + Name: xml.Name{Space: "urn:example", Local: "ready"}, + Parse: func(ctx context.Context, d *xml.Decoder, start *xml.StartElement) (bool, interface{}, error) { + _, err := d.Token() + return false, nil, err + }, + Negotiate: func(ctx context.Context, session *xmpp.Session, data interface{}) (xmpp.SessionState, io.ReadWriter, error) { + return xmpp.Ready, nil, nil + }, + } + negotiator = xmpp.NewNegotiator(func(*xmpp.Session, *xmpp.StreamConfig) xmpp.StreamConfig { + return xmpp.StreamConfig{ + Features: []xmpp.StreamFeature{readyFeature}, + } + }) + client = `` + + to = jid.MustParse("to@example.net") + from = jid.MustParse("from@example.net") + msg = stanza.Message{ + XMLName: xml.Name{Local: "message"}, + ID: "123", + To: to, + From: from, + Lang: "te", + Type: stanza.ChatMessage, + } + start = msg.StartElement() +) + +func setenv(t *testing.T, id string) { + t.Setenv("WAYBACK_XMPP_JID", id) + t.Setenv("WAYBACK_XMPP_PASSWORD", pass) + t.Setenv("WAYBACK_XMPP_NOTLS", notls) + t.Setenv("WAYBACK_ENABLE_IA", "true") +} + +func parseOpts(t *testing.T) service.Options { + parser := config.NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf("Parse environment variables or flags failed, error: %v", err) + } + + pool := &pooling.Pool{} + pub := &publish.Publish{} + return service.ParseOptions( + service.Config(opts), + service.Pool(pool), + service.Publish(pub), + ) +} + +func xmppSession(ctx context.Context, t *testing.T) *xmpp.Session { + buf := &bytes.Buffer{} + rw := struct { + io.Reader + io.Writer + }{ + Reader: strings.NewReader(client), + Writer: buf, + } + + session, err := xmpp.NewSession(ctx, jid.JID{}, jid.JID{}, rw, xmpp.SessionState(0), negotiator) + if err != nil { + t.Fatalf("Unexpected new xmpp client session: %v", err) + } + + return session +} + +func TestServe(t *testing.T) { + opts := parseOpts(t) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + s := xmppSession(ctx, t) + x := &XMPP{ + bot: s, + ctx: ctx, + opts: opts.Config, + } + err := x.Serve() + if err != ErrServiceClosed { + t.Fatalf("Unexpected serve xmpp session: %v", err) + } +} + +func TestShutdown(t *testing.T) { + opts := parseOpts(t) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + s := xmppSession(ctx, t) + x := &XMPP{ + bot: s, + ctx: ctx, + opts: opts.Config, + } + err := x.Shutdown() + if err != nil { + t.Fatalf("Unexpected shutdown xmpp session: %v", err) + } +} + +func TestProcess(t *testing.T) { + if testing.Short() { + t.Skip("Skip test in short mode.") + } + + opts := parseOpts(t) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + s := xmppSession(ctx, t) + x := &XMPP{ + bot: s, + ctx: ctx, + opts: opts.Config, + pool: opts.Pool, + } + + mb := messageBody{msg, "foo uri"} + err := x.process(mb) + if err != nil { + t.Fatalf("Error decoding message: %q", err) + } +} + +func TestWayback(t *testing.T) { + if testing.Short() { + t.Skip("Skip test in short mode.") + } + + opts := parseOpts(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, mux, server := helper.MockServer() + defer server.Close() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + // TODO: handle request + }) + + s := xmppSession(ctx, t) + x := &XMPP{ + bot: s, + ctx: ctx, + opts: opts.Config, + pool: opts.Pool, + } + + tests := [...]struct { + name string + uri string + err error + }{ + {"without uri", "", fmt.Errorf("URL no found")}, + {"with uri", server.URL, context.DeadlineExceeded}, // TODO: need a complete testing + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mb := messageBody{msg, tt.uri} + err := x.wayback(ctx, mb) + if err != nil && !reflect.DeepEqual(err, tt.err) { + t.Fatalf("Error wayback: %q", err) + } + }) + } +} + +func TestPlayback(t *testing.T) { + if testing.Short() { + t.Skip("Skip test in short mode.") + } + + opts := parseOpts(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, mux, server := helper.MockServer() + defer server.Close() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + // TODO: handle request + }) + + s := xmppSession(ctx, t) + x := &XMPP{ + bot: s, + ctx: ctx, + opts: opts.Config, + pool: opts.Pool, + } + + tests := [...]struct { + name string + uri string + err error + }{ + {"without uri", "", fmt.Errorf("URL no found")}, + // {"with uri", server.URL, context.DeadlineExceeded}, // TODO: need a complete testing + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mb := messageBody{msg, tt.uri} + err := x.playback(ctx, mb) + if err != nil && !reflect.DeepEqual(err, tt.err) { + t.Fatalf("Error wayback: %q", err) + } + }) + } +} diff --git a/template/render/xmpp.go b/template/render/xmpp.go new file mode 100644 index 00000000..92aa1b51 --- /dev/null +++ b/template/render/xmpp.go @@ -0,0 +1,57 @@ +// Copyriit 2023 Wayback Archiver. All riits reserved. +// Use of this source code is governed by the GNU GPL v3 +// license that can be found in the LICENSE file. + +package render // import "github.com/wabarc/wayback/template/render" + +import ( + "bytes" + "text/template" + + "github.com/wabarc/logger" + "github.com/wabarc/wayback" +) + +var _ Renderer = (*XMPP)(nil) + +// XMPP represents a XMPP template data for render. +type XMPP struct { + Cols []wayback.Collect + Data interface{} +} + +// ForReply implements the standard Renderer interface: +// it returns a Render from the ForPublish. +func (x *XMPP) ForReply() *Render { + return x.ForPublish() +} + +// ForPublish implements the standard Renderer interface: +// it reads `[]wayback.Collect` from the XMPP and returns a *Render. +func (x *XMPP) ForPublish() (r *Render) { + var tmplBytes bytes.Buffer + + const tmpl = `{{range $ := .}}{{ $.Arc | name }}: +{{ range $map := $.Dst -}} +{{ range $src, $dst := $map -}} +• {{ if $dst | isURL }}{{ $dst }}{{ else }}{{ $dst | escapeString }}{{ end }} +{{ end }}{{ end }} +{{ end }}` + + tpl, err := template.New("message").Funcs(funcMap()).Parse(tmpl) + if err != nil { + logger.Error("parse Telegram template failed, %v", err) + return r + } + + groups := groupBySlot(x.Cols) + logger.Debug("for reply telegram: %#v", groups) + if err = tpl.Execute(&tmplBytes, groups); err != nil { + logger.Error("execute Telegram template failed, %v", err) + return r + } + tmplBytes = *bytes.NewBuffer(bytes.TrimSpace(tmplBytes.Bytes())) + tmplBytes.WriteString("\n") + + return &Render{buf: tmplBytes} +} diff --git a/template/render/xmpp_test.go b/template/render/xmpp_test.go new file mode 100644 index 00000000..d08bf237 --- /dev/null +++ b/template/render/xmpp_test.go @@ -0,0 +1,28 @@ +// Copyright 2023 Wayback Archiver. All rights reserved. +// Use of this source code is governed by the GNU GPL v3 +// license that can be found in the LICENSE file. + +package render // import "github.com/wabarc/wayback/template/render" + +import ( + "testing" +) + +func TestRenderForXMPP(t *testing.T) { + expected := `Internet Archive: +• https://web.archive.org/web/20211000000001/https://example.com/ + +IPFS: +• https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr + +archive.today: +• http://archive.today/abcdE + +Telegraph: +• http://telegra.ph/title-01-01` + + got := ForPublish(&XMPP{Cols: collects, Data: bundleExample}).String() + if got != expected { + t.Errorf("Unexpected render template for XMPP, got \n%s\ninstead of \n%s", got, expected) + } +} diff --git a/wayback.1 b/wayback.1 index 636f4c71..f34fcb8f 100644 --- a/wayback.1 +++ b/wayback.1 @@ -252,6 +252,18 @@ Channel ID of Slack channel\&. .B WAYBACK_SLACK_HELPTEXT The help text for Slack slash command\&. .TP +.B WAYBACK_XMPP_JID +The JID of a XMPP account\&. +.TP +.B WAYBACK_XMPP_PASSWORD +The password of a XMPP account\&. +.TP +.B WAYBACK_XMPP_NOTLS +Connect to XMPP server without TLS\&. +.TP +.B WAYBACK_XMPP_HELPTEXT +The help text for XMPP command\&. +.TP .B WAYBACK_GITHUB_TOKEN GitHub Personal Access Token, required the `repo` scope\&. .TP diff --git a/wayback.conf b/wayback.conf index a6893913..922454bb 100644 --- a/wayback.conf +++ b/wayback.conf @@ -45,6 +45,10 @@ WAYBACK_NOTION_TOKEN= WAYBACK_NOTION_DATABASE_ID= WAYBACK_NOSTR_RELAY_URL=wss://nostr.developer.li WAYBACK_NOSTR_PRIVATE_KEY= +WAYBACK_XMPP_JID= +WAYBACK_XMPP_PASSWORD= +WAYBACK_XMPP_NOTLS= +WAYBACK_XMPP_HELPTEXT=Hi,\n\nI'm a 🤖 to help you backup webpages more easily. Send me any text containing the URL and I'll give you the result back 😀\n\nProject: https://github.com/wabarc\n\nExample:\nSome text, https://example.com foo https://example.org WAYBACK_MEILI_ENDPOINT= WAYBACK_MEILI_INDEXING=capsules WAYBACK_MEILI_APIKEY=