Skip to content

Commit

Permalink
Add publish to GitHub Issues support
Browse files Browse the repository at this point in the history
  • Loading branch information
web-flow committed Feb 24, 2021
1 parent 4ab0e69 commit 6290490
Show file tree
Hide file tree
Showing 11 changed files with 229 additions and 10 deletions.
10 changes: 9 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

## [0.7.0] - 2021-02-24

### Added
- Add publish to GitHub Issues support.

### Changed
- Styling channel message.

## [0.6.3] - 2021-02-21

### Changed
Expand All @@ -18,7 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [0.6.0] - 2021-01-28

### Add
### Added
- Add wayback to Telegraph support.

### Changed
Expand Down
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,9 @@ You can specify configuration options either via command flags or via environmen
| `--ipfs-host` | `WAYBACK_IPFS_HOST` | `127.0.0.1` | IPFS daemon service host |
| `-p`, `--ipfs-port` | `WAYBACK_IPFS_PORT` | `5001` | IPFS daemon service port |
| `-m`, `--ipfs-mode` | `WAYBACK_IPFS_MODE` | `pinner` | IPFS mode for preserve webpage, e.g. `daemon`, `pinner` |
| - | `WAYBACK_GITHUB_TOKEN` | - | GitHub Personal Access Token, required the `repo` scope |
| - | `WAYBACK_GITHUB_OWNER` | - | GitHub account name |
| - | `WAYBACK_GITHUB_REPO` | - | GitHub repository to publish results |
| `-t`, `--token` | `WAYBACK_TELEGRAM_TOKEN` | - | Telegram Bot API Token |
| `-c`, `--chatid` | `WAYBACK_TELEGRAM_CHANNEL` | - | The **Telegram Channel** name for publish archived result |
| `--tor` | `WAYBACK_USE_TOR` | `false` | Snapshot webpage via Tor anonymity network |
Expand Down
54 changes: 54 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,3 +343,57 @@ func TestDefaultTorRemotePortsValue(t *testing.T) {
t.Fatalf(`Unexpected Tor private key, got %v instead of %v`, got, expected)
}
}

func TestGitHubToken(t *testing.T) {
os.Clearenv()
os.Setenv("WAYBACK_GITHUB_TOKEN", "github:token")

parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing environment variables failed: %v`, err)
}

expected := "github:token"
got := opts.GitHubToken()

if got != expected {
t.Fatalf(`Unexpected GitHub personal access token, got %v instead of %s`, got, expected)
}
}

func TestGitHubOwner(t *testing.T) {
os.Clearenv()
os.Setenv("WAYBACK_GITHUB_OWNER", "github-owner")

parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing environment variables failed: %v`, err)
}

expected := "github-owner"
got := opts.GitHubOwner()

if got != expected {
t.Fatalf(`Unexpected GitHub owner, got %v instead of %s`, got, expected)
}
}

func TestGitHubRepo(t *testing.T) {
os.Clearenv()
os.Setenv("WAYBACK_GITHUB_REPO", "github-repo")

parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing environment variables failed: %v`, err)
}

expected := "github-repo"
got := opts.GitHubRepo()

if got != expected {
t.Fatalf(`Unexpected GitHub repository, got %v instead of %s`, got, expected)
}
}
40 changes: 40 additions & 0 deletions config/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ const (

defTelegramToken = ""
defTelegramChannel = ""
defGitHubToken = ""
defGitHubOwner = ""
defGitHubRepo = ""

defTorPrivateKey = ""
defTorLocalPort = 0
Expand All @@ -38,6 +41,7 @@ type Options struct {
ipfs *ipfs
slots map[string]bool
telegram *telegram
github *github
tor *tor
}

Expand All @@ -59,6 +63,12 @@ type telegram struct {
channel string
}

type github struct {
token string
owner string
repo string
}

type tor struct {
pvk string

Expand Down Expand Up @@ -88,6 +98,11 @@ func NewOptions() *Options {
token: defTelegramToken,
channel: defTelegramChannel,
},
github: &github{
token: defGitHubToken,
owner: defGitHubOwner,
repo: defGitHubRepo,
},
tor: &tor{
pvk: defTorPrivateKey,
localPort: defTorLocalPort,
Expand Down Expand Up @@ -144,6 +159,31 @@ func (o *Options) TelegramChannel() string {
return o.telegram.channel
}

// PublishToChannel returns whether to publish results to Telegram Channel.
func (o *Options) PublishToChannel() bool {
return o.telegram.token != "" && o.telegram.channel != ""
}

// GitHubToken returns the personal access token of GitHub account.
func (o *Options) GitHubToken() string {
return o.github.token
}

// GitHubOwner returns the user id of GitHub account.
func (o *Options) GitHubOwner() string {
return o.github.owner
}

// GitHubRepo returns the GitHub repository which to publish results.
func (o *Options) GitHubRepo() string {
return o.github.repo
}

// PublishToIssues returns whether to publish results to GitHub issues.
func (o *Options) PublishToIssues() bool {
return o.github.token != "" && o.github.owner != "" && o.github.repo != ""
}

// TorPrivKey returns the private key of Tor service.
func (o *Options) TorPrivKey() string {
return o.tor.pvk
Expand Down
6 changes: 6 additions & 0 deletions config/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.telegram.token = parseString(val, defTelegramToken)
case "WAYBACK_TELEGRAM_CHANNEL":
p.opts.telegram.channel = parseString(val, defTelegramChannel)
case "WAYBACK_GITHUB_TOKEN":
p.opts.github.token = parseString(val, defGitHubToken)
case "WAYBACK_GITHUB_OWNER":
p.opts.github.owner = parseString(val, defGitHubOwner)
case "WAYBACK_GITHUB_REPO":
p.opts.github.repo = parseString(val, defGitHubRepo)
case "WAYBACK_TOR_PRIVKEY":
p.opts.tor.pvk = parseString(val, defTorPrivateKey)
case "WAYBACK_TOR_LOCAL_PORT":
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
module github.com/wabarc/wayback

go 1.16
go 1.13

require (
github.com/btcsuite/btcd v0.21.0-beta // indirect
github.com/chromedp/cdproto v0.0.0-20210211211551-bf465a41a1f9 // indirect
github.com/cretz/bine v0.1.0
github.com/go-shiori/obelisk v0.0.0-20201115143556-8de0d40b0a9b // indirect
github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible
github.com/google/go-github/v33 v33.0.0
// github.com/ipsn/go-libtor v1.0.329
github.com/libp2p/go-libp2p-core v0.8.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,10 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM=
github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
Expand Down Expand Up @@ -172,8 +176,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs=
github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
Expand Down
4 changes: 2 additions & 2 deletions publish/channel.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ func Render(vars []*Collect) string {
var tmplBytes bytes.Buffer

const tmpl = `{{range $ := .}}<b>{{ $.Arc }}</b>:
{{ range $url := $.Dst -}}
• {{ $url }}
{{ range $src, $dst := $.Dst -}}
<a href="{{ $src }}">origin</a> - {{ $dst }}
{{end}}
{{end}}`

Expand Down
83 changes: 83 additions & 0 deletions publish/github.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright 2021 Wayback Archiver. All rights reserved.
// Use of this source code is governed by the GNU GPL v3
// license that can be found in the LICENSE file.

package publish // import "github.com/wabarc/wayback/publish"

import (
"bytes"
"context"
"net/http"
"text/template"
"time"

"github.com/google/go-github/v33/github"
"github.com/wabarc/wayback/config"
"github.com/wabarc/wayback/logger"
)

type GitHub struct {
client *http.Client
}

func NewGitHub() *GitHub {
return &GitHub{}
}

func ToIssues(ctx context.Context, opts *config.Options, text string) bool {
if opts.GitHubToken() == "" {
logger.Error("GitHub personal access token is required")
return false
}

// Authenticated user must grant repo:public_repo scope,
// private repository need whole repo scope.
auth := github.BasicAuthTransport{
Username: opts.GitHubOwner(),
Password: opts.GitHubToken(),
}
client := github.NewClient(auth.Client())

if opts.HasDebugMode() {
user, _, _ := client.Users.Get(ctx, "")
logger.Debug("Authorized GitHub user: %v", user)
}

// Create an issue to GitHub
t := "Published at " + time.Now().Format("2006-01-02T15:04:05")
ir := &github.IssueRequest{Title: github.String(t), Body: github.String(text)}
issue, _, err := client.Issues.Create(ctx, opts.GitHubOwner(), opts.GitHubRepo(), ir)
if err != nil {
logger.Debug("%v", err)
return false
}
logger.Debug("Created issue: %v", issue)

return true
}

func (gh *GitHub) Render(vars []*Collect) string {
var tmplBytes bytes.Buffer

const tmpl = `{{range $ := .}}**{{ $.Arc }}**:
{{ range $src, $dst := $.Dst -}}
> origin: {{ $src }}
> archived: {{ $dst }}
{{end}}
{{end}}`

tpl, err := template.New("message").Parse(tmpl)
if err != nil {
logger.Debug("GitHub: parse template failed, %v", err)
return ""
}

err = tpl.Execute(&tmplBytes, vars)
if err != nil {
logger.Debug("Telegram: execute template failed, %v", err)
return ""
}

return tmplBytes.String()
}
18 changes: 16 additions & 2 deletions service/anonymity/tor.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,14 @@ func (t *tor) process(w http.ResponseWriter, r *http.Request, ctx context.Contex
if data, err := json.Marshal(collector); err != nil {
logger.Error("Web: encode for response failed, %v", err)
} else {
go publish.ToChannel(t.opts, nil, publish.Render(col))
if t.opts.PublishToChannel() {
logger.Debug("Web: publishing to channel...")
go publish.ToChannel(t.opts, nil, publish.Render(col))
}
if t.opts.PublishToIssues() {
logger.Debug("Web: publishing to GitHub issues...")
go publish.ToIssues(ctx, t.opts, publish.NewGitHub().Render(col))
}
w.Write(data)
}

Expand All @@ -144,7 +151,14 @@ func (t *tor) process(w http.ResponseWriter, r *http.Request, ctx context.Contex
w.Header().Set("Content-Type", "text/html; charset=utf-8")

if html, ok := collector.Render(); ok {
go publish.ToChannel(t.opts, nil, publish.Render(col))
if t.opts.PublishToChannel() {
logger.Debug("Web: publishing to channel...")
go publish.ToChannel(t.opts, nil, publish.Render(col))
}
if t.opts.PublishToIssues() {
logger.Debug("Web: publishing to GitHub issues...")
go publish.ToIssues(ctx, t.opts, publish.NewGitHub().Render(col))
}
w.Write(html)
} else {
logger.Error("Web: render template for response failed")
Expand Down
12 changes: 8 additions & 4 deletions service/telegram/telegram.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func New(opts *config.Options) *telegram {

// Serve loop request message from the Telegram api server.
// Serve always returns a nil error.
func (t *telegram) Serve(_ context.Context) (err error) {
func (t *telegram) Serve(ctx context.Context) (err error) {
if t.bot, err = tgbotapi.NewBotAPI(t.opts.TelegramToken()); err != nil {
return errors.New("Initialize telegram failed, error: %v", err)
}
Expand All @@ -55,13 +55,13 @@ func (t *telegram) Serve(_ context.Context) (err error) {
}

t.upd = update
go t.process()
go t.process(ctx)
}

return nil
}

func (t *telegram) process() {
func (t *telegram) process(ctx context.Context) {
bot, update := t.bot, t.upd
message := update.Message
text := message.Text
Expand Down Expand Up @@ -93,10 +93,14 @@ func (t *telegram) process() {

bot.Send(msg)

if t.opts.TelegramChannel() != "" {
if t.opts.PublishToChannel() {
logger.Debug("Telegram: publishing to channel...")
publish.ToChannel(t.opts, bot, replyText)
}
if t.opts.PublishToIssues() {
logger.Debug("Telegram: publishing to GitHub issues...")
publish.ToIssues(ctx, t.opts, publish.NewGitHub().Render(col))
}
}

func (t *telegram) archive(urls []string) (col []*publish.Collect, err error) {
Expand Down

0 comments on commit 6290490

Please sign in to comment.