diff --git a/.gitignore b/.gitignore index b5d05809..817889d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.zsign_cache vendor/ coverage.txt packrd/ diff --git a/Dockerfile b/Dockerfile index 4cc0e334..df315365 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,10 @@ +# zsign builder +FROM alpine:3.11 as zsign-build +RUN apk add --no-cache --virtual .build-deps git g++ openssl-dev libgcc libstdc++ zip unzip +RUN git clone https://github.com/zhlynn/zsign +WORKDIR zsign +RUN g++ ./*.cpp common/*.cpp -lcrypto -O3 -o zsign + # web build FROM node:10 as web-build WORKDIR /app @@ -26,7 +33,8 @@ RUN make install # minimalist runtime FROM alpine:3.11 -RUN apk add --update --no-cache ca-certificates +RUN apk add --update --no-cache ca-certificates libstdc++ unzip zip COPY --from=go-build /go/bin/yolo /bin/ +COPY --from=zsign-build zsign/zsign /bin/ ENTRYPOINT ["yolo"] EXPOSE 8000 diff --git a/deployments/yolo.berty.io/.env.example b/deployments/yolo.berty.io/.env.example new file mode 100644 index 00000000..15f2974a --- /dev/null +++ b/deployments/yolo.berty.io/.env.example @@ -0,0 +1,12 @@ +GITHUB_TOKEN= +CIRCLE_TOKEN= +HOSTNAME= +BUILDKITE_TOKEN= +BASIC_AUTH_PASSWORD= +BINTRAY_TOKEN= +BINTRAY_USERNAME= +BEARER_SECRETKEY= +AUTH_SALT= +IOS_PASS= +IOS_PROV= +IOS_PRIVKEY= diff --git a/deployments/yolo.berty.io/docker-compose.yml b/deployments/yolo.berty.io/docker-compose.yml index a3ea9266..085a3cb2 100644 --- a/deployments/yolo.berty.io/docker-compose.yml +++ b/deployments/yolo.berty.io/docker-compose.yml @@ -5,18 +5,33 @@ services: image: bertytech/yolo:latest restart: unless-stopped network_mode: bridge + working_dir: /tmp volumes: - ./data:/data + - ~/codesign:/codesign:ro expose: - 8000 environment: - - BUILDKITE_TOKEN=${YOLO_BUILDKITE_TOKEN} - - CIRCLE_TOKEN=${YOLO_CIRCLE_TOKEN} - - GITHUB_TOKEN=${YOLO_GITHUB_TOKEN} - - BINTRAY_TOKEN=${YOLO_BINTRAY_TOKEN} - - BINTRAY_USERNAME=${YOLO_BINTRAY_USERNAME} - - BEARER_SECRETKEY=${YOLO_BEARER_SECRETKEY} - command: -v server --cors-allowed-origins="*" --max-builds=30 --db-path=/data/yolo.sqlite --basic-auth-password="${YOLO_BASIC_AUTH_PASSWORD}" --request-timeout=10s --shutdown-timeout=11s --http-cache-path=/data/httpcache --artifacts-cache-path=/data/artifacts-cache + - BUILDKITE_TOKEN + - CIRCLE_TOKEN + - GITHUB_TOKEN + - BINTRAY_TOKEN + - BINTRAY_USERNAME + - BEARER_SECRETKEY + - BASIC_AUTH_PASSWORD + - IOS_PASS + - IOS_PROV + - IOS_PRIVKEY + command: + - -v + - server + - --cors-allowed-origins=* + - --max-builds=30 + - --db-path=/data/yolo.sqlite + - --request-timeout=10s + - --shutdown-timeout=11s + - --http-cache-path=/data/httpcache + - --artifacts-cache-path=/data/artifacts-cache labels: - 'com.centurylinklabs.watchtower.enable=true' # traefik specific labels @@ -28,5 +43,5 @@ services: - 'traefik.http.routers.yolo.tls.certresolver=cf' - 'traefik.http.routers.yolo.tls.domains[0].main=berty.io' - 'traefik.http.routers.yolo.tls.domains[0].sans=yolo.berty.io' - - 'traefik.http.services.yolo.loadbalancer.server.port=8000' + - 'traefik.http.services.yolo.loadbalancer.healthcheck.port=9090' diff --git a/go.mod b/go.mod index 211164b4..c2b5a57e 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,6 @@ require ( moul.io/godev v1.6.0 moul.io/hcfilters v1.3.1 moul.io/pkgman v1.3.1 - moul.io/u v1.13.0 + moul.io/u v1.16.0 moul.io/zapgorm v1.0.0 ) diff --git a/go.sum b/go.sum index f7cf7e35..fe0962e3 100644 --- a/go.sum +++ b/go.sum @@ -184,6 +184,7 @@ github.com/peterbourgon/ff/v2 v2.0.0 h1:lx0oYI5qr/FU1xnpNhQ+EZM04gKgn46jyYvGEEqB github.com/peterbourgon/ff/v2 v2.0.0/go.mod h1:xjwr+t+SjWm4L46fcj/D+Ap+6ME7+HqFzaP22pP5Ggk= github.com/peterbourgon/ff/v3 v3.0.0 h1:eQzEmNahuOjQXfuegsKQTSTDbf4dNvr/eNLrmJhiH7M= github.com/peterbourgon/ff/v3 v3.0.0/go.mod h1:UILIFjRH5a/ar8TjXYLTkIvSvekZqPm5Eb/qbGk6CT0= +github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -210,6 +211,7 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.19.0 h1:hYz4ZVdUgjXTBUmrkrw55j1nHx68LfOKIQk5IYtyScg= github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -237,6 +239,7 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/tracer v0.0.0-20140124184152-66d3696bba97 h1:ZXZ3Ko4supnaInt/pSZnq3QL65Qx/KSZTUPMJH5RlIk= github.com/stretchr/tracer v0.0.0-20140124184152-66d3696bba97/go.mod h1:H0mYc1JTiYc9K0keLMYcR2ybyeom20X4cOYrKya1M1Y= +github.com/tailscale/depaware v0.0.0-20200914232109-e09ee10c1824/go.mod h1:nyzwKFaLuckPu3dAJHH7B6lMi4xDBWzD0r3pEpGZm2Y= github.com/tdewolff/minify/v2 v2.7.6 h1:b6UzNphZeDm3AVmk0a69orkNLPJzJx3k/AQ/W2xoMs8= github.com/tdewolff/minify/v2 v2.7.6/go.mod h1:Mt3hGbK/ETDplEP9EMNZo1lPkM3TZq0rDIVV76nFgY0= github.com/tdewolff/parse/v2 v2.4.3 h1:k24zHgTRGm7LkvbTEreuavyZTf0k8a/lIenggv62OiU= @@ -256,9 +259,13 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -406,5 +413,7 @@ moul.io/pkgman v1.3.1 h1:3b5p5pDQjoJyyz4gkr4pBDtnBo7+icJXB2hwZfj/VFs= moul.io/pkgman v1.3.1/go.mod h1:+eYoFBmiTgE7NGLV6cD0jLG2ECV8nnoEZeH/FC4FZ3g= moul.io/u v1.13.0 h1:ijCMDQ1KnCjhjltNh4iaELHrKliu4IpG7PqgP1hdmvU= moul.io/u v1.13.0/go.mod h1:bfsObgTWe/6jwjxpacwG+gUXoSNrfwZj7Avl19Wq0w4= +moul.io/u v1.16.0 h1:r2+SLT8ZY4OcHpSWe8w7KrfMLI96e+oPDXvf/nSlnTs= +moul.io/u v1.16.0/go.mod h1:1HyJtKaRkmcakL4pVjJjCMNaxzyDIUXzeyjic6T9wIw= moul.io/zapgorm v1.0.0 h1:HpO9x1TmsKFd4JoLNHrSIc1uZn6kmyDxLQK4xjfz8JE= moul.io/zapgorm v1.0.0/go.mod h1:JDE3xz5BQ1ccnAijE5+T8Qin6T256Bw2Cpdi+qMfWgw= diff --git a/go/cmd/yolo/main.go b/go/cmd/yolo/main.go index 933f9c2d..ef7e5a1a 100644 --- a/go/cmd/yolo/main.go +++ b/go/cmd/yolo/main.go @@ -71,6 +71,9 @@ func yolo(args []string) error { httpCachePath string realm string once bool + iosPrivkeyPath string + iosProvPath string + iosPrivkeyPass string ) var ( rootFlagSet = flag.NewFlagSet("yolo", flag.ExitOnError) @@ -102,6 +105,9 @@ func yolo(args []string) error { serverFlagSet.StringVar(&authSalt, "auth-salt", "", "salt used to generate authentication tokens at the end of the URLs") serverFlagSet.StringVar(&httpCachePath, "http-cache-path", "", "if set, will cache http client requests") serverFlagSet.BoolVar(&once, "once", false, "just run workers once") + serverFlagSet.StringVar(&iosPrivkeyPath, "ios-privkey", "", "iOS signing: path to private key or p12 file (PEM or DER format)") + serverFlagSet.StringVar(&iosProvPath, "ios-prov", "", "iOS signing: path to mobile provisioning profile") + serverFlagSet.StringVar(&iosPrivkeyPass, "ios-pass", "", "iOS signing: password for private key or p12 file") storeFlagSet.StringVar(&dbStorePath, "db-path", ":memory:", "DB Store path") storeFlagSet.BoolVar(&withPreloading, "with-preloading", false, "with auto DB preloading") @@ -180,6 +186,9 @@ func yolo(args []string) error { AuthSalt: authSalt, DevMode: devMode, ArtifactsCachePath: artifactsCachePath, + IOSPrivkeyPath: iosPrivkeyPath, + IOSProvPath: iosProvPath, + IOSPrivkeyPass: iosPrivkeyPass, }) if err != nil { return err diff --git a/go/pkg/yolosvc/api_download.go b/go/pkg/yolosvc/api_download.go index 470a07f0..c030c7fa 100644 --- a/go/pkg/yolosvc/api_download.go +++ b/go/pkg/yolosvc/api_download.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "os" + "os/exec" "path" "path/filepath" "strconv" @@ -21,9 +22,12 @@ import ( "github.com/go-chi/chi" "go.uber.org/zap" "google.golang.org/grpc/codes" + "moul.io/u" ) func (svc *service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) { + // FIXME: if caching enabled, lock by artifact ID + id := chi.URLParam(r, "artifactID") var artifact yolopb.Artifact err := svc.db.First(&artifact, "ID = ?", id).Error @@ -32,6 +36,119 @@ func (svc *service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) { return } + switch ext := filepath.Ext(artifact.LocalPath); ext { + case ".unsigned-ipa", ".dummy-signed-ipa": + if !u.CommandExists("zsign") { + httpError(w, fmt.Errorf("missing signing binary"), codes.Internal) + return + } + if svc.iosPrivkeyPath == "" || svc.iosProvPath == "" { + httpError(w, fmt.Errorf("missing iOS signing configuration"), codes.InvalidArgument) + return + } + if !u.FileExists(svc.iosPrivkeyPath) || !u.FileExists(svc.iosProvPath) { + httpError(w, fmt.Errorf("invalid iOS signing configuration"), codes.InvalidArgument) + return + } + + // send some headers early, to make loading icon appearing soon on the iOS device + { + filename := strings.TrimSuffix(path.Base(artifact.LocalPath), ext) + ".ipa" + w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename)) + if artifact.MimeType != "" { + w.Header().Add("Content-Type", artifact.MimeType) + } + } + + // FIXME: cache file + send content-length + + err := svc.signAndStreamIPA(artifact, w) + if err != nil { + httpError(w, err, codes.Internal) + } + default: + base := path.Base(artifact.LocalPath) + w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=%s", base)) + if artifact.FileSize > 0 { + w.Header().Add("Content-Length", fmt.Sprintf("%d", artifact.FileSize)) + } + if artifact.MimeType != "" { + w.Header().Add("Content-Type", artifact.MimeType) + } + + err := svc.artifactToStream(artifact, w) + if err != nil { + httpError(w, err, codes.Internal) + } + } +} + +func (svc *service) signAndStreamIPA(artifact yolopb.Artifact, w io.Writer) error { + // sign ipa + var signed string + { + tempdir, err := ioutil.TempDir("", "yolo") + if err != nil { + return err + } + defer os.RemoveAll(tempdir) + + // write unsigned-file to tempdir + unsigned := filepath.Join(tempdir, "unsigned.ipa") + signed = filepath.Join(tempdir, "signed.ipa") + f, err := os.OpenFile(unsigned, os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return err + } + err = svc.artifactToStream(artifact, f) + if err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + + // zsign the archive + zsignArgs := []string{ + "-k", svc.iosPrivkeyPath, + "-m", svc.iosProvPath, // should be retrieved from the artifact archive directly + "-o", signed, + "-z", "1", + } + if svc.iosPrivkeyPass != "" { + zsignArgs = append(zsignArgs, + "-p", svc.iosPrivkeyPass, + ) + } + zsignArgs = append(zsignArgs, unsigned) + cmd := exec.Command("zsign", zsignArgs...) + svc.logger.Info("zsign", zap.Strings("args", zsignArgs)) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + err = cmd.Run() + if err != nil { + return err + } + } + + // send the signed iPA + { + f, err := os.Open(signed) + if err != nil { + return err + } + + // content-length + + _, err = io.Copy(w, f) + if err != nil { + return err + } + } + return nil +} + +func (svc *service) artifactToStream(artifact yolopb.Artifact, w io.Writer) error { cache := filepath.Join(svc.artifactsCachePath, artifact.ID) // download missing cache if svc.artifactsCachePath != "" { @@ -40,22 +157,12 @@ func (svc *service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) { err := svc.artifactDownloadToFile(&artifact, cache) if err != nil { svc.artifactsCacheMutex.Unlock() - httpError(w, err, codes.Internal) - return + return err } } svc.artifactsCacheMutex.Unlock() } - base := path.Base(artifact.LocalPath) - w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=%s", base)) - if artifact.FileSize > 0 { - w.Header().Add("Content-Length", fmt.Sprintf("%d", artifact.FileSize)) - } - if artifact.MimeType != "" { - w.Header().Add("Content-Type", artifact.MimeType) - } - // save download now := time.Now() download := yolopb.Download{ @@ -63,31 +170,31 @@ func (svc *service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) { CreatedAt: &now, // FIXME: user agent for analytics? } - err = svc.db.Create(&download).Error + err := svc.db.Create(&download).Error if err != nil { - svc.logger.Warn("add download entry", zap.Error(err)) + svc.logger.Warn("failed to add download log entry", zap.Error(err)) } if svc.artifactsCachePath != "" { // send cache f, err := os.Open(cache) if err != nil { - httpError(w, err, codes.Internal) - return + return err } defer f.Close() _, err = io.Copy(w, f) if err != nil { - httpError(w, err, codes.Internal) + return err } } else { // proxy ctx := context.Background() err = svc.artifactDownloadFromProvider(ctx, &artifact, w) if err != nil { - httpError(w, err, codes.Internal) + return err } } + return nil } func (svc *service) artifactDownloadToFile(artifact *yolopb.Artifact, dest string) error { diff --git a/go/pkg/yolosvc/service.go b/go/pkg/yolosvc/service.go index a0b708e8..b0278d3c 100644 --- a/go/pkg/yolosvc/service.go +++ b/go/pkg/yolosvc/service.go @@ -14,6 +14,7 @@ import ( circleci "github.com/jszwedko/go-circleci" "github.com/tevino/abool" "go.uber.org/zap" + "moul.io/u" ) type Service interface { @@ -43,6 +44,9 @@ type service struct { clearCache *abool.AtomicBool artifactsCachePath string artifactsCacheMutex sync.Mutex + iosPrivkeyPath string + iosProvPath string + iosPrivkeyPass string } type ServiceOpts struct { @@ -55,6 +59,9 @@ type ServiceOpts struct { DevMode bool ClearCache *abool.AtomicBool ArtifactsCachePath string + IOSPrivkeyPath string + IOSProvPath string + IOSPrivkeyPass string } func NewService(db *gorm.DB, opts ServiceOpts) (Service, error) { @@ -77,6 +84,9 @@ func NewService(db *gorm.DB, opts ServiceOpts) (Service, error) { devMode: opts.DevMode, clearCache: opts.ClearCache, artifactsCachePath: opts.ArtifactsCachePath, + iosPrivkeyPath: u.MustExpandUser(opts.IOSPrivkeyPath), + iosProvPath: u.MustExpandUser(opts.IOSProvPath), + iosPrivkeyPass: opts.IOSPrivkeyPass, }, nil } diff --git a/go/pkg/yolosvc/util.go b/go/pkg/yolosvc/util.go index 0a101aba..3c0b4189 100644 --- a/go/pkg/yolosvc/util.go +++ b/go/pkg/yolosvc/util.go @@ -17,7 +17,7 @@ var ( func artifactKindByPath(path string) yolopb.Artifact_Kind { switch filepath.Ext(path) { - case ".ipa": + case ".ipa", ".unsigned-ipa", ".dummy-signed-ipa": return yolopb.Artifact_IPA case ".dmg": return yolopb.Artifact_DMG @@ -29,7 +29,7 @@ func artifactKindByPath(path string) yolopb.Artifact_Kind { func mimetypeByPath(path string) string { switch filepath.Ext(path) { - case ".ipa": + case ".ipa", ".unsigned-ipa", ".dummy-signed-ipa": return "application/octet-stream" case ".apk": return "application/vnd.android.package-archive"