Update/v7.4.0

.github/workflows/ci.yml

@@ -1,62 +1,62 @@
 name: CI
 on: [push, pull_request]
 env:
-  go-version: '1.17.x'
+  go-version: "1.18.x"
 jobs:
   test:
     name: Test
     strategy:
       matrix:
-        pg-version: ['12', '13']
+        pg-version: ["12", "13"]
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v1
+      - name: Checkout code
+        uses: actions/checkout@v1
 
-    - name: Install PostgreSQL
-      uses: harmon758/postgresql-action@v1
-      with:
-        postgresql version: ${{ matrix.pg-version }}
-        postgresql db: archiver_test
-        postgresql user: temba
-        postgresql password: temba
+      - name: Install PostgreSQL
+        uses: harmon758/postgresql-action@v1
+        with:
+          postgresql version: ${{ matrix.pg-version }}
+          postgresql db: archiver_test
+          postgresql user: temba
+          postgresql password: temba
 
-    - name: Install Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: ${{ env.go-version }}
+      - name: Install Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: ${{ env.go-version }}
 
-    - name: Run tests
-      run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./...
-      env:
-        ARCHIVER_AWS_ACCESS_KEY_ID: ${{ secrets.ARCHIVER_AWS_ACCESS_KEY_ID }}
-        ARCHIVER_AWS_SECRET_ACCESS_KEY: ${{ secrets.ARCHIVER_AWS_SECRET_ACCESS_KEY }}
+      - name: Run tests
+        run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./...
+        env:
+          ARCHIVER_AWS_ACCESS_KEY_ID: ${{ secrets.ARCHIVER_AWS_ACCESS_KEY_ID }}
+          ARCHIVER_AWS_SECRET_ACCESS_KEY: ${{ secrets.ARCHIVER_AWS_SECRET_ACCESS_KEY }}
+
+      - name: Upload coverage
+        if: success()
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
 
-    - name: Upload coverage
-      if: success()
-      uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-
   release:
     name: Release
     needs: [test]
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v1
+      - name: Checkout code
+        uses: actions/checkout@v1
 
-    - name: Install Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: ${{ env.go-version }}
+      - name: Install Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: ${{ env.go-version }}
 
-    - name: Publish release
-      uses: goreleaser/goreleaser-action@v1
-      with:
-        version: v0.147.2
-        args: release --rm-dist
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        fail_ci_if_error: true
+      - name: Publish release
+        uses: goreleaser/goreleaser-action@v1
+        with:
+          version: v0.147.2
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          fail_ci_if_error: true

.gitignore

@@ -1,5 +1,4 @@
 # Binaries for programs and plugins
-rp-archiver
 *.exe
 *.exe~
 *.dll

CHANGELOG.md

@@ -1,3 +1,70 @@
+v7.4.0
+----------
+ * Include rollups in monthlies failed metric as well as monthlies created from scratch
+
+v7.3.7
+----------
+ * Change query used to update rollup_id on dailies
+ * Remove temporary logging
+
+v7.3.6
+----------
+ * Add temporary additional logging
+ * Replace ExitOnCompletion config option with Once which makes it run once and exit
+
+v7.3.5
+----------
+ * Improve librato analytics and add tests
+
+v7.3.4
+----------
+ * Rework stats reporting
+ * Log version at startup
+
+v7.3.3
+----------
+ * Fix parsing start times after midday
+
+v7.3.2
+----------
+ * Don't log entire run JSON on error, just UUID
+ * Make archival happen at configured start time even on first pass
+
+v7.3.1
+----------
+ * Add librato analytics for time elapsed and number of orgs, msgs and runs
+
+v7.3.0
+----------
+ * Update to go 1.18 and upgrade dependencies
+ * Add support for Msg.visibility=X (deleted by sender)
+ * Add arm64 as a build target
+
+v7.2.0
+----------
+ * Tweak README
+
+v7.1.6
+----------
+ * Stop setting delete_reason on runs before deletion
+
+v7.1.5
+----------
+ * Stop updating msgs_msg.delete_reason which is no longer needed
+
+v7.1.4
+----------
+ * Record flow on msgs
+
+v7.1.3
+----------
+ * Remove deletion of recent runs as these are no longer created
+
+v7.1.2
+----------
+ * Use run status instead of is_active and exit_type
+ * No longer include events in run archives
+
 v7.1.1
 ----------
  * Remove references to flowrun.parent_id which is no longer set by mailroom

README.md

@@ -8,16 +8,17 @@ RP-Archiver is the [RapidPro](https://github.com/rapidpro/rapidpro) component re
 old runs and messages. It interacts directly with the RapidPro database and writes archive files to an 
 S3 compatible endpoint.
 
-# Deploying
+## Deploying
 
 As Archiver is a Go application, it compiles to a binary and that binary along with the config file is all
 you need to run it on your server. You can find bundles for each platform in the
 [releases directory](https://github.com/nyaruka/rp-archiver/releases). You should only run a single archiver
 instance for a deployment.
 
-# Configuration
+## Configuration
 
 Archiver uses a tiered configuration system, each option takes precendence over the ones above it:
+
  1. The configuration file
  2. Environment variables starting with `ARCHIVER_` 
  3. Command line parameters
@@ -26,7 +27,7 @@ We recommend running Archiver with no changes to the configuration and no parame
 environment variables to configure it. You can use `% rp-archiver --help` to see a list of the
 environment variables and parameters and for more details on each option.
 
-# RapidPro Configuration
+### RapidPro Configuration
 
 For use with RapidPro, you will want to configure these settings:
 
@@ -46,7 +47,7 @@ Recommended settings for error reporting:
 
  * `ARCHIVER_SENTRY_DSN`: The DSN to use when logging errors to Sentry
 
-# Development
+## Development
 
 Once you've checked out the code, you can build Archiver with:
 
@@ -65,70 +66,58 @@ $ createdb archiver_test
 To run all of the tests:
 
 ```
-go test ./... -p=1
+go test -p=1 ./...
 ```
 
-## Usage
+## Reference
 
-```
-Archives RapidPro runs and msgs to S3
+These are the configuration options that can be provided as parameters or environment variables. If using environment 
+varibles, convert to uppercase, replace dashes with underscores and prefix the name with `ARCHIVER_`, e.g. `-archive-messages` 
+becomes `ARCHIVER_ARCHIVE_MESSAGES`.
 
-Usage of archiver:
+```
   -archive-messages
-    	whether we should archive messages (default true)
+      whether we should archive messages (default true)
   -archive-runs
-    	whether we should archive runs (default true)
+      whether we should archive runs (default true)
   -aws-access-key-id string
-    	the access key id to use when authenticating S3 (default "missing_aws_access_key_id")
+      the access key id to use when authenticating S3 (default "missing_aws_access_key_id")
   -aws-secret-access-key string
-    	the secret access key id to use when authenticating S3 (default "missing_aws_secret_access_key")
+      the secret access key id to use when authenticating S3 (default "missing_aws_secret_access_key")
   -db string
-    	the connection string for our database (default "postgres://localhost/archiver_test?sslmode=disable")
+      the connection string for our database (default "postgres://localhost/archiver_test?sslmode=disable")
   -debug-conf
-    	print where config values are coming from
+      print where config values are coming from
   -delete
-    	whether to delete messages and runs from the db after archival (default false)
+      whether to delete messages and runs from the db after archival (default false)
   -help
-    	print usage information
+      print usage information
   -keep-files
-    	whether we should keep local archive files after upload (default false)
+      whether we should keep local archive files after upload (default false)
+  -librato-username
+      the Librato username for metrics reporting
+  -librato-token
+      the Librato token for metrics reporting
   -log-level string
-    	the log level, one of error, warn, info, debug (default "info")
+      the log level, one of error, warn, info, debug (default "info")
+  -once
+      run archving immediately and then exit
   -retention-period int
-    	the number of days to keep before archiving (default 90)
+      the number of days to keep before archiving (default 90)
   -s3-bucket string
-    	the S3 bucket we will write archives to (default "dl-archiver-test")
+      the S3 bucket we will write archives to (default "dl-archiver-test")
   -s3-disable-ssl
-    	whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network
+      whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network
   -s3-endpoint string
-    	the S3 endpoint we will write archives to (default "https://s3.amazonaws.com")
+      the S3 endpoint we will write archives to (default "https://s3.amazonaws.com")
   -s3-force-path-style
-    	whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service
+      whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service
   -s3-region string
-    	the S3 region we will write archives to (default "us-east-1")
+      the S3 region we will write archives to (default "us-east-1")
   -sentry-dsn string
-    	the sentry configuration to log errors to, if any
+      the sentry configuration to log errors to, if any
   -temp-dir string
-    	directory where temporary archive files are written (default "/tmp")
+      directory where temporary archive files are written (default "/tmp")
   -upload-to-s3
-    	whether we should upload archive to S3 (default true)
-
-Environment variables:
-                   ARCHIVER_ARCHIVE_MESSAGES - bool
-                       ARCHIVER_ARCHIVE_RUNS - bool
-                  ARCHIVER_AWS_ACCESS_KEY_ID - string
-              ARCHIVER_AWS_SECRET_ACCESS_KEY - string
-                                 ARCHIVER_DB - string
-                             ARCHIVER_DELETE - bool
-                         ARCHIVER_KEEP_FILES - bool
-                          ARCHIVER_LOG_LEVEL - string
-                   ARCHIVER_RETENTION_PERIOD - int
-                          ARCHIVER_S3_BUCKET - string
-                     ARCHIVER_S3_DISABLE_SSL - bool
-                        ARCHIVER_S3_ENDPOINT - string
-                ARCHIVER_S3_FORCE_PATH_STYLE - bool
-                          ARCHIVER_S3_REGION - string
-                         ARCHIVER_SENTRY_DSN - string
-                           ARCHIVER_TEMP_DIR - string
-                       ARCHIVER_UPLOAD_TO_S3 - bool
+      whether we should upload archive to S3 (default true)
 ```

WENI-CHANGELOG.md

@@ -1,3 +1,7 @@
+1.2.3-archiver-7.4.0
+----------
+ * Update to rp-archiver v7.4.0
+
 1.2.2-archiver-7.1.1
 ----------
  * No longer consider retention time on delete operation