From dfbc3562d0f73ec7f47eb6806dc45c0f59a2e0b6 Mon Sep 17 00:00:00 2001 From: Tyler Britten <1933680+tybritten@users.noreply.github.com> Date: Tue, 7 Jan 2020 19:29:42 -0500 Subject: [PATCH 1/3] Add configurable archive length --- archiver.go | 4 ++-- cmd/rp-archiver/main.go | 2 +- config.go | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/archiver.go b/archiver.go index a97549d..b2504e6 100644 --- a/archiver.go +++ b/archiver.go @@ -105,7 +105,7 @@ WHERE o.is_active = TRUE order by o.id ` // GetActiveOrgs returns the active organizations sorted by id -func GetActiveOrgs(ctx context.Context, db *sqlx.DB) ([]Org, error) { +func GetActiveOrgs(ctx context.Context, db *sqlx.DB, conf *Config) ([]Org, error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() @@ -117,7 +117,7 @@ func GetActiveOrgs(ctx context.Context, db *sqlx.DB) ([]Org, error) { orgs := make([]Org, 0, 10) for rows.Next() { - org := Org{ActiveDays: 90} + org := Org{ActiveDays: conf.ArchiveLength} err = rows.StructScan(&org) if err != nil { return nil, errors.Wrapf(err, "error scanning active org") diff --git a/cmd/rp-archiver/main.go b/cmd/rp-archiver/main.go index a6e3693..eebad4e 100644 --- a/cmd/rp-archiver/main.go +++ b/cmd/rp-archiver/main.go @@ -84,7 +84,7 @@ func main() { // get our active orgs ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - orgs, err := archiver.GetActiveOrgs(ctx, db) + orgs, err := archiver.GetActiveOrgs(ctx, db, config) cancel() if err != nil { diff --git a/config.go b/config.go index 061b279..1f98994 100644 --- a/config.go +++ b/config.go @@ -21,6 +21,7 @@ type Config struct { ArchiveMessages bool `help:"whether we should archive messages"` ArchiveRuns bool `help:"whether we should archive runs"` + ArchiveLength int `help:"how many days back to archive"` Delete bool `help:"whether to delete messages and runs from the db after archival (default false)"` } @@ -45,6 +46,7 @@ func NewConfig() *Config { ArchiveMessages: true, ArchiveRuns: true, + ArchiveLength: 90, Delete: false, } From 20c83615ee06dadd819719cb5828b0fc76a8e1ba Mon Sep 17 00:00:00 2001 From: Tyler Britten <1933680+tybritten@users.noreply.github.com> Date: Wed, 8 Jan 2020 11:34:20 -0500 Subject: [PATCH 2/3] updated tests --- archiver_test.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/archiver_test.go b/archiver_test.go index 334b6b4..d3225cf 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -34,7 +34,8 @@ func TestGetMissingDayArchives(t *testing.T) { // get the tasks for our org ctx := context.Background() - orgs, err := GetActiveOrgs(ctx, db) + config := NewConfig() + orgs, err := GetActiveOrgs(ctx, db, config) assert.NoError(t, err) now := time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC) @@ -65,7 +66,8 @@ func TestGetMissingMonthArchives(t *testing.T) { // get the tasks for our org ctx := context.Background() - orgs, err := GetActiveOrgs(ctx, db) + config := NewConfig() + orgs, err := GetActiveOrgs(ctx, db, config) assert.NoError(t, err) now := time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC) @@ -96,7 +98,8 @@ func TestCreateMsgArchive(t *testing.T) { err := EnsureTempArchiveDirectory("/tmp") assert.NoError(t, err) - orgs, err := GetActiveOrgs(ctx, db) + config := NewConfig() + orgs, err := GetActiveOrgs(ctx, db, config) assert.NoError(t, err) now := time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC) @@ -172,7 +175,8 @@ func TestCreateRunArchive(t *testing.T) { err := EnsureTempArchiveDirectory("/tmp") assert.NoError(t, err) - orgs, err := GetActiveOrgs(ctx, db) + config := NewConfig() + orgs, err := GetActiveOrgs(ctx, db, config) assert.NoError(t, err) now := time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC) @@ -228,7 +232,8 @@ func TestWriteArchiveToDB(t *testing.T) { db := setup(t) ctx := context.Background() - orgs, err := GetActiveOrgs(ctx, db) + config := NewConfig() + orgs, err := GetActiveOrgs(ctx, db, config) assert.NoError(t, err) now := time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC) @@ -281,11 +286,11 @@ func TestArchiveOrgMessages(t *testing.T) { ctx := context.Background() deleteTransactionSize = 1 - orgs, err := GetActiveOrgs(ctx, db) + config := NewConfig() + orgs, err := GetActiveOrgs(ctx, db, config) assert.NoError(t, err) now := time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC) - config := NewConfig() os.Args = []string{"rp-archiver"} loader := ezconf.NewLoader(&config, "archiver", "Archives RapidPro runs and msgs to S3", nil) @@ -422,11 +427,11 @@ func TestArchiveOrgRuns(t *testing.T) { db := setup(t) ctx := context.Background() - orgs, err := GetActiveOrgs(ctx, db) + config := NewConfig() + orgs, err := GetActiveOrgs(ctx, db, config) assert.NoError(t, err) now := time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC) - config := NewConfig() os.Args = []string{"rp-archiver"} loader := ezconf.NewLoader(&config, "archiver", "Archives RapidPro runs and msgs to S3", nil) From d2cdc550a376aa55a5b99035a405667a8d6cbdce Mon Sep 17 00:00:00 2001 From: Tyler Britten <1933680+tybritten@users.noreply.github.com> Date: Wed, 8 Jan 2020 11:58:36 -0500 Subject: [PATCH 3/3] added tests for changing the archivelength --- archiver_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/archiver_test.go b/archiver_test.go index d3225cf..1ea744d 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -59,6 +59,22 @@ func TestGetMissingDayArchives(t *testing.T) { assert.Equal(t, time.Date(2017, 8, 11, 0, 0, 0, 0, time.UTC), tasks[0].StartDate) assert.Equal(t, time.Date(2017, 10, 1, 0, 0, 0, 0, time.UTC), tasks[21].StartDate) assert.Equal(t, time.Date(2017, 10, 10, 0, 0, 0, 0, time.UTC), tasks[30].StartDate) + + // org 3 again, but changing the archive period so we have no tasks + orgs[2].ActiveDays = 200 + tasks, err = GetMissingDailyArchives(ctx, db, now, orgs[2], MessageType) + assert.NoError(t, err) + assert.Equal(t, 0, len(tasks)) + + // org 1 again, but lowering the archive period so we have tasks + orgs[0].ActiveDays = 2 + tasks, err = GetMissingDailyArchives(ctx, db, now, orgs[0], MessageType) + assert.NoError(t, err) + assert.Equal(t, 58, len(tasks)) + assert.Equal(t, time.Date(2017, 11, 10, 0, 0, 0, 0, time.UTC), tasks[0].StartDate) + assert.Equal(t, time.Date(2017, 12, 1, 0, 0, 0, 0, time.UTC), tasks[21].StartDate) + assert.Equal(t, time.Date(2017, 12, 10, 0, 0, 0, 0, time.UTC), tasks[30].StartDate) + } func TestGetMissingMonthArchives(t *testing.T) { @@ -89,6 +105,7 @@ func TestGetMissingMonthArchives(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 1, len(tasks)) assert.Equal(t, time.Date(2017, 8, 1, 0, 0, 0, 0, time.UTC), tasks[0].StartDate) + } func TestCreateMsgArchive(t *testing.T) {