diff --git a/changelog/28064.txt b/changelog/28064.txt new file mode 100644 index 000000000000..6f18843cad3a --- /dev/null +++ b/changelog/28064.txt @@ -0,0 +1,7 @@ +```release-note:improvement +activity: The [activity export API](https://developer.hashicorp.com/vault/api-docs/system/internal-counters#activity-export) now includes identity metadata about entity clients. +``` + +```release-note:change +activity: The [activity export API](https://developer.hashicorp.com/vault/api-docs/system/internal-counters#activity-export) now responds with a status of 204 instead 400 when no data exists within the time range specified by `start_time` and `end_time`. +``` diff --git a/tools/semgrep/ci/no-nil-check.yml b/tools/semgrep/ci/no-nil-check.yml index 00c64c3641bb..a470f35c87fb 100644 --- a/tools/semgrep/ci/no-nil-check.yml +++ b/tools/semgrep/ci/no-nil-check.yml @@ -99,6 +99,13 @@ rules: ... } ... + - pattern-not: | + $VAR, $ERR = NamespaceByID(...) + ... + if !a.includeInResponse(..., $VAR) { + ... + } + ... message: missed nil check languages: - go diff --git a/vault/activity/test_fixtures/aug.csv b/vault/activity/test_fixtures/aug.csv index 66b63c563955..0f95fa07e2af 100644 --- a/vault/activity/test_fixtures/aug.csv +++ b/vault/activity/test_fixtures/aug.csv @@ -1,21 +1,21 @@ -client_id,client_type,namespace_id,namespace_path,mount_accessor,timestamp -111122222-3333-4444-5555-000000000000,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000001,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000002,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000003,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000004,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000005,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000006,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000007,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000008,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000009,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000010,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000011,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000012,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000013,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000014,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000015,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000016,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000017,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000018,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000019,entity,root,,auth_4,"1970-01-01T00:00:02Z" +identity_name,alias_name,client_id,client_type,namespace_id,namespace_path,mount_accessor,mount_path,mount_type,timestamp +,,111122222-3333-4444-5555-000000000000,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000001,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000002,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000003,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000004,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000005,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000006,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000007,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000008,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000009,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000010,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000011,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000012,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000013,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000014,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000015,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000016,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000017,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000018,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000019,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" diff --git a/vault/activity/test_fixtures/aug_oct.csv b/vault/activity/test_fixtures/aug_oct.csv index 0df8ba3ac3c5..12bbb69e1cae 100644 --- a/vault/activity/test_fixtures/aug_oct.csv +++ b/vault/activity/test_fixtures/aug_oct.csv @@ -1,41 +1,41 @@ -client_id,client_type,namespace_id,namespace_path,mount_accessor,timestamp -111122222-3333-4444-5555-000000000000,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000001,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000002,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000003,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000004,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000005,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000006,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000007,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000008,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000009,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000010,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000011,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000012,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000013,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000014,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000015,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000016,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000017,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000018,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000019,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000020,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000021,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000022,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000023,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000024,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000025,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000026,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000027,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000028,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000029,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000030,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000031,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000032,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000033,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000034,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000035,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000036,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000037,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000038,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000039,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" +identity_name,alias_name,client_id,client_type,namespace_id,namespace_path,mount_accessor,mount_path,mount_type,timestamp +,,111122222-3333-4444-5555-000000000000,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000001,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000002,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000003,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000004,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000005,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000006,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000007,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000008,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000009,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000010,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000011,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000012,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000013,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000014,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000015,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000016,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000017,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000018,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000019,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000020,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000021,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000022,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000023,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000024,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000025,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000026,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000027,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000028,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000029,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000030,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000031,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000032,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000033,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000034,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000035,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000036,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000037,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000038,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000039,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" diff --git a/vault/activity/test_fixtures/aug_sep.csv b/vault/activity/test_fixtures/aug_sep.csv index 5ee2b4829739..6f4f682877b3 100644 --- a/vault/activity/test_fixtures/aug_sep.csv +++ b/vault/activity/test_fixtures/aug_sep.csv @@ -1,31 +1,31 @@ -client_id,client_type,namespace_id,namespace_path,mount_accessor,timestamp -111122222-3333-4444-5555-000000000000,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000001,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000002,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000003,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000004,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000005,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000006,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000007,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000008,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000009,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000010,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000011,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000012,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000013,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000014,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000015,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000016,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000017,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000018,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000019,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000020,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000021,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000022,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000023,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000024,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000025,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000026,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000027,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000028,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000029,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" +identity_name,alias_name,client_id,client_type,namespace_id,namespace_path,mount_accessor,mount_path,mount_type,timestamp +,,111122222-3333-4444-5555-000000000000,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000001,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000002,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000003,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000004,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000005,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000006,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000007,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000008,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000009,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000010,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000011,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000012,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000013,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000014,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000015,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000016,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000017,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000018,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000019,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000020,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000021,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000022,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000023,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000024,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000025,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000026,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000027,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000028,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000029,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" diff --git a/vault/activity/test_fixtures/aug_sep.json b/vault/activity/test_fixtures/aug_sep.json index 46f0e016c241..181e562e40c7 100644 --- a/vault/activity/test_fixtures/aug_sep.json +++ b/vault/activity/test_fixtures/aug_sep.json @@ -1,30 +1,30 @@ -{"client_id":"111122222-3333-4444-5555-000000000000","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1"} -{"client_id":"111122222-3333-4444-5555-000000000001","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1"} -{"client_id":"111122222-3333-4444-5555-000000000002","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1"} -{"client_id":"111122222-3333-4444-5555-000000000003","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1"} -{"client_id":"111122222-3333-4444-5555-000000000004","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1"} -{"client_id":"111122222-3333-4444-5555-000000000005","client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2"} -{"client_id":"111122222-3333-4444-5555-000000000006","client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2"} -{"client_id":"111122222-3333-4444-5555-000000000007","client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2"} -{"client_id":"111122222-3333-4444-5555-000000000008","client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2"} -{"client_id":"111122222-3333-4444-5555-000000000009","client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2"} -{"client_id":"111122222-3333-4444-5555-000000000010","client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_3"} -{"client_id":"111122222-3333-4444-5555-000000000011","client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_3"} -{"client_id":"111122222-3333-4444-5555-000000000012","client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_3"} -{"client_id":"111122222-3333-4444-5555-000000000013","client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_3"} -{"client_id":"111122222-3333-4444-5555-000000000014","client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_3"} -{"client_id":"111122222-3333-4444-5555-000000000015","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4"} -{"client_id":"111122222-3333-4444-5555-000000000016","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4"} -{"client_id":"111122222-3333-4444-5555-000000000017","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4"} -{"client_id":"111122222-3333-4444-5555-000000000018","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4"} -{"client_id":"111122222-3333-4444-5555-000000000019","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4"} -{"client_id":"111122222-3333-4444-5555-000000000020","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5"} -{"client_id":"111122222-3333-4444-5555-000000000021","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5"} -{"client_id":"111122222-3333-4444-5555-000000000022","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5"} -{"client_id":"111122222-3333-4444-5555-000000000023","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5"} -{"client_id":"111122222-3333-4444-5555-000000000024","client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5"} -{"client_id":"111122222-3333-4444-5555-000000000025","client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6"} -{"client_id":"111122222-3333-4444-5555-000000000026","client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6"} -{"client_id":"111122222-3333-4444-5555-000000000027","client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6"} -{"client_id":"111122222-3333-4444-5555-000000000028","client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6"} -{"client_id":"111122222-3333-4444-5555-000000000029","client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6"} +{"client_id":"111122222-3333-4444-5555-000000000000","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000001","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000002","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000003","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000004","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_1","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000005","non_entity":false,"client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000006","non_entity":false,"client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000007","non_entity":false,"client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000008","non_entity":false,"client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000009","non_entity":false,"client_type":"entity","namespace_path":"aaaaa/","namespace_id":"aaaaa","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_2","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000010","non_entity":false,"client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_3","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000011","non_entity":false,"client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_3","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000012","non_entity":false,"client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:01Z","mount_accessor":"auth_3","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000013","non_entity":false,"client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_3","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000014","non_entity":false,"client_type":"entity","namespace_path":"bbbbb/","namespace_id":"bbbbb","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_3","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000015","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000016","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000017","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000018","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000019","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:02Z","mount_accessor":"auth_4","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000020","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000021","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000022","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000023","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000024","non_entity":false,"client_type":"entity","namespace_path":"","namespace_id":"root","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_5","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000025","non_entity":false,"client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000026","non_entity":false,"client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000027","non_entity":false,"client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000028","non_entity":false,"client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} +{"client_id":"111122222-3333-4444-5555-000000000029","non_entity":false,"client_type":"entity","namespace_path":"ccccc/","namespace_id":"ccccc","timestamp":"1970-01-01T00:00:03Z","mount_accessor":"auth_6","identity_name":"","alias_name":"","mount_type":"","mount_path":"","policies":null,"identity_group_ids":null,"identity_metadata":null,"alias_metadata":null,"alias_custom_metadata":null} diff --git a/vault/activity/test_fixtures/full_history.csv b/vault/activity/test_fixtures/full_history.csv index 2cb1ba91fa77..b04c5a56fbc3 100644 --- a/vault/activity/test_fixtures/full_history.csv +++ b/vault/activity/test_fixtures/full_history.csv @@ -1,46 +1,46 @@ -client_id,client_type,namespace_id,namespace_path,mount_accessor,timestamp -111122222-3333-4444-5555-000000000040,entity,rrrrr,rrrrr/,auth_9,"1970-01-01T00:00:00Z" -111122222-3333-4444-5555-000000000041,entity,rrrrr,rrrrr/,auth_9,"1970-01-01T00:00:00Z" -111122222-3333-4444-5555-000000000042,entity,rrrrr,rrrrr/,auth_9,"1970-01-01T00:00:00Z" -111122222-3333-4444-5555-000000000043,entity,rrrrr,rrrrr/,auth_9,"1970-01-01T00:00:00Z" -111122222-3333-4444-5555-000000000044,entity,rrrrr,rrrrr/,auth_9,"1970-01-01T00:00:00Z" -111122222-3333-4444-5555-000000000000,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000001,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000002,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000003,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000004,entity,root,,auth_1,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000005,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000006,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000007,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000008,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000009,entity,aaaaa,aaaaa/,auth_2,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000010,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000011,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000012,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:01Z" -111122222-3333-4444-5555-000000000013,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000014,entity,bbbbb,bbbbb/,auth_3,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000015,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000016,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000017,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000018,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000019,entity,root,,auth_4,"1970-01-01T00:00:02Z" -111122222-3333-4444-5555-000000000020,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000021,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000022,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000023,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000024,entity,root,,auth_5,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000025,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000026,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000027,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000028,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000029,entity,ccccc,ccccc/,auth_6,"1970-01-01T00:00:03Z" -111122222-3333-4444-5555-000000000030,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000031,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000032,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000033,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000034,entity,root,,auth_7,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000035,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000036,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000037,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000038,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" -111122222-3333-4444-5555-000000000039,entity,bbbbb,bbbbb/,auth_8,"1970-01-01T00:00:04Z" +identity_name,alias_name,client_id,client_type,namespace_id,namespace_path,mount_accessor,mount_path,mount_type,timestamp +,,111122222-3333-4444-5555-000000000040,entity,rrrrr,rrrrr/,auth_9,,,"1970-01-01T00:00:00Z" +,,111122222-3333-4444-5555-000000000041,entity,rrrrr,rrrrr/,auth_9,,,"1970-01-01T00:00:00Z" +,,111122222-3333-4444-5555-000000000042,entity,rrrrr,rrrrr/,auth_9,,,"1970-01-01T00:00:00Z" +,,111122222-3333-4444-5555-000000000043,entity,rrrrr,rrrrr/,auth_9,,,"1970-01-01T00:00:00Z" +,,111122222-3333-4444-5555-000000000044,entity,rrrrr,rrrrr/,auth_9,,,"1970-01-01T00:00:00Z" +,,111122222-3333-4444-5555-000000000000,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000001,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000002,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000003,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000004,entity,root,,auth_1,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000005,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000006,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000007,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000008,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000009,entity,aaaaa,aaaaa/,auth_2,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000010,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000011,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000012,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:01Z" +,,111122222-3333-4444-5555-000000000013,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000014,entity,bbbbb,bbbbb/,auth_3,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000015,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000016,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000017,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000018,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000019,entity,root,,auth_4,,,"1970-01-01T00:00:02Z" +,,111122222-3333-4444-5555-000000000020,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000021,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000022,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000023,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000024,entity,root,,auth_5,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000025,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000026,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000027,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000028,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000029,entity,ccccc,ccccc/,auth_6,,,"1970-01-01T00:00:03Z" +,,111122222-3333-4444-5555-000000000030,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000031,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000032,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000033,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000034,entity,root,,auth_7,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000035,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000036,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000037,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000038,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" +,,111122222-3333-4444-5555-000000000039,entity,bbbbb,bbbbb/,auth_8,,,"1970-01-01T00:00:04Z" diff --git a/vault/activity_log.go b/vault/activity_log.go index 45fa2f3fc398..ba3dc8de9ee3 100644 --- a/vault/activity_log.go +++ b/vault/activity_log.go @@ -12,6 +12,7 @@ import ( "io" "net/http" "os" + "slices" "sort" "strconv" "strings" @@ -22,11 +23,13 @@ import ( "github.com/axiomhq/hyperloglog" "github.com/golang/protobuf/proto" log "github.com/hashicorp/go-hclog" + lru "github.com/hashicorp/golang-lru" "github.com/hashicorp/vault/helper/metricsutil" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/helper/timeutil" "github.com/hashicorp/vault/sdk/logical" "github.com/hashicorp/vault/vault/activity" + "github.com/mitchellh/mapstructure" "go.uber.org/atomic" ) @@ -100,6 +103,12 @@ const ( // ActivityExportInvalidFormatPrefix is used to check validation errors for the // activity log export API handler ActivityExportInvalidFormatPrefix = "invalid format" + + // exportCSVFlatteningInitIndex is used within the activity export API when the "csv" + // format is requested. Map and slice values will be flattened and accumulated by the + // CSV encoder. Indexes will be generated to ensure that values are slotted into the + // correct column. This initial value is used prior to finalizing the CSV header. + exportCSVFlatteningInitIndex = -1 ) var ( @@ -108,10 +117,6 @@ var ( // ErrActivityExportInProgress is used to check validation errors for the // activity log export API handler ErrActivityExportInProgress = errors.New("existing export in progress") - - // ErrActivityExportNoDataInRange is used to check validation errors for the - // activity log export API handler - ErrActivityExportNoDataInRange = errors.New("no data to export in provided time range") ) type segmentInfo struct { @@ -232,21 +237,56 @@ type ActivityLogCoreConfig struct { } // ActivityLogExportRecord is the output structure for activity export -// API records. The omitempty JSON tag is not used to ensure that the -// fields are consistent between CSV and JSON output. +// API records. The fields below are all associated with the token used to +// perform the logged activity. The omitempty JSON tag is not used to ensure +// that the fields are consistent between CSV and JSON output. type ActivityLogExportRecord struct { - ClientID string `json:"client_id" mapstructure:"client_id"` - NamespaceID string `json:"namespace_id" mapstructure:"namespace_id"` + // IdentityName is the name of the entity + IdentityName string `json:"identity_name" mapstructure:"identity_name"` + + // AliasName is the entity alias name provided by the auth backend upon login + AliasName string `json:"alias_name" mapstructure:"alias_name"` + + // ClientID is the unique identifier assigned to the entity that performed the activity + ClientID string `json:"client_id" mapstructure:"client_id"` + + // ClientType identifies the source of the entity record (entity, non-entity, acme, etc.) + ClientType string `json:"client_type" mapstructure:"client_type"` + + // NamespaceID is the identifier of the namespace in which the associated auth backend resides + NamespaceID string `json:"namespace_id" mapstructure:"namespace_id"` + + // NamespacePath is the path of the namespace in which the associated auth backend resides NamespacePath string `json:"namespace_path" mapstructure:"namespace_path"` - Timestamp string `json:"timestamp" mapstructure:"timestamp"` - // MountAccessor is the auth mount accessor of the token used to perform the - // activity. + // MountAccessor is the auth mount accessor associated with the token used MountAccessor string `json:"mount_accessor" mapstructure:"mount_accessor"` - // ClientType identifies the source of the entity record (entity, - // non-entity, acme, etc.) - ClientType string `json:"client_type" mapstructure:"client_type"` + // MountType is the type of the auth mount associated with the token used + MountType string `json:"mount_type" mapstructure:"mount_type"` + + // MountPath is the path of the auth mount associated with the token used + MountPath string `json:"mount_path" mapstructure:"mount_path"` + + // Timestamp denotes the time at which the activity occurred formatted using RFC3339 + Timestamp string `json:"timestamp" mapstructure:"timestamp"` + + // Policies are the list of policy names attached to the token used + Policies []string `json:"policies" mapstructure:"policies"` + + // IdentityMetadata represents explicit metadata set by clients. Multiple entities can have the + // same metadata which enables virtual groupings of entities. + IdentityMetadata map[string]string `json:"identity_metadata" mapstructure:"identity_metadata"` + + // AliasMetadata represents the metadata associated with the identity alias. Multiple aliases can + // have the same custom metadata which enables virtual grouping of aliases. + AliasMetadata map[string]string `json:"alias_metadata" mapstructure:"alias_metadata"` + + // AliasCustomMetadata represents the custom metadata associated with the identity alias + AliasCustomMetadata map[string]string `json:"alias_custom_metadata" mapstructure:"alias_custom_metadata"` + + // IdentityGroupIDs provides a list of all of the identity group IDs in which an entity belongs + IdentityGroupIDs []string `json:"identity_group_ids" mapstructure:"identity_group_ids"` } // NewActivityLog creates an activity log. @@ -2950,7 +2990,7 @@ func (a *ActivityLog) writeExport(ctx context.Context, rw http.ResponseWriter, f } if len(filteredList) == 0 { a.logger.Info("no data to export", "start_time", startTime, "end_time", endTime) - return ErrActivityExportNoDataInRange + return nil } actualStartTime := filteredList[len(filteredList)-1] @@ -2979,53 +3019,215 @@ func (a *ActivityLog) writeExport(ctx context.Context, rw http.ResponseWriter, f a.logger.Info("starting activity log export", "start_time", startTime, "end_time", endTime, "format", format) - dedupedIds := make(map[string]struct{}) + dedupIDs := make(map[string]struct{}) reqNS, err := namespace.FromContext(ctx) if err != nil { return err } + // an LRU cache is used to optimistically prevent frequent + // lookup of common identity backends + identityBackendCache, err := lru.New2Q(10) + if err != nil { + return err + } + walkEntities := func(l *activity.EntityActivityLog, startTime time.Time, hll *hyperloglog.Sketch) error { for _, e := range l.Clients { - if _, ok := dedupedIds[e.ClientID]; ok { + if _, ok := dedupIDs[e.ClientID]; ok { continue } - dedupedIds[e.ClientID] = struct{}{} + dedupIDs[e.ClientID] = struct{}{} ns, err := NamespaceByID(ctx, e.NamespaceID, a.core) if err != nil { return err } - if a.includeInResponse(reqNS, ns) { - ts := time.Unix(e.Timestamp, 0) + if !a.includeInResponse(reqNS, ns) { + continue + } + + ts := time.Unix(e.Timestamp, 0) + + record := &ActivityLogExportRecord{ + ClientID: e.ClientID, + ClientType: e.ClientType, + NamespaceID: ns.ID, + NamespacePath: ns.Path, + Timestamp: ts.UTC().Format(time.RFC3339), + MountAccessor: e.MountAccessor, + } + + if e.MountAccessor != "" { + cacheKey := ns.Path + mountPathIdentity + + var identityBackend logical.Backend - record := &ActivityLogExportRecord{ - ClientID: e.ClientID, - NamespaceID: ns.ID, - NamespacePath: ns.Path, - Timestamp: ts.UTC().Format(time.RFC3339), - MountAccessor: e.MountAccessor, - ClientType: e.ClientType, + val, ok := identityBackendCache.Get(cacheKey) + if !ok { + identityBackend = a.core.router.MatchingBackend(namespace.ContextWithNamespace(ctx, ns), mountPathIdentity) + + if identityBackend != nil { + identityBackendCache.Add(cacheKey, identityBackend) + } + } else { + identityBackend = val.(logical.Backend) + } + + if identityBackend != nil { + req := &logical.Request{ + Path: "lookup/entity", + Storage: a.core.systemBarrierView, + Operation: logical.UpdateOperation, + Data: map[string]interface{}{ + "id": e.ClientID, + }, + } + + entityResp, err := identityBackend.HandleRequest(namespace.ContextWithNamespace(ctx, ns), req) + if err != nil { + return fmt.Errorf("failed to lookup entity: %w", err) + } + + if entityResp != nil { + record.IdentityName, ok = entityResp.Data["name"].(string) + if !ok { + return fmt.Errorf("failed to process identity name") + } + + record.Policies, ok = entityResp.Data["policies"].([]string) + if !ok { + return fmt.Errorf("failed to process policies") + } + + slices.Sort(record.Policies) + + record.IdentityMetadata, ok = entityResp.Data["metadata"].(map[string]string) + if !ok { + return fmt.Errorf("failed to process identity metadata") + } + + record.IdentityGroupIDs, ok = entityResp.Data["group_ids"].([]string) + if !ok { + return fmt.Errorf("failed to process identity group IDs") + } + + slices.Sort(record.IdentityGroupIDs) + + aliases, ok := entityResp.Data["aliases"].([]interface{}) + if !ok { + return fmt.Errorf("failed to process aliases") + } + + // filter for appropriate identity alias based on the + // mount accessor associated with the EntityRecord + for _, rawAlias := range aliases { + alias, ok := rawAlias.(map[string]interface{}) + + if !ok { + return fmt.Errorf("failed to process alias") + } + + aliasMountAccessor, ok := alias["mount_accessor"].(string) + + if !ok || aliasMountAccessor != e.MountAccessor { + continue + } + + record.AliasName, ok = alias["name"].(string) + if !ok { + return fmt.Errorf("failed to process alias name") + } + + record.MountType, ok = alias["mount_type"].(string) + if !ok { + return fmt.Errorf("failed to process mount type") + } + + record.MountPath, ok = alias["mount_path"].(string) + if !ok { + return fmt.Errorf("failed to process mount path") + } + + record.AliasMetadata, ok = alias["metadata"].(map[string]string) + if !ok { + return fmt.Errorf("failed to process alias metadata") + } + + record.AliasCustomMetadata, ok = alias["custom_metadata"].(map[string]string) + if !ok { + return fmt.Errorf("failed to process alias custom metadata") + } + } + } } + } + // the format is validated above and thus we do not require a default + switch format { + case "json": err := encoder.Encode(record) if err != nil { return err } + case "csv": + csvEnc := encoder.(*csvEncoder) + + if csvEnc.wroteHeader { + err := csvEnc.Encode(record) + if err != nil { + return err + } + } else { + err = csvEnc.accumulateHeaderFields(record) + if err != nil { + return err + } + } } } return nil } - // For each month in the filtered list walk all the log segments - for _, startTime := range filteredList { - err := a.WalkEntitySegments(ctx, startTime, nil, walkEntities) - if err != nil { - a.logger.Error("failed to load segments for export", "error", err) - return fmt.Errorf("failed to load segments for export: %w", err) + // JSON will always walk once, CSV should only walk twice if we've processed + // records during the first pass + shouldWalk := true + + // CSV must perform two passes over the data to generate the column list + if format == "csv" { + for _, startTime := range filteredList { + err := a.WalkEntitySegments(ctx, startTime, nil, walkEntities) + if err != nil { + a.logger.Error("failed to load segments for export", "error", err) + return fmt.Errorf("failed to load segments for export: %w", err) + } + } + + if len(dedupIDs) > 0 { + // only write header if we've seen some records + err = encoder.(*csvEncoder).writeHeader() + if err != nil { + return err + } + + // clear dedupIDs for second pass + dedupIDs = make(map[string]struct{}) + } else { + shouldWalk = false + } + } + + if shouldWalk { + // For each month in the filtered list walk all the log segments + for _, startTime := range filteredList { + err := a.WalkEntitySegments(ctx, startTime, nil, walkEntities) + if err != nil { + a.logger.Error("failed to load segments for export", "error", err) + return fmt.Errorf("failed to load segments for export: %w", err) + } } } @@ -3072,30 +3274,129 @@ var _ encoder = (*csvEncoder)(nil) type csvEncoder struct { *csv.Writer + // columnIndex stores all CSV columns and their respective column index. + columnIndex map[string]int + + // wroteHeader indicates to the encoder whether or not a header row has been written wroteHeader bool } +// baseActivityExportCSVHeader returns the base CSV header for the activity +// export API. The existing order should not be changed. New fields should +// be appended to the end. +func baseActivityExportCSVHeader() []string { + return []string{ + "identity_name", + "alias_name", + "client_id", + "client_type", + "namespace_id", + "namespace_path", + "mount_accessor", + "mount_path", + "mount_type", + "timestamp", + } +} + +// newCSVEncoder instantiates a csvEncoder with a new csv.Writer and base +// columnIndex based on the base activity export CSV header. func newCSVEncoder(w io.Writer) (*csvEncoder, error) { writer := csv.NewWriter(w) + baseColumnIndex := make(map[string]int) + + for i, col := range baseActivityExportCSVHeader() { + baseColumnIndex[col] = i + } + return &csvEncoder{ - Writer: writer, + Writer: writer, + columnIndex: baseColumnIndex, }, nil } -// Encode converts an export bundle into a set of strings and writes them to the -// csv writer. -func (c *csvEncoder) Encode(record *ActivityLogExportRecord) error { +// flattenMapField generates a flattened column name for a map field (e.g. foo.bar). +func (c *csvEncoder) flattenMapField(fieldName string, subKey string) string { + return fmt.Sprintf("%s.%s", fieldName, subKey) +} + +// flattenSliceField generates a flattened column name for a slice field (e.g. foo.0). +func (c *csvEncoder) flattenSliceField(fieldName string, index int) string { + return fmt.Sprintf("%s.%d", fieldName, index) +} + +// accumulateHeaderFields populates the columnIndex with newly discovered activity +// export fields. Map keys and slice indices will be flattened into individual column +// names. A map key "identity_metadata" that contains sub-keys "foo" and "bar" +// will result in indexing the columns "identity_metadata.foo" and "identity_metadata.bar". +// A slice "policies" with two values will result in indexing the columns "policies.0" and +// "policies.1". +func (c *csvEncoder) accumulateHeaderFields(record *ActivityLogExportRecord) error { + var recordMap map[string]interface{} + + err := mapstructure.Decode(record, &recordMap) + if err != nil { + return err + } + + for field, rawValue := range recordMap { + switch typedValue := rawValue.(type) { + case map[string]string: + for key := range typedValue { + columnName := c.flattenMapField(field, key) + + if _, exists := c.columnIndex[columnName]; !exists { + // final index value will be chosen upon generating header + c.columnIndex[columnName] = exportCSVFlatteningInitIndex + } + } + + case []string: + for idx := range typedValue { + columnName := c.flattenSliceField(field, idx) + + if _, exists := c.columnIndex[columnName]; !exists { + // final index value will be chosen upon generating header + c.columnIndex[columnName] = exportCSVFlatteningInitIndex + } + } + } + } + + return nil +} + +// generateHeader initially finalizes column indices for flattened fields. The +// flattened fields are appended to the base header in lexicographical order. +func (c *csvEncoder) generateHeader() []string { + header := baseActivityExportCSVHeader() + + flattenedColumnNames := make([]string, 0) + + for k, idx := range c.columnIndex { + // base header fields already have non-zero index values + if idx == -1 { + flattenedColumnNames = append(flattenedColumnNames, k) + } + } + + // sort to provide deterministic column ordering for flattened fields + slices.Sort(flattenedColumnNames) + + for _, columnName := range flattenedColumnNames { + c.columnIndex[columnName] = len(header) + header = append(header, columnName) + } + + return header +} + +// writeHeader will write a CSV header if it has not already been written +func (c *csvEncoder) writeHeader() error { if !c.wroteHeader { - err := c.Writer.Write([]string{ - "client_id", - "client_type", - "namespace_id", - "namespace_path", - "mount_accessor", - "timestamp", - }) + err := c.Writer.Write(c.generateHeader()) if err != nil { return err } @@ -3103,12 +3404,47 @@ func (c *csvEncoder) Encode(record *ActivityLogExportRecord) error { c.wroteHeader = true } - return c.Writer.Write([]string{ - record.ClientID, - record.ClientType, - record.NamespaceID, - record.NamespacePath, - record.MountAccessor, - record.Timestamp, - }) + return nil +} + +// Encode converts an ActivityLogExportRecord into a row of CSV data. Map and +// slice fields are flattened in the process. The resulting CSV row is +// written to the underlying CSV writer. +func (c *csvEncoder) Encode(record *ActivityLogExportRecord) error { + row := make([]string, len(c.columnIndex)) + + var recordMap map[string]interface{} + err := mapstructure.Decode(record, &recordMap) + if err != nil { + return err + } + + for col, rawValue := range recordMap { + switch typedValue := rawValue.(type) { + case string: + if idx, ok := c.columnIndex[col]; ok { + row[idx] = typedValue + } + + case map[string]string: + for key, val := range typedValue { + columnName := c.flattenMapField(col, key) + + if idx, ok := c.columnIndex[columnName]; ok { + row[idx] = val + } + } + + case []string: + for idx, val := range typedValue { + columnName := c.flattenSliceField(col, idx) + + if idx, ok := c.columnIndex[columnName]; ok { + row[idx] = val + } + } + } + } + + return c.Writer.Write(row) } diff --git a/vault/activity_log_test.go b/vault/activity_log_test.go index 2871e733c2a2..13965381be67 100644 --- a/vault/activity_log_test.go +++ b/vault/activity_log_test.go @@ -4896,3 +4896,131 @@ func TestActivityLog_reportPrecomputedQueryMetrics(t *testing.T) { hasMetric(t, data, "identity.pki_acme.active.reporting_period", 3, nil) }) } + +// TestActivityLog_Export_CSV_Header verifies that the export API properly +// generates a CSV column index and header. Various ActivityLogExportRecords +// are used to mimic an export discovering new map and slice fields that are +// meant to be flattened as new columns. +func TestActivityLog_Export_CSV_Header(t *testing.T) { + encoder, err := newCSVEncoder(nil) + require.NoError(t, err) + + expectedColumnIndex := make(map[string]int) + + // set expected index as base columnIndex upon encoder initialization + for k, v := range encoder.columnIndex { + expectedColumnIndex[k] = v + } + + err = encoder.accumulateHeaderFields(&ActivityLogExportRecord{ + Policies: []string{ + "foo", + }, + IdentityMetadata: map[string]string{ + "email_address": "jdoe@abc.com", + }, + }) + require.NoError(t, err) + + expectedColumnIndex["policies.0"] = exportCSVFlatteningInitIndex + expectedColumnIndex["identity_metadata.email_address"] = exportCSVFlatteningInitIndex + + require.Empty(t, deep.Equal(expectedColumnIndex, encoder.columnIndex)) + + err = encoder.accumulateHeaderFields(&ActivityLogExportRecord{ + Policies: []string{ + "foo", + "bar", + "baz", + }, + AliasCustomMetadata: map[string]string{ + "region": "west", + "group": "san_francisco", + }, + }) + require.NoError(t, err) + + expectedColumnIndex["policies.1"] = exportCSVFlatteningInitIndex + expectedColumnIndex["policies.2"] = exportCSVFlatteningInitIndex + expectedColumnIndex["alias_custom_metadata.group"] = exportCSVFlatteningInitIndex + expectedColumnIndex["alias_custom_metadata.region"] = exportCSVFlatteningInitIndex + + require.Empty(t, deep.Equal(expectedColumnIndex, encoder.columnIndex)) + + err = encoder.accumulateHeaderFields(&ActivityLogExportRecord{ + Policies: []string{ + "foo", + }, + IdentityGroupIDs: []string{ + "97798e02-51e5-4ef3-906e-82c76d1a396e", + }, + IdentityMetadata: map[string]string{ + "first_name": "John", + "last_name": "Doe", + }, + AliasMetadata: map[string]string{ + "contact_email": "foo@abc.com", + }, + }) + require.NoError(t, err) + + expectedColumnIndex["identity_metadata.first_name"] = exportCSVFlatteningInitIndex + expectedColumnIndex["identity_metadata.last_name"] = exportCSVFlatteningInitIndex + expectedColumnIndex["alias_metadata.contact_email"] = exportCSVFlatteningInitIndex + expectedColumnIndex["identity_group_ids.0"] = exportCSVFlatteningInitIndex + + require.Empty(t, deep.Equal(expectedColumnIndex, encoder.columnIndex)) + + // no change because all the fields have seen before + err = encoder.accumulateHeaderFields(&ActivityLogExportRecord{ + AliasCustomMetadata: map[string]string{ + "group": "does-not-matter", + "region": "does-not-matter", + }, + AliasMetadata: map[string]string{ + "contact_email": "does-not-matter", + }, + IdentityGroupIDs: []string{ + "does-not-matter", + }, + IdentityMetadata: map[string]string{ + "first_name": "does-not-matter", + "last_name": "does-not-matter", + }, + Policies: []string{ + "does-not-matter", + "does-not-matter", + "does-not-matter", + }, + }) + require.NoError(t, err) + require.Empty(t, deep.Equal(expectedColumnIndex, encoder.columnIndex)) + + // no change because there are no slice or map fields + err = encoder.accumulateHeaderFields(&ActivityLogExportRecord{}) + require.NoError(t, err) + require.Empty(t, deep.Equal(expectedColumnIndex, encoder.columnIndex)) + + expectedHeader := append(baseActivityExportCSVHeader(), + "alias_custom_metadata.group", + "alias_custom_metadata.region", + "alias_metadata.contact_email", + "identity_group_ids.0", + "identity_metadata.email_address", + "identity_metadata.first_name", + "identity_metadata.last_name", + "policies.0", + "policies.1", + "policies.2") + + header := encoder.generateHeader() + require.Empty(t, deep.Equal(expectedHeader, header)) + + expectedColumnIndex = make(map[string]int) + + for idx, col := range expectedHeader { + expectedColumnIndex[col] = idx + } + + require.Empty(t, deep.Equal(expectedColumnIndex, encoder.columnIndex)) +} diff --git a/vault/external_tests/activity_testonly/activity_testonly_test.go b/vault/external_tests/activity_testonly/activity_testonly_test.go index 23a72e23e2ae..7d723e5358df 100644 --- a/vault/external_tests/activity_testonly/activity_testonly_test.go +++ b/vault/external_tests/activity_testonly/activity_testonly_test.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "math" + "strings" "testing" "time" @@ -519,6 +520,9 @@ func getJSONExport(t *testing.T, client *api.Client, monthsPreviousTo int, now t clients := make(map[string]vault.ActivityLogExportRecord) for { + if !decoder.More() { + break + } var record vault.ActivityLogExportRecord err := decoder.Decode(&record) @@ -527,20 +531,28 @@ func getJSONExport(t *testing.T, client *api.Client, monthsPreviousTo int, now t } clients[record.ClientID] = record - - if !decoder.More() { - break - } } return clients, nil } -// getCSVExport is used to fetch activity export records using csv format. -// The records will returned as a map keyed by client ID. +// getCSVExport fetches activity export records using csv format. All flattened +// map and slice fields will be unflattened so that the a proper ActivityLogExportRecord +// can be formed. The records will returned as a map keyed by client ID. func getCSVExport(t *testing.T, client *api.Client, monthsPreviousTo int, now time.Time) (map[string]vault.ActivityLogExportRecord, error) { t.Helper() + mapFields := map[string]struct{}{ + "alias_custom_metadata": {}, + "alias_metadata": {}, + "identity_metadata": {}, + } + + sliceFields := map[string]struct{}{ + "identity_group_ids": {}, + "policies": {}, + } + resp, err := client.Logical().ReadRawWithData("sys/internal/counters/activity/export", map[string][]string{ "start_time": {timeutil.StartOfMonth(timeutil.MonthsPreviousTo(monthsPreviousTo, now)).Format(time.RFC3339)}, "end_time": {timeutil.EndOfMonth(now).Format(time.RFC3339)}, @@ -560,13 +572,59 @@ func getCSVExport(t *testing.T, client *api.Client, monthsPreviousTo int, now ti return nil, err } - csvColumns := csvRecords[0] + if len(csvRecords) == 0 { + return clients, nil + } + + csvHeader := csvRecords[0] - for i := 1; i < len(csvRecords); i++ { + // skip initial row as it is header + for rowIdx := 1; rowIdx < len(csvRecords); rowIdx++ { recordMap := make(map[string]interface{}) - for j, k := range csvColumns { - recordMap[k] = csvRecords[i][j] + for columnIdx, columnName := range csvHeader { + val := csvRecords[rowIdx][columnIdx] + + // determine if column has been flattened + columnNameParts := strings.SplitN(columnName, ".", 2) + + if len(columnNameParts) > 1 { + prefix := columnNameParts[0] + + if _, ok := mapFields[prefix]; ok { + m, mOK := recordMap[prefix] + + // ensure output contains non-nil map + if !mOK { + m = make(map[string]string) + recordMap[prefix] = m + } + + // ignore empty CSV column value + if val != "" { + m.(map[string]string)[columnNameParts[1]] = val + recordMap[prefix] = m + } + } else if _, ok := sliceFields[prefix]; ok { + // ensure output contains non-nil slice + s, sOK := recordMap[prefix] + if !sOK { + s = make([]string, 0) + recordMap[prefix] = s + } + + // ignore empty CSV column value + if val != "" { + s = append(s.([]string), val) + recordMap[prefix] = s + } + } else { + t.Fatalf("unexpected CSV field: %s", columnName) + } + + } else { + recordMap[columnName] = val + } } var record vault.ActivityLogExportRecord diff --git a/vault/logical_system_activity.go b/vault/logical_system_activity.go index 4c7b743cbf6e..224480d3912e 100644 --- a/vault/logical_system_activity.go +++ b/vault/logical_system_activity.go @@ -287,7 +287,7 @@ func (b *SystemBackend) handleClientExport(ctx context.Context, req *logical.Req err = a.writeExport(runCtx, req.ResponseWriter, d.Get("format").(string), startTime, endTime) if err != nil { - if errors.Is(err, ErrActivityExportNoDataInRange) || errors.Is(err, ErrActivityExportInProgress) || strings.HasPrefix(err.Error(), ActivityExportInvalidFormatPrefix) { + if errors.Is(err, ErrActivityExportInProgress) || strings.HasPrefix(err.Error(), ActivityExportInvalidFormatPrefix) { return logical.ErrorResponse(err.Error()), nil } else { return nil, err