Merge pull request #420 from atlanhq/DVX-169

Audit and search log enhancements

samples/packages/relational-assets-builder/src/test/kotlin/com/atlan/pkg/rab/CreateThenUpsertRABTest.kt

@@ -15,11 +15,13 @@ import com.atlan.model.assets.Table
 import com.atlan.model.assets.View
 import com.atlan.model.core.AtlanTag
 import com.atlan.model.enums.AtlanConnectorType
+import com.atlan.model.enums.AtlanDeleteType
 import com.atlan.model.enums.AtlanIcon
 import com.atlan.model.enums.AtlanStatus
 import com.atlan.model.enums.AtlanTagColor
 import com.atlan.model.enums.CertificateStatus
 import com.atlan.model.fields.AtlanField
+import com.atlan.model.search.FluentSearch
 import com.atlan.model.typedefs.AtlanTagDef
 import com.atlan.pkg.PackageTest
 import org.testng.Assert.assertFalse
@@ -499,9 +501,30 @@ class CreateThenUpsertRABTest : PackageTest() {
 
     @AfterClass(alwaysRun = true)
     fun afterClass(context: ITestContext) {
+        // Purge view and its columns first, otherwise we end up with stale
+        // references between tag and columns in the view
+        purgeView()
         removeConnection(conn1, conn1Type)
         AtlanTagDef.purge(tag1)
         AtlanTagDef.purge(tag2)
         teardown(context.failedTests.size() > 0)
     }
+
+    private fun purgeView() {
+        val c1 = Connection.findByName(conn1, conn1Type, connectionAttrs)[0]!!
+        val toPurge = mutableListOf<String>()
+        val found = View.select(true)
+            .where(FluentSearch.ARCHIVED)
+            .where(View.CONNECTION_QUALIFIED_NAME.eq(c1.qualifiedName))
+            .includesOnResults(tableAttrs)
+            .includeOnRelations(Asset.NAME)
+            .stream()
+            .forEach { view ->
+                toPurge.add(view.guid)
+                (view as View).columns.forEach { column ->
+                    toPurge.add(column.guid)
+                }
+            }
+        Atlan.getDefaultClient().assets.delete(toPurge, AtlanDeleteType.PURGE)
+    }
 }

sdk/src/main/java/com/atlan/api/AssetEndpoint.java

@@ -135,6 +135,31 @@ public AuditSearchResponse auditLogs(AuditSearchRequest request) throws AtlanExc
      */
     public AuditSearchResponse auditLogs(AuditSearchRequest request, RequestOptions options) throws AtlanException {
         String url = String.format("%s%s", getBaseUrl(), audit_endpoint);
+        boolean missingSort =
+                request.getDsl().getSort() == null || request.getDsl().getSort().isEmpty();
+        boolean missingGuidSort = true;
+        if (!missingSort) {
+            // If there is some sort, see whether GUID is already included
+            for (SortOptions option : request.getDsl().getSort()) {
+                if (option.isField()) {
+                    String fieldName = option.field().field();
+                    if (AuditSearchRequest.ENTITY_ID.getKeywordFieldName().equals(fieldName)) {
+                        missingGuidSort = false;
+                        break;
+                    }
+                }
+            }
+        }
+        if (missingGuidSort) {
+            // If there is no sort by GUID, always add it as a final (tie-breaker) criteria
+            // to ensure there is consistent paging (unfortunately sorting by _doc still has duplicates
+            // across large number of pages)
+            request = request.toBuilder()
+                    .dsl(request.getDsl().toBuilder()
+                            .sortOption(AuditSearchRequest.ENTITY_ID.order(SortOrder.Asc))
+                            .build())
+                    .build();
+        }
         AuditSearchResponse response = ApiResource.request(
                 client, ApiResource.RequestMethod.POST, url, request, AuditSearchResponse.class, options);
         response.setClient(client);

sdk/src/main/java/com/atlan/api/SearchLogEndpoint.java

@@ -2,6 +2,7 @@
    Copyright 2023 Atlan Pte. Ltd. */
 package com.atlan.api;
 
+import co.elastic.clients.elasticsearch._types.SortOptions;
 import co.elastic.clients.elasticsearch._types.SortOrder;
 import com.atlan.AtlanClient;
 import com.atlan.exception.AtlanException;
@@ -43,11 +44,26 @@ public SearchLogResponse search(SearchLogRequest request) throws AtlanException
      */
     public SearchLogResponse search(SearchLogRequest request, RequestOptions options) throws AtlanException {
         String url = String.format("%s%s", getBaseUrl(), search_endpoint);
-        if (request.getDsl().getSort() == null || request.getDsl().getSort().isEmpty()) {
-            // If no sort has been provided, explicitly sort by time of the search for consistency of paging
+        boolean missingSort =
+                request.getDsl().getSort() == null || request.getDsl().getSort().isEmpty();
+        boolean missingTimeSort = true;
+        if (!missingSort) {
+            // If there is some sort, see whether time is already included
+            for (SortOptions option : request.getDsl().getSort()) {
+                if (option.isField()) {
+                    String fieldName = option.field().field();
+                    if (SearchLogEntry.SEARCHED_AT.getNumericFieldName().equals(fieldName)) {
+                        missingTimeSort = false;
+                        break;
+                    }
+                }
+            }
+        }
+        if (missingTimeSort) {
+            // If there is no sort by time, always add it as a final (tie-breaker) criteria
             // (there is not a guaranteed-unique key in a search log entry, but if we sort by timestamp in
             // ascending order then earlier pages should never have additional entries - at least not until
-            // there is full bi-temporal support in the search index, or time machines are invented...
+            // there is full bi-temporal support in the search index, or time machines are invented...)
             request = request.toBuilder()
                     .dsl(request.getDsl().toBuilder()
                             .sortOption(SearchLogEntry.SEARCHED_AT.order(SortOrder.Asc))

sdk/src/main/java/com/atlan/model/search/AuditSearchRequest.java

@@ -27,12 +27,36 @@
 public class AuditSearchRequest extends AtlanObject {
     private static final long serialVersionUID = 2L;
 
+    /** When the asset was created. */
     public static final NumericField CREATED = new NumericField("createTime", "created");
+
+    /** Unique identifier (GUID) of the asset that was created or changed. */
     public static final KeywordField ENTITY_ID = new KeywordField("guid", "entityId");
+
+    /** Type of the asset that was created or changed. */
     public static final KeywordField ENTITY_TYPE = new KeywordField("typeName", "typeName");
+
+    /** Unique name of the asset that was created or changed. */
     public static final KeywordField QUALIFIED_NAME = new KeywordField("qualifiedName", "entityQualifiedName");
+
+    /** User who made the update to the asset. */
     public static final KeywordField USER = new KeywordField("updatedBy", "user");
 
+    /** Type of action made against the asset. */
+    public static final KeywordField ACTION = new KeywordField("action", "action");
+
+    /** Type of actor (e.g. {@code workflow}) that created or changed the asset, if it was done programmatically. */
+    public static final KeywordField AGENT = new KeywordField("headers", "headers.x-atlan-agent");
+
+    /** Name of the package that created or changed the asset. */
+    public static final KeywordField PACKAGE_NAME = new KeywordField("headers", "headers.x-atlan-agent-package-name");
+
+    /** Name of the workflow (specific configuration of a package) that created or changed the asset. */
+    public static final KeywordField WORKFLOW_ID = new KeywordField("headers", "headers.x-atlan-agent-workflow-id");
+
+    /** Name of the agent (specific run of a workflow) that created or changed the asset. */
+    public static final KeywordField AGENT_ID = new KeywordField("headers", "headers.x-atlan-agent-id");
+
     private static final SortOptions LATEST_FIRST = CREATED.order(SortOrder.Desc);
 
     /** Parameters for the search itself. */
@@ -145,4 +169,31 @@ public AuditSearchResponse search(AtlanClient client) throws AtlanException {
                 .sort(LATEST_FIRST)
                 .toRequestBuilder();
     }
+
+    /**
+     * Start building an audit search request for the last common action made to any assets.
+     *
+     * @param action type of action (e.g. {@code ENTITY_CREATE})
+     * @param size number of changes to retrieve
+     * @return a request builder pre-configured with these criteria
+     */
+    public static AuditSearchRequestBuilder<?, ?> byAction(String action, int size) {
+        return byAction(Atlan.getDefaultClient(), action, size);
+    }
+
+    /**
+     * Start building an audit search request for the last common action made to any assets.
+     *
+     * @param client connectivity to the Atlan tenant on which to search the audit logs
+     * @param action type of action (e.g. {@code ENTITY_CREATE})
+     * @param size number of changes to retrieve
+     * @return a request builder pre-configured with these criteria
+     */
+    public static AuditSearchRequestBuilder<?, ?> byAction(AtlanClient client, String action, int size) {
+        return AuditSearch.builder(client)
+                .where(ACTION.eq(action))
+                .pageSize(size)
+                .sort(LATEST_FIRST)
+                .toRequestBuilder();
+    }
 }

sdk/src/main/java/com/atlan/model/search/AuditSearchResponse.java

@@ -45,9 +45,8 @@ public class AuditSearchResponse extends ApiResource implements Iterable<EntityA
     /** List of results from the search. */
     List<EntityAudit> entityAudits;
 
-    /** Unused. */
-    @JsonIgnore
-    final Object aggregations = null;
+    /** Map of results for the requested aggregations. */
+    Map<String, AggregationResult> aggregations;
 
     /** Number of results returned in this response. */
     Long count;