[Data Usage] add option for data streams route to return data streams…

… with no size (elastic#201817)

## Summary
- update data_streams route to accept includeZeroStorage flag to make
testing easier. the metering api only collects data once every 30s and
that might be changed to being longer so we can't wait for data to be
aggregated. We can potentially surface this option to the user in the
future.
- unskip data stream api test. will enable in MKI after manual
confirmation it passes with a followup PR.

---------

Co-authored-by: Ash <[email protected]>

x-pack/plugins/data_usage/common/rest_types/data_streams.ts

@@ -7,6 +7,14 @@
 
 import { schema, TypeOf } from '@kbn/config-schema';
 
+export const DataStreamsRequestSchema = {
+  query: schema.object({
+    includeZeroStorage: schema.boolean({ defaultValue: false }),
+  }),
+};
+
+export type DataStreamsRequestQuery = TypeOf<typeof DataStreamsRequestSchema.query>;
+
 export const DataStreamsResponseSchema = {
   body: () =>
     schema.arrayOf(

x-pack/plugins/data_usage/server/routes/internal/data_streams.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { DataStreamsResponseSchema } from '../../../common/rest_types';
+import { DataStreamsResponseSchema, DataStreamsRequestSchema } from '../../../common/rest_types';
 import { DATA_USAGE_DATA_STREAMS_API_ROUTE } from '../../../common';
 import { DataUsageContext, DataUsageRouter } from '../../types';
 import { getDataStreamsHandler } from './data_streams_handler';
@@ -23,7 +23,7 @@ export const registerDataStreamsRoute = (
       {
         version: '1',
         validate: {
-          request: {},
+          request: DataStreamsRequestSchema,
           response: {
             200: DataStreamsResponseSchema,
           },

x-pack/plugins/data_usage/server/routes/internal/data_streams_handler.ts

@@ -9,12 +9,15 @@ import { RequestHandler } from '@kbn/core/server';
 import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types';
 import { errorHandler } from '../error_handler';
 import { getMeteringStats } from '../../utils/get_metering_stats';
+import { DataStreamsRequestQuery } from '../../../common/rest_types/data_streams';
 
 export const getDataStreamsHandler = (
   dataUsageContext: DataUsageContext
-): RequestHandler<never, never, unknown, DataUsageRequestHandlerContext> => {
+): RequestHandler<never, DataStreamsRequestQuery, unknown, DataUsageRequestHandlerContext> => {
   const logger = dataUsageContext.logFactory.get('dataStreamsRoute');
-  return async (context, _, response) => {
+  return async (context, request, response) => {
+    const { includeZeroStorage } = request.query;
+
     logger.debug('Retrieving user data streams');
 
     try {
@@ -28,7 +31,7 @@ export const getDataStreamsHandler = (
           ? meteringStats
               .sort((a, b) => b.size_in_bytes - a.size_in_bytes)
               .reduce<Array<{ name: string; storageSizeBytes: number }>>((acc, stat) => {
-                if (stat.size_in_bytes > 0) {
+                if (includeZeroStorage || stat.size_in_bytes > 0) {
                   acc.push({
                     name: stat.name,
                     storageSizeBytes: stat.size_in_bytes ?? 0,

x-pack/test_serverless/api_integration/test_suites/common/data_usage/tests/data_streams.ts

@@ -14,10 +14,10 @@ import { FtrProviderContext } from '../../../../ftr_provider_context';
 export default function ({ getService }: FtrProviderContext) {
   const svlDatastreamsHelpers = getService('svlDatastreamsHelpers');
   const roleScopedSupertest = getService('roleScopedSupertest');
+  const retry = getService('retry');
   let supertestAdminWithCookieCredentials: SupertestWithRoleScope;
   const testDataStreamName = 'test-data-stream';
   describe(`GET ${DATA_USAGE_DATA_STREAMS_API_ROUTE}`, function () {
-    // due to the plugin depending on yml config (xpack.dataUsage.enabled), we cannot test in MKI until it is on by default
     this.tags(['skipMKI']);
     before(async () => {
       await svlDatastreamsHelpers.createDataStream(testDataStreamName);
@@ -33,18 +33,31 @@ export default function ({ getService }: FtrProviderContext) {
       await svlDatastreamsHelpers.deleteDataStream(testDataStreamName);
     });
 
-    // skipped because we filter out data streams with 0 storage size,
-    // and metering api does not pick up indexed data here
-    // TODO: route should potentially not depend solely on metering API
-    it.skip('returns created data streams', async () => {
+    it('returns created data streams', async () => {
+      await retry.tryForTime(10000, async () => {
+        const res = await supertestAdminWithCookieCredentials
+          .get(DATA_USAGE_DATA_STREAMS_API_ROUTE)
+          .query({ includeZeroStorage: true })
+          .set('elastic-api-version', '1');
+        const dataStreams: DataStreamsResponseBodySchemaBody = res.body;
+        const foundStream = dataStreams.find((stream) => stream.name === testDataStreamName);
+        if (!foundStream) {
+          throw new Error(`Data stream "${testDataStreamName}" not found. Retrying...`);
+        }
+        expect(res.statusCode).to.be(200);
+        expect(foundStream?.name).to.be(testDataStreamName);
+        expect(foundStream?.storageSizeBytes).to.be(0);
+        return true;
+      });
+    });
+    it('does not return created data streams without size', async () => {
       const res = await supertestAdminWithCookieCredentials
         .get(DATA_USAGE_DATA_STREAMS_API_ROUTE)
         .set('elastic-api-version', '1');
       const dataStreams: DataStreamsResponseBodySchemaBody = res.body;
       const foundStream = dataStreams.find((stream) => stream.name === testDataStreamName);
-      expect(foundStream?.name).to.be(testDataStreamName);
-      expect(foundStream?.storageSizeBytes).to.be(0);
       expect(res.statusCode).to.be(200);
+      expect(foundStream).to.be(undefined);
     });
   });
 }