Skip to content

Commit

Permalink
[Usage Collection] add caching layer for stats (elastic#119312) (elas…
Browse files Browse the repository at this point in the history
  • Loading branch information
Bamieh authored Dec 16, 2021
1 parent dd2ab00 commit a18ed0f
Showing 18 changed files with 312 additions and 21 deletions.
Original file line number Diff line number Diff line change
@@ -18,17 +18,20 @@ describe('TelemetryService', () => {

await telemetryService.fetchTelemetry();
expect(telemetryService['http'].post).toBeCalledWith('/api/telemetry/v2/clusters/_stats', {
body: JSON.stringify({ unencrypted: false }),
body: JSON.stringify({ unencrypted: false, refreshCache: false }),
});
});
});

describe('fetchExample', () => {
it('calls fetchTelemetry with unencrupted: true', async () => {
it('calls fetchTelemetry with unencrypted: true, refreshCache: true', async () => {
const telemetryService = mockTelemetryService();
telemetryService.fetchTelemetry = jest.fn();
await telemetryService.fetchExample();
expect(telemetryService.fetchTelemetry).toBeCalledWith({ unencrypted: true });
expect(telemetryService.fetchTelemetry).toBeCalledWith({
unencrypted: true,
refreshCache: true,
});
});
});

5 changes: 3 additions & 2 deletions src/plugins/telemetry/public/services/telemetry_service.ts
Original file line number Diff line number Diff line change
@@ -140,7 +140,7 @@ export class TelemetryService {

/** Fetches an unencrypted telemetry payload so we can show it to the user **/
public fetchExample = async (): Promise<UnencryptedTelemetryPayload> => {
return await this.fetchTelemetry({ unencrypted: true });
return await this.fetchTelemetry({ unencrypted: true, refreshCache: true });
};

/**
@@ -149,9 +149,10 @@ export class TelemetryService {
*/
public fetchTelemetry = async <T = EncryptedTelemetryPayload | UnencryptedTelemetryPayload>({
unencrypted = false,
refreshCache = false,
} = {}): Promise<T> => {
return this.http.post('/api/telemetry/v2/clusters/_stats', {
body: JSON.stringify({ unencrypted }),
body: JSON.stringify({ unencrypted, refreshCache }),
});
};

16 changes: 16 additions & 0 deletions src/plugins/telemetry/schema/oss_root.json
Original file line number Diff line number Diff line change
@@ -18,6 +18,22 @@
"collectionSource": {
"type": "keyword"
},
"cacheDetails": {
"properties": {
"updatedAt": {
"type": "date",
"_meta": {
"description": "The timestamp the payload was last cached."
}
},
"fetchedAt": {
"type": "date",
"_meta": {
"description": "The timestamp the payload was grabbed from cache."
}
}
}
},
"stack_stats": {
"properties": {
"data": {
4 changes: 3 additions & 1 deletion src/plugins/telemetry/server/routes/telemetry_usage_stats.ts
Original file line number Diff line number Diff line change
@@ -24,16 +24,18 @@ export function registerTelemetryUsageStatsRoutes(
validate: {
body: schema.object({
unencrypted: schema.boolean({ defaultValue: false }),
refreshCache: schema.boolean({ defaultValue: false }),
}),
},
},
async (context, req, res) => {
const { unencrypted } = req.body;
const { unencrypted, refreshCache } = req.body;

try {
const statsConfig: StatsGetterConfig = {
request: req,
unencrypted,
refreshCache,
};

const stats = await telemetryCollectionManager.getStats(statsConfig);
Original file line number Diff line number Diff line change
@@ -80,6 +80,7 @@ function mockStatsCollectionConfig(
esClient: mockGetLocalStats(clusterInfo, clusterStats),
usageCollection: mockUsageCollection(kibana),
kibanaRequest: httpServerMock.createKibanaRequest(),
refreshCache: false,
};
}

7 changes: 7 additions & 0 deletions src/plugins/telemetry_collection_manager/common/index.ts
Original file line number Diff line number Diff line change
@@ -8,3 +8,10 @@

export const PLUGIN_ID = 'telemetryCollectionManager';
export const PLUGIN_NAME = 'telemetry_collection_manager';

/**
* The duration, in milliseconds, to cache stats
* Currently 4 hours.
*/
const hour = 1000 * 60 * 60;
export const CACHE_DURATION_MS = 4 * hour;
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { CacheManager } from './cache_manager';

describe('CacheManager', () => {
const mockCacheKey = 'mock_key';
const mockCacheItem = 'cache_item';
const cacheDurationMs = 10000;
let mockNow: number;

beforeEach(() => {
jest.useFakeTimers('modern');
mockNow = jest.getRealSystemTime();
jest.setSystemTime(mockNow);
});
afterEach(() => jest.clearAllMocks());
afterAll(() => jest.useRealTimers());

it('caches object for the cache duration only', () => {
const cacheManager = new CacheManager({ cacheDurationMs });
cacheManager.setCache(mockCacheKey, mockCacheItem);
expect(cacheManager.getFromCache(mockCacheKey)).toEqual(mockCacheItem);
jest.advanceTimersByTime(cacheDurationMs + 100);
expect(cacheManager.getFromCache(mockCacheKey)).toEqual(undefined);
});

it('#resetCache removes cached objects', () => {
const cacheManager = new CacheManager({ cacheDurationMs });
cacheManager.setCache(mockCacheKey, mockCacheItem);
expect(cacheManager.getFromCache(mockCacheKey)).toEqual(mockCacheItem);
cacheManager.resetCache();
expect(cacheManager.getFromCache(mockCacheKey)).toEqual(undefined);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import LRUCache from 'lru-cache';

export interface CacheManagerConfig {
// cache duration of objects in ms
cacheDurationMs: number;
}

export class CacheManager {
private readonly cache: LRUCache<string, unknown>;

constructor({ cacheDurationMs }: CacheManagerConfig) {
this.cache = new LRUCache({
max: 1,
maxAge: cacheDurationMs,
});
}

/**
* Cache an object by key
*/
public setCache = (cacheKey: string, data: unknown): void => {
this.cache.set(cacheKey, data);
};

/**
* returns cached object. If the key is not found will return undefined.
*/
public getFromCache = <T = unknown>(cacheKey: string): T | undefined => {
return this.cache.get(cacheKey) as T;
};

/**
* Removes all cached objects
*/
public resetCache(): void {
this.cache.reset();
}
}
10 changes: 10 additions & 0 deletions src/plugins/telemetry_collection_manager/server/cache/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

export { CacheManager } from './cache_manager';
export type { CacheManagerConfig } from './cache_manager';
32 changes: 31 additions & 1 deletion src/plugins/telemetry_collection_manager/server/plugin.test.ts
Original file line number Diff line number Diff line change
@@ -45,6 +45,12 @@ describe('Telemetry Collection Manager', () => {
const telemetryCollectionManager = new TelemetryCollectionManagerPlugin(initializerContext);
const setupApi = telemetryCollectionManager.setup(coreMock.createSetup(), { usageCollection });
const collectionStrategy = createCollectionStrategy(1);
beforeEach(() => {
// Reset cache on every request.
// 10s cache to avoid misatekly invalidating cache during test runs
// eslint-disable-next-line dot-notation
telemetryCollectionManager['cacheManager'].resetCache();
});

describe('before start', () => {
test('registers a collection strategy', () => {
@@ -196,13 +202,37 @@ describe('Telemetry Collection Manager', () => {
await expect(setupApi.getStats(config)).resolves.toStrictEqual([
{
clusterUuid: 'clusterUuid',
stats: { ...basicStats, collectionSource: 'test_collection' },
stats: {
...basicStats,
cacheDetails: { updatedAt: expect.any(String), fetchedAt: expect.any(String) },
collectionSource: 'test_collection',
},
},
]);

expect(
collectionStrategy.clusterDetailsGetter.mock.calls[0][0].soClient
).not.toBeInstanceOf(TelemetrySavedObjectsClient);
});

test('returns cached object on multiple calls', async () => {
collectionStrategy.clusterDetailsGetter.mockResolvedValue([
{ clusterUuid: 'clusterUuid' },
]);
collectionStrategy.statsGetter.mockResolvedValue([basicStats]);
await setupApi.getStats(config);

await expect(setupApi.getStats(config)).resolves.toStrictEqual([
{
clusterUuid: 'clusterUuid',
stats: {
...basicStats,
cacheDetails: { updatedAt: expect.any(String), fetchedAt: expect.any(String) },
collectionSource: 'test_collection',
},
},
]);
});
});

describe('getOptInStats', () => {
51 changes: 46 additions & 5 deletions src/plugins/telemetry_collection_manager/server/plugin.ts
Original file line number Diff line number Diff line change
@@ -32,9 +32,12 @@ import type {
StatsCollectionContext,
UnencryptedStatsGetterConfig,
EncryptedStatsGetterConfig,
ClusterDetails,
} from './types';
import { encryptTelemetry } from './encryption';
import { TelemetrySavedObjectsClient } from './telemetry_saved_objects_client';
import { CacheManager } from './cache';
import { CACHE_DURATION_MS } from '../common';

interface TelemetryCollectionPluginsDepsSetup {
usageCollection: UsageCollectionSetup;
@@ -51,6 +54,7 @@ export class TelemetryCollectionManagerPlugin
private savedObjectsService?: SavedObjectsServiceStart;
private readonly isDistributable: boolean;
private readonly version: string;
private cacheManager = new CacheManager({ cacheDurationMs: CACHE_DURATION_MS });

constructor(initializerContext: PluginInitializerContext) {
this.logger = initializerContext.logger.get();
@@ -125,9 +129,10 @@ export class TelemetryCollectionManagerPlugin
const soClient = this.getSavedObjectsClient(config);
// Provide the kibanaRequest so opted-in plugins can scope their custom clients only if the request is not encrypted
const kibanaRequest = config.unencrypted ? config.request : void 0;
const refreshCache = !!config.refreshCache;

if (esClient && soClient) {
return { usageCollection, esClient, soClient, kibanaRequest };
return { usageCollection, esClient, soClient, kibanaRequest, refreshCache };
}
}

@@ -284,6 +289,25 @@ export class TelemetryCollectionManagerPlugin
return [];
}

private createCacheKey(collectionSource: string, clustersDetails: ClusterDetails[]) {
const clusterUUids = clustersDetails
.map(({ clusterUuid }) => clusterUuid)
.sort()
.join('_');

return `${collectionSource}::${clusterUUids}`;
}

private updateFetchedAt(statsPayload: UsageStatsPayload[]): UsageStatsPayload[] {
return statsPayload.map((stat) => ({
...stat,
cacheDetails: {
...stat.cacheDetails,
fetchedAt: new Date().toISOString(),
},
}));
}

private async getUsageForCollection(
collection: CollectionStrategy,
statsCollectionConfig: StatsCollectionConfig
@@ -292,17 +316,34 @@ export class TelemetryCollectionManagerPlugin
logger: this.logger.get(collection.title),
version: this.version,
};

const clustersDetails = await collection.clusterDetailsGetter(statsCollectionConfig, context);
const { refreshCache } = statsCollectionConfig;
const { title: collectionSource } = collection;

// on `refreshCache: true` clear all cache to store a fresh copy
if (refreshCache) {
this.cacheManager.resetCache();
}

if (clustersDetails.length === 0) {
// don't bother doing a further lookup.
return [];
}

const cacheKey = this.createCacheKey(collectionSource, clustersDetails);
const cachedUsageStatsPayload = this.cacheManager.getFromCache<UsageStatsPayload[]>(cacheKey);
if (cachedUsageStatsPayload) {
return this.updateFetchedAt(cachedUsageStatsPayload);
}

const now = new Date().toISOString();
const stats = await collection.statsGetter(clustersDetails, statsCollectionConfig, context);
const usageStatsPayload = stats.map((stat) => ({
collectionSource,
cacheDetails: { updatedAt: now, fetchedAt: now },
...stat,
}));
this.cacheManager.setCache(cacheKey, usageStatsPayload);

// Add the `collectionSource` to the resulting payload
return stats.map((stat) => ({ collectionSource: collection.title, ...stat }));
return this.updateFetchedAt(usageStatsPayload);
}
}
8 changes: 8 additions & 0 deletions src/plugins/telemetry_collection_manager/server/types.ts
Original file line number Diff line number Diff line change
@@ -37,6 +37,7 @@ export interface TelemetryOptInStats {

export interface BaseStatsGetterConfig {
unencrypted: boolean;
refreshCache?: boolean;
request?: KibanaRequest;
}

@@ -58,6 +59,12 @@ export interface StatsCollectionConfig {
esClient: ElasticsearchClient;
soClient: SavedObjectsClientContract;
kibanaRequest: KibanaRequest | undefined; // intentionally `| undefined` to enforce providing the parameter
refreshCache: boolean;
}

export interface CacheDetails {
updatedAt: string;
fetchedAt: string;
}

export interface BasicStatsPayload {
@@ -71,6 +78,7 @@ export interface BasicStatsPayload {
}

export interface UsageStatsPayload extends BasicStatsPayload {
cacheDetails: CacheDetails;
collectionSource: string;
}

Loading

0 comments on commit a18ed0f

Please sign in to comment.