From 415421fa646e8d1cf8fd7c9e6ff7ac8cb1870078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 20 Oct 2020 12:37:14 +0200 Subject: [PATCH 01/12] Return from the adapter if a response has more pages --- .../log_entries/kibana_log_entries_adapter.ts | 19 +++++++++++++++---- .../log_entries_domain/log_entries_domain.ts | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index c5b667fb20538..a999619004a65 100644 --- a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -43,8 +43,9 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { sourceConfiguration: InfraSourceConfiguration, fields: string[], params: LogEntriesParams - ): Promise { - const { startTimestamp, endTimestamp, query, cursor, size, highlightTerm } = params; + ): Promise<{ documents: LogEntryDocument[]; hasMoreBefore?: boolean; hasMoreAfter?: boolean }> { + const { startTimestamp, endTimestamp, query, cursor, highlightTerm } = params; + const size = params.size ?? LOG_ENTRIES_PAGE_SIZE; const { sortDirection, searchAfterClause } = processCursor(cursor); @@ -80,7 +81,7 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { index: sourceConfiguration.logAlias, ignoreUnavailable: true, body: { - size: typeof size !== 'undefined' ? size : LOG_ENTRIES_PAGE_SIZE, + size: size + 1, // Extra one to test if it has more before or after track_total_hits: false, _source: false, fields, @@ -113,7 +114,17 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { ); const hits = sortDirection === 'asc' ? esResult.hits.hits : esResult.hits.hits.reverse(); - return mapHitsToLogEntryDocuments(hits, fields); + const hasMore = hits.length > size; + + if (hasMore) { + hits.pop(); + } + + return { + documents: mapHitsToLogEntryDocuments(hits, fields), + hasMoreBefore: sortDirection === 'desc' ? hasMore : undefined, + hasMoreAfter: sortDirection === 'asc' ? hasMore : undefined, + }; } public async getContainedLogSummaryBuckets( diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index e211f72b4e076..995c5c084eeb8 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -136,7 +136,7 @@ export class InfraLogEntriesDomain { const requiredFields = getRequiredFields(configuration, messageFormattingRules); - const documents = await this.adapter.getLogEntries( + const { documents } = await this.adapter.getLogEntries( requestContext, configuration, requiredFields, @@ -323,7 +323,7 @@ export interface LogEntriesAdapter { sourceConfiguration: InfraSourceConfiguration, fields: string[], params: LogEntriesParams - ): Promise; + ): Promise<{ documents: LogEntryDocument[]; hasMoreBefore?: boolean; hasMoreAfter?: boolean }>; getContainedLogSummaryBuckets( requestContext: RequestHandlerContext, From 2a496a3304273872dacc65f9811528e0382fd15a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 20 Oct 2020 14:18:43 +0200 Subject: [PATCH 02/12] Add pagination information to `/log_entries/entries` API Extend the API response with `hasMoreBefore` and `hasMoreAfter` attributes. The value is a three-state boolean, where `undefined` means "we don't know if there are more pages". The API calls go in one direction, either before a `cursor`, or after it. To check if there are entries in the opposite direction we would need to do an extra API call. Instead of doing that for every request, we delegate the responsibility of doing the extra API call to the clients. --- .../common/http_api/log_entries/entries.ts | 16 ++++-- .../log_entries_domain/log_entries_domain.ts | 50 +++++++++++-------- .../server/routes/log_entries/entries.ts | 41 +++++++++------ 3 files changed, 67 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/infra/common/http_api/log_entries/entries.ts b/x-pack/plugins/infra/common/http_api/log_entries/entries.ts index 5014aeb52a3f7..ab1b972546b6e 100644 --- a/x-pack/plugins/infra/common/http_api/log_entries/entries.ts +++ b/x-pack/plugins/infra/common/http_api/log_entries/entries.ts @@ -98,11 +98,17 @@ export type LogEntryContext = rt.TypeOf; export type LogEntry = rt.TypeOf; export const logEntriesResponseRT = rt.type({ - data: rt.type({ - entries: rt.array(logEntryRT), - topCursor: rt.union([logEntriesCursorRT, rt.null]), - bottomCursor: rt.union([logEntriesCursorRT, rt.null]), - }), + data: rt.intersection([ + rt.type({ + entries: rt.array(logEntryRT), + topCursor: rt.union([logEntriesCursorRT, rt.null]), + bottomCursor: rt.union([logEntriesCursorRT, rt.null]), + }), + rt.partial({ + hasMoreBefore: rt.boolean, + hasMoreAfter: rt.boolean, + }), + ]), }); export type LogEntriesResponse = rt.TypeOf; diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 995c5c084eeb8..24cd190b2d3fc 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -74,7 +74,7 @@ export class InfraLogEntriesDomain { requestContext: RequestHandlerContext, sourceId: string, params: LogEntriesAroundParams - ) { + ): Promise<{ entries: LogEntry[]; hasMoreBefore?: boolean; hasMoreAfter?: boolean }> { const { startTimestamp, endTimestamp, center, query, size, highlightTerm } = params; /* @@ -87,14 +87,18 @@ export class InfraLogEntriesDomain { */ const halfSize = (size || LOG_ENTRIES_PAGE_SIZE) / 2; - const entriesBefore = await this.getLogEntries(requestContext, sourceId, { - startTimestamp, - endTimestamp, - query, - cursor: { before: center }, - size: Math.floor(halfSize), - highlightTerm, - }); + const { entries: entriesBefore, hasMoreBefore } = await this.getLogEntries( + requestContext, + sourceId, + { + startTimestamp, + endTimestamp, + query, + cursor: { before: center }, + size: Math.floor(halfSize), + highlightTerm, + } + ); /* * Elasticsearch's `search_after` returns documents after the specified cursor. @@ -108,23 +112,27 @@ export class InfraLogEntriesDomain { ? entriesBefore[entriesBefore.length - 1].cursor : { time: center.time - 1, tiebreaker: 0 }; - const entriesAfter = await this.getLogEntries(requestContext, sourceId, { - startTimestamp, - endTimestamp, - query, - cursor: { after: cursorAfter }, - size: Math.ceil(halfSize), - highlightTerm, - }); + const { entries: entriesAfter, hasMoreAfter } = await this.getLogEntries( + requestContext, + sourceId, + { + startTimestamp, + endTimestamp, + query, + cursor: { after: cursorAfter }, + size: Math.ceil(halfSize), + highlightTerm, + } + ); - return [...entriesBefore, ...entriesAfter]; + return { entries: [...entriesBefore, ...entriesAfter], hasMoreBefore, hasMoreAfter }; } public async getLogEntries( requestContext: RequestHandlerContext, sourceId: string, params: LogEntriesParams - ): Promise { + ): Promise<{ entries: LogEntry[]; hasMoreBefore?: boolean; hasMoreAfter?: boolean }> { const { configuration } = await this.libs.sources.getSourceConfiguration( requestContext.core.savedObjects.client, sourceId @@ -136,7 +144,7 @@ export class InfraLogEntriesDomain { const requiredFields = getRequiredFields(configuration, messageFormattingRules); - const { documents } = await this.adapter.getLogEntries( + const { documents, hasMoreBefore, hasMoreAfter } = await this.adapter.getLogEntries( requestContext, configuration, requiredFields, @@ -173,7 +181,7 @@ export class InfraLogEntriesDomain { }; }); - return entries; + return { entries, hasMoreBefore, hasMoreAfter }; } public async getLogSummaryBucketsBetween( diff --git a/x-pack/plugins/infra/server/routes/log_entries/entries.ts b/x-pack/plugins/infra/server/routes/log_entries/entries.ts index c1f63d9c29577..2baf3fd7aa990 100644 --- a/x-pack/plugins/infra/server/routes/log_entries/entries.ts +++ b/x-pack/plugins/infra/server/routes/log_entries/entries.ts @@ -34,14 +34,21 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) } = payload; let entries; + let hasMoreBefore; + let hasMoreAfter; + if ('center' in payload) { - entries = await logEntries.getLogEntriesAround(requestContext, sourceId, { - startTimestamp, - endTimestamp, - query: parseFilterQuery(query), - center: payload.center, - size, - }); + ({ entries, hasMoreBefore, hasMoreAfter } = await logEntries.getLogEntriesAround( + requestContext, + sourceId, + { + startTimestamp, + endTimestamp, + query: parseFilterQuery(query), + center: payload.center, + size, + } + )); } else { let cursor: LogEntriesParams['cursor']; if ('before' in payload) { @@ -50,13 +57,17 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) cursor = { after: payload.after }; } - entries = await logEntries.getLogEntries(requestContext, sourceId, { - startTimestamp, - endTimestamp, - query: parseFilterQuery(query), - cursor, - size, - }); + ({ entries, hasMoreBefore, hasMoreAfter } = await logEntries.getLogEntries( + requestContext, + sourceId, + { + startTimestamp, + endTimestamp, + query: parseFilterQuery(query), + cursor, + size, + } + )); } const hasEntries = entries.length > 0; @@ -67,6 +78,8 @@ export const initLogEntriesRoute = ({ framework, logEntries }: InfraBackendLibs) entries, topCursor: hasEntries ? entries[0].cursor : null, bottomCursor: hasEntries ? entries[entries.length - 1].cursor : null, + hasMoreBefore, + hasMoreAfter, }, }), }); From 398a6448f9bc470b9fb2d22a3b558dfdb993344e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 20 Oct 2020 16:04:40 +0200 Subject: [PATCH 03/12] Use `hasMoreBefore` and `hasMoreAfter` from the API response --- .../containers/logs/log_entries/index.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_entries/index.ts b/x-pack/plugins/infra/public/containers/logs/log_entries/index.ts index 4c8c610794b2e..c57b9de0b4a9e 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_entries/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_entries/index.ts @@ -359,16 +359,16 @@ const logEntriesStateReducer = (prevState: LogEntriesStateParams, action: Action case Action.ReceiveNewEntries: return { ...prevState, - ...action.payload, + entries: action.payload.entries, + topCursor: action.payload.topCursor, + bottomCursor: action.payload.bottomCursor, centerCursor: getCenterCursor(action.payload.entries), lastLoadedTime: new Date(), isReloading: false, - - // Be optimistic. If any of the before/after requests comes empty, set - // the corresponding flag to `false` - hasMoreBeforeStart: true, - hasMoreAfterEnd: true, + hasMoreBeforeStart: action.payload.hasMoreBefore ?? prevState.hasMoreBeforeStart, + hasMoreAfterEnd: action.payload.hasMoreAfter ?? prevState.hasMoreAfterEnd, }; + case Action.ReceiveEntriesBefore: { const newEntries = action.payload.entries; const prevEntries = cleanDuplicateItems(prevState.entries, newEntries); @@ -377,7 +377,7 @@ const logEntriesStateReducer = (prevState: LogEntriesStateParams, action: Action const update = { entries, isLoadingMore: false, - hasMoreBeforeStart: newEntries.length > 0, + hasMoreBeforeStart: action.payload.hasMoreBefore ?? prevState.hasMoreBeforeStart, // Keep the previous cursor if request comes empty, to easily extend the range. topCursor: newEntries.length > 0 ? action.payload.topCursor : prevState.topCursor, centerCursor: getCenterCursor(entries), @@ -394,7 +394,7 @@ const logEntriesStateReducer = (prevState: LogEntriesStateParams, action: Action const update = { entries, isLoadingMore: false, - hasMoreAfterEnd: newEntries.length > 0, + hasMoreAfterEnd: action.payload.hasMoreAfter ?? prevState.hasMoreAfterEnd, // Keep the previous cursor if request comes empty, to easily extend the range. bottomCursor: newEntries.length > 0 ? action.payload.bottomCursor : prevState.bottomCursor, centerCursor: getCenterCursor(entries), @@ -411,6 +411,8 @@ const logEntriesStateReducer = (prevState: LogEntriesStateParams, action: Action topCursor: null, bottomCursor: null, centerCursor: null, + // Assume there are more pages on both ends unless proven wrong by the + // API with an explicit `false` response. hasMoreBeforeStart: true, hasMoreAfterEnd: true, }; From 54c9b21dd0ecc4d69f8e50d703f9176d0484fc9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 20 Oct 2020 16:20:48 +0200 Subject: [PATCH 04/12] Track pagination state inside `useLogStream` --- .../containers/logs/log_stream/index.ts | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index 4a6da6063e960..f9e6dea04e778 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useState, useMemo } from 'react'; +import { useMemo } from 'react'; +import { useSetState } from 'react-use'; import { esKuery } from '../../../../../../../src/plugins/data/public'; import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; @@ -21,19 +22,35 @@ interface LogStreamProps { interface LogStreamState { entries: LogEntry[]; + topCursor: LogEntriesCursor | null; + bottomCursor: LogEntriesCursor | null; + hasMoreBefore?: boolean; + hasMoreAfter?: boolean; +} + +interface LogStreamReturn extends LogStreamState { fetchEntries: () => void; loadingState: 'uninitialized' | 'loading' | 'success' | 'error'; } +const INITIAL_STATE: LogStreamState = { + entries: [], + topCursor: null, + bottomCursor: null, + // Assume there are pages available until the API proves us wrong + hasMoreBefore: true, + hasMoreAfter: true, +}; + export function useLogStream({ sourceId, startTimestamp, endTimestamp, query, center, -}: LogStreamProps): LogStreamState { +}: LogStreamProps): LogStreamReturn { const { services } = useKibanaContextForPlugin(); - const [entries, setEntries] = useState([]); + const [state, setState] = useSetState(INITIAL_STATE); const parsedQuery = useMemo(() => { return query @@ -46,7 +63,7 @@ export function useLogStream({ { cancelPreviousOn: 'creation', createPromise: () => { - setEntries([]); + setState(INITIAL_STATE); const fetchPosition = center ? { center } : { before: 'last' }; return fetchLogEntries( @@ -61,7 +78,11 @@ export function useLogStream({ ); }, onResolve: ({ data }) => { - setEntries(data.entries); + setState((prevState) => ({ + ...data, + hasMoreBefore: data.hasMoreBefore ?? prevState.hasMoreBefore, + hasMoreAfter: data.hasMoreAfter ?? prevState.hasMoreAfter, + })); }, }, [sourceId, startTimestamp, endTimestamp, query] @@ -72,7 +93,7 @@ export function useLogStream({ ]); return { - entries, + ...state, fetchEntries, loadingState, }; @@ -80,7 +101,7 @@ export function useLogStream({ function convertPromiseStateToLoadingState( state: 'uninitialized' | 'pending' | 'resolved' | 'rejected' -): LogStreamState['loadingState'] { +): LogStreamReturn['loadingState'] { switch (state) { case 'uninitialized': return 'uninitialized'; From 5fcc039a1f0b1439c8d63c9e1189d82e1bf1ef26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 20 Oct 2020 17:13:56 +0200 Subject: [PATCH 05/12] Add pagination callbacks to `useLogStream` --- .../containers/logs/log_stream/index.ts | 125 +++++++++++++++++- 1 file changed, 118 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index f9e6dea04e778..b890a45caea3f 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -24,13 +24,18 @@ interface LogStreamState { entries: LogEntry[]; topCursor: LogEntriesCursor | null; bottomCursor: LogEntriesCursor | null; - hasMoreBefore?: boolean; - hasMoreAfter?: boolean; + hasMoreBefore: boolean; + hasMoreAfter: boolean; } +type LoadingState = 'uninitialized' | 'loading' | 'success' | 'error'; + interface LogStreamReturn extends LogStreamState { fetchEntries: () => void; - loadingState: 'uninitialized' | 'loading' | 'success' | 'error'; + fetchPreviousEntries: () => void; + fetchNextEntries: () => void; + loadingState: LoadingState; + pageLoadingState: LoadingState; } const INITIAL_STATE: LogStreamState = { @@ -42,6 +47,12 @@ const INITIAL_STATE: LogStreamState = { hasMoreAfter: true, }; +const EMPTY_DATA = { + entries: [], + topCursor: null, + bottomCursor: null, +}; + export function useLogStream({ sourceId, startTimestamp, @@ -88,20 +99,120 @@ export function useLogStream({ [sourceId, startTimestamp, endTimestamp, query] ); - const loadingState = useMemo(() => convertPromiseStateToLoadingState(entriesPromise.state), [ - entriesPromise.state, - ]); + const [previousEntriesPromise, fetchPreviousEntries] = useTrackedPromise( + { + cancelPreviousOn: 'creation', + createPromise: () => { + if (state.topCursor === null) { + throw new Error( + 'useLogState: Cannot fetch previous entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' + ); + } + + if (!state.hasMoreBefore) { + return Promise.resolve({ data: EMPTY_DATA }); + } + + return fetchLogEntries( + { + sourceId, + startTimestamp, + endTimestamp, + query: parsedQuery, + before: state.topCursor, + }, + services.http.fetch + ); + }, + onResolve: ({ data }) => { + if (!data.entries.length) { + return; + } + setState((prevState) => ({ + entries: [...data.entries, ...prevState.entries], + hasMoreBefore: data.hasMoreBefore ?? prevState.hasMoreBefore, + topCursor: data.topCursor ?? prevState.topCursor, + })); + }, + }, + [sourceId, startTimestamp, endTimestamp, query, state.topCursor] + ); + + const [nextEntriesPromise, fetchNextEntries] = useTrackedPromise( + { + cancelPreviousOn: 'creation', + createPromise: () => { + if (state.bottomCursor === null) { + throw new Error( + 'useLogState: Cannot fetch next entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' + ); + } + + if (!state.hasMoreAfter) { + return Promise.resolve({ data: EMPTY_DATA }); + } + + return fetchLogEntries( + { + sourceId, + startTimestamp, + endTimestamp, + query: parsedQuery, + after: state.bottomCursor, + }, + services.http.fetch + ); + }, + onResolve: ({ data }) => { + if (!data.entries.length) { + return; + } + setState((prevState) => ({ + entries: [...prevState.entries, ...data.entries], + hasMoreAfter: data.hasMoreAfter ?? prevState.hasMoreAfter, + bottomCursor: data.bottomCursor ?? prevState.bottomCursor, + })); + }, + }, + [sourceId, startTimestamp, endTimestamp, query, state.bottomCursor] + ); + + const loadingState = useMemo( + () => convertPromiseStateToLoadingState(entriesPromise.state), + [entriesPromise.state] + ); + + const pageLoadingState = useMemo(() => { + const states = [previousEntriesPromise.state, nextEntriesPromise.state]; + + if (states.includes('pending')) { + return 'loading'; + } + + if (states.includes('rejected')) { + return 'error'; + } + + if (states.includes('resolved')) { + return 'success'; + } + + return 'uninitialized'; + }, [previousEntriesPromise.state, nextEntriesPromise.state]); return { ...state, fetchEntries, + fetchPreviousEntries, + fetchNextEntries, loadingState, + pageLoadingState, }; } function convertPromiseStateToLoadingState( state: 'uninitialized' | 'pending' | 'resolved' | 'rejected' -): LogStreamReturn['loadingState'] { +): LoadingState { switch (state) { case 'uninitialized': return 'uninitialized'; From aebf81f2e8ba3d0b2a5971fd1815e9e929fea962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 20 Oct 2020 17:47:24 +0200 Subject: [PATCH 06/12] Handle pagination in the log stream shared component --- .../public/components/log_stream/index.tsx | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/infra/public/components/log_stream/index.tsx b/x-pack/plugins/infra/public/components/log_stream/index.tsx index f9bfbf9564798..ac472c68daa6d 100644 --- a/x-pack/plugins/infra/public/components/log_stream/index.tsx +++ b/x-pack/plugins/infra/public/components/log_stream/index.tsx @@ -17,6 +17,8 @@ import { useLogStream } from '../../containers/logs/log_stream'; import { ScrollableLogTextStreamView } from '../logging/log_text_stream'; +const PAGE_THRESHOLD = 2; + export interface LogStreamProps { sourceId?: string; startTimestamp: number; @@ -58,7 +60,16 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re }); // Internal state - const { loadingState, entries, fetchEntries } = useLogStream({ + const { + loadingState, + pageLoadingState, + entries, + hasMoreBefore, + hasMoreAfter, + fetchEntries, + fetchPreviousEntries, + fetchNextEntries, + } = useLogStream({ sourceId, startTimestamp, endTimestamp, @@ -70,6 +81,8 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re const isReloading = isLoadingSourceConfiguration || loadingState === 'uninitialized' || loadingState === 'loading'; + const isLoadingMore = pageLoadingState === 'loading'; + const columnConfigurations = useMemo(() => { return sourceConfiguration ? sourceConfiguration.configuration.logColumns : []; }, [sourceConfiguration]); @@ -101,13 +114,23 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re scale="medium" wrap={false} isReloading={isReloading} - isLoadingMore={false} - hasMoreBeforeStart={false} - hasMoreAfterEnd={false} + isLoadingMore={isLoadingMore} + hasMoreBeforeStart={hasMoreBefore} + hasMoreAfterEnd={hasMoreAfter} isStreaming={false} lastLoadedTime={null} jumpToTarget={noop} - reportVisibleInterval={noop} + reportVisibleInterval={({ fromScroll, pagesBeforeStart, pagesAfterEnd }) => { + if (!fromScroll) { + return; + } + + if (pagesBeforeStart < PAGE_THRESHOLD) { + fetchPreviousEntries(); + } else if (pagesAfterEnd < PAGE_THRESHOLD) { + fetchNextEntries(); + } + }} loadNewerItems={noop} reloadItems={fetchEntries} highlightedItem={highlight ?? null} From 0319c808f35b502e3033fccce03b4cd9d9bcc7c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 21 Oct 2020 11:33:10 +0200 Subject: [PATCH 07/12] Fix highlights API --- x-pack/plugins/infra/server/routes/log_entries/highlights.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/server/routes/log_entries/highlights.ts b/x-pack/plugins/infra/server/routes/log_entries/highlights.ts index be6be88e4b5ec..c29946fba16ab 100644 --- a/x-pack/plugins/infra/server/routes/log_entries/highlights.ts +++ b/x-pack/plugins/infra/server/routes/log_entries/highlights.ts @@ -79,7 +79,7 @@ export const initLogEntriesHighlightsRoute = ({ framework, logEntries }: InfraBa return response.ok({ body: logEntriesHighlightsResponseRT.encode({ - data: entriesPerHighlightTerm.map((entries) => { + data: entriesPerHighlightTerm.map(({ entries }) => { if (entries.length > 0) { return { entries, From fe16d22ff13629c9c478141abddf8c0e5756f0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 21 Oct 2020 12:04:45 +0200 Subject: [PATCH 08/12] Fix `before` pagination --- .../lib/adapters/log_entries/kibana_log_entries_adapter.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index a999619004a65..98ec7ccc0e609 100644 --- a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -113,13 +113,17 @@ export class InfraKibanaLogEntriesAdapter implements LogEntriesAdapter { esQuery ); - const hits = sortDirection === 'asc' ? esResult.hits.hits : esResult.hits.hits.reverse(); + const hits = esResult.hits.hits; const hasMore = hits.length > size; if (hasMore) { hits.pop(); } + if (sortDirection === 'desc') { + hits.reverse(); + } + return { documents: mapHitsToLogEntryDocuments(hits, fields), hasMoreBefore: sortDirection === 'desc' ? hasMore : undefined, From d770bcf2ff9c9aed0d2b4912682268d2516278eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 10 Nov 2020 15:36:37 +0100 Subject: [PATCH 09/12] Update `react-use` import style --- x-pack/plugins/infra/public/containers/logs/log_stream/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index b890a45caea3f..5de92cdebe8c4 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -5,7 +5,7 @@ */ import { useMemo } from 'react'; -import { useSetState } from 'react-use'; +import useSetState from 'react-use/lib/useSetState'; import { esKuery } from '../../../../../../../src/plugins/data/public'; import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; From 7e5c89248ea1e9cfce55866430abaeca1ff9b90c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 10 Nov 2020 16:18:57 +0100 Subject: [PATCH 10/12] Prevent fetch storms if the UI is already fetching --- x-pack/plugins/infra/public/components/log_stream/index.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/infra/public/components/log_stream/index.tsx b/x-pack/plugins/infra/public/components/log_stream/index.tsx index ac472c68daa6d..45ecf9d5b536c 100644 --- a/x-pack/plugins/infra/public/components/log_stream/index.tsx +++ b/x-pack/plugins/infra/public/components/log_stream/index.tsx @@ -125,6 +125,10 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re return; } + if (isLoadingMore) { + return; + } + if (pagesBeforeStart < PAGE_THRESHOLD) { fetchPreviousEntries(); } else if (pagesAfterEnd < PAGE_THRESHOLD) { From 378803a83d6b866256e3facd9c3e88fe13dd8064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Tue, 10 Nov 2020 16:44:47 +0100 Subject: [PATCH 11/12] Wrap pagination handler with a `useCallback` --- .../public/components/log_stream/index.tsx | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/infra/public/components/log_stream/index.tsx b/x-pack/plugins/infra/public/components/log_stream/index.tsx index 45ecf9d5b536c..20a9fb84b14f0 100644 --- a/x-pack/plugins/infra/public/components/log_stream/index.tsx +++ b/x-pack/plugins/infra/public/components/log_stream/index.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useMemo } from 'react'; +import React, { useMemo, useCallback } from 'react'; import { noop } from 'lodash'; import { useMount } from 'react-use'; import { euiStyled } from '../../../../observability/public'; @@ -97,13 +97,33 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re [entries] ); + const parsedHeight = typeof height === 'number' ? `${height}px` : height; + // Component lifetime useMount(() => { loadSourceConfiguration(); fetchEntries(); }); - const parsedHeight = typeof height === 'number' ? `${height}px` : height; + // Pagination handler + const handlePagination = useCallback( + ({ fromScroll, pagesBeforeStart, pagesAfterEnd }) => { + if (!fromScroll) { + return; + } + + if (isLoadingMore) { + return; + } + + if (pagesBeforeStart < PAGE_THRESHOLD) { + fetchPreviousEntries(); + } else if (pagesAfterEnd < PAGE_THRESHOLD) { + fetchNextEntries(); + } + }, + [isLoadingMore, fetchPreviousEntries, fetchNextEntries] + ); return ( @@ -120,21 +140,7 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re isStreaming={false} lastLoadedTime={null} jumpToTarget={noop} - reportVisibleInterval={({ fromScroll, pagesBeforeStart, pagesAfterEnd }) => { - if (!fromScroll) { - return; - } - - if (isLoadingMore) { - return; - } - - if (pagesBeforeStart < PAGE_THRESHOLD) { - fetchPreviousEntries(); - } else if (pagesAfterEnd < PAGE_THRESHOLD) { - fetchNextEntries(); - } - }} + reportVisibleInterval={handlePagination} loadNewerItems={noop} reloadItems={fetchEntries} highlightedItem={highlight ?? null} From 9b2f2a85027bfae5e643f853e1c48bca714a6096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Wed, 11 Nov 2020 13:01:59 +0100 Subject: [PATCH 12/12] Reset pagination flags when the time range is extended --- .../containers/logs/log_stream/index.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index 5de92cdebe8c4..566edcce91318 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -4,8 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useMemo } from 'react'; +import { useMemo, useEffect } from 'react'; import useSetState from 'react-use/lib/useSetState'; +import usePrevious from 'react-use/lib/usePrevious'; import { esKuery } from '../../../../../../../src/plugins/data/public'; import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; @@ -63,6 +64,22 @@ export function useLogStream({ const { services } = useKibanaContextForPlugin(); const [state, setState] = useSetState(INITIAL_STATE); + // Ensure the pagination keeps working when the timerange gets extended + const prevStartTimestamp = usePrevious(startTimestamp); + const prevEndTimestamp = usePrevious(endTimestamp); + + useEffect(() => { + if (prevStartTimestamp && prevStartTimestamp > startTimestamp) { + setState({ hasMoreBefore: true }); + } + }, [prevStartTimestamp, startTimestamp, setState]); + + useEffect(() => { + if (prevEndTimestamp && prevEndTimestamp < endTimestamp) { + setState({ hasMoreAfter: true }); + } + }, [prevEndTimestamp, endTimestamp, setState]); + const parsedQuery = useMemo(() => { return query ? JSON.stringify(esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query)))