Skip to content

Commit

Permalink
Use advanced settings for leading wildcards in query for csv reports (o…
Browse files Browse the repository at this point in the history
…pensearch-project#549)

* Fetch allowLeadingWildcards from config

Signed-off-by: Rupal Mahajan <[email protected]>

* Fix tests

Signed-off-by: Rupal Mahajan <[email protected]>

* nit

Signed-off-by: Rupal Mahajan <[email protected]>

* add default value for allowLeadingWildCards

Co-authored-by: Joshua Li <[email protected]>
Signed-off-by: Rupal Mahajan <[email protected]>

Signed-off-by: Rupal Mahajan <[email protected]>
Co-authored-by: Joshua Li <[email protected]>
  • Loading branch information
rupal-bq and joshuali925 authored Nov 30, 2022
1 parent 95bc42a commit d7097d6
Show file tree
Hide file tree
Showing 6 changed files with 40 additions and 5 deletions.
2 changes: 2 additions & 0 deletions public/components/utils/settings_service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@ export const uiSettingsService = {
: rawTimeZone;
const dateFormat = this.get('dateFormat');
const csvSeparator = this.get('csv:separator');
const allowLeadingWildcards = this.get('query:allowLeadingWildcards');
return {
timezone,
dateFormat,
csvSeparator,
allowLeadingWildcards,
};
},
};
4 changes: 4 additions & 0 deletions server/routes/lib/createReport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ export const createReport = async (
request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat;
// @ts-ignore
const csvSeparator = request.query.csvSeparator || ',';
// @ts-ignore
const allowLeadingWildcards = !!request.query.allowLeadingWildcards;

const protocol = config.get('osd_server', 'protocol');
const hostname = config.get('osd_server', 'hostname');
const port = config.get('osd_server', 'port');
Expand Down Expand Up @@ -76,6 +79,7 @@ export const createReport = async (
opensearchClient,
dateFormat,
csvSeparator,
allowLeadingWildcards,
isScheduledTask,
logger
);
Expand Down
3 changes: 3 additions & 0 deletions server/routes/report.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ export default function (router: IRouter, config: ReportingConfig) {
timezone: schema.maybe(schema.string()),
dateFormat: schema.maybe(schema.string()),
csvSeparator: schema.maybe(schema.string()),
allowLeadingWildcards: schema.maybe(schema.string()),
}),
},
},
Expand Down Expand Up @@ -98,6 +99,7 @@ export default function (router: IRouter, config: ReportingConfig) {
timezone: schema.string(),
dateFormat: schema.string(),
csvSeparator: schema.string(),
allowLeadingWildcards: schema.string(),
}),
},
},
Expand Down Expand Up @@ -164,6 +166,7 @@ export default function (router: IRouter, config: ReportingConfig) {
timezone: schema.string(),
dateFormat: schema.string(),
csvSeparator: schema.string(),
allowLeadingWildcards: schema.string(),
}),
},
},
Expand Down
17 changes: 17 additions & 0 deletions server/routes/utils/__tests__/savedSearchReportHelper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand All @@ -76,6 +77,7 @@ describe('test create saved search report', () => {
mockOpenSearchClient([]),
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand All @@ -87,6 +89,7 @@ describe('test create saved search report', () => {
mockOpenSearchClient([]),
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand All @@ -101,6 +104,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand All @@ -121,6 +125,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -155,6 +160,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -192,6 +198,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -221,6 +228,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -254,6 +262,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -281,6 +290,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand All @@ -305,6 +315,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
'|',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -338,6 +349,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand All @@ -363,6 +375,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -394,6 +407,7 @@ describe('test create saved search report', () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -421,6 +435,7 @@ test('create report for data set contains null field value', async () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -452,6 +467,7 @@ test('create report for data set with metadata fields', async () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down Expand Up @@ -506,6 +522,7 @@ test('create report with empty/one/multiple(list) date values', async () => {
client,
mockDateFormat,
',',
true,
undefined,
mockLogger
);
Expand Down
12 changes: 9 additions & 3 deletions server/routes/utils/dataReportHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
buildOpenSearchQuery,
Filter,
Query,
OpenSearchQueryConfig,
} from '../../../../../src/plugins/data/common';

export var metaData = {
Expand Down Expand Up @@ -49,16 +50,21 @@ export const getSelectedFields = async (columns) => {

// Build the OpenSearch query from the meta data
// is_count is set to 1 if we building the count query but 0 if we building the fetch data query
export const buildRequestBody = (report: any, is_count: number) => {
export const buildRequestBody = (report: any, allowLeadingWildcards: boolean, is_count: number) => {
let esbBoolQuery = esb.boolQuery();
const searchSourceJSON = report._source.searchSourceJSON;

const savedObjectQuery: Query = JSON.parse(searchSourceJSON).query;
const savedObjectFilter: Filter = JSON.parse(searchSourceJSON).filter;
const savedObjectConfig: OpenSearchQueryConfig = {
allowLeadingWildcards: allowLeadingWildcards,
queryStringOptions: {},
ignoreFilterIfFieldNotInIndex: false,
}
const QueryFromSavedObject = buildOpenSearchQuery(
undefined,
savedObjectQuery,
savedObjectFilter
savedObjectFilter,
savedObjectConfig,
);
// Add time range
if (report._source.timeFieldName && report._source.timeFieldName.length > 0) {
Expand Down
7 changes: 5 additions & 2 deletions server/routes/utils/savedSearchReportHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export async function createSavedSearchReport(
client: ILegacyClusterClient | ILegacyScopedClusterClient,
dateFormat: string,
csvSeparator: string,
allowLeadingWildcards: boolean,
isScheduledTask: boolean = true,
logger: Logger
): Promise<CreateReportResultType> {
Expand All @@ -43,6 +44,7 @@ export async function createSavedSearchReport(
params.core_params,
dateFormat,
csvSeparator,
allowLeadingWildcards,
isScheduledTask,
logger
);
Expand Down Expand Up @@ -130,6 +132,7 @@ async function generateReportData(
params: any,
dateFormat: string,
csvSeparator: string,
allowLeadingWildcards: boolean,
isScheduledTask: boolean,
logger: Logger
) {
Expand All @@ -145,7 +148,7 @@ async function generateReportData(
return '';
}

const reqBody = buildRequestBody(report, 0);
const reqBody = buildRequestBody(report, allowLeadingWildcards, 0);
logger.info(
`[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}`
);
Expand Down Expand Up @@ -182,7 +185,7 @@ async function generateReportData(

// Build the OpenSearch Count query to count the size of result
async function getOpenSearchDataSize() {
const countReq = buildRequestBody(report, 1);
const countReq = buildRequestBody(report, allowLeadingWildcards, 1);
return await callCluster(
client,
'count',
Expand Down

0 comments on commit d7097d6

Please sign in to comment.