From d204e9c83b07b7b535de06d39a7dd19c5672a314 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Mon, 19 Oct 2020 10:34:24 -0700 Subject: [PATCH 1/4] improve input validation --- .../components/context_menu/context_menu.js | 42 ++-- .../context_menu/context_menu_helpers.js | 85 ++++---- .../public/components/main/main_utils.tsx | 4 +- .../report_definition_details.tsx | 2 +- .../main/report_details/report_details.tsx | 2 +- .../report_settings_helpers.tsx | 6 +- kibana-reports/server/model/index.ts | 75 +++++-- kibana-reports/server/routes/report.ts | 1 + .../utils/__tests__/reportHelper.test.ts | 2 +- kibana-reports/server/routes/utils/helpers.ts | 61 ++++++ .../server/routes/utils/reportHelper.ts | 200 +----------------- kibana-reports/server/routes/utils/types.ts | 15 ++ .../server/routes/utils/visualReportHelper.ts | 149 +++++++++++++ kibana-reports/server/utils/constants.ts | 2 +- kibana-reports/server/utils/executor.ts | 13 +- .../server/utils/validationHelper.ts | 36 ++++ 16 files changed, 399 insertions(+), 296 deletions(-) create mode 100644 kibana-reports/server/routes/utils/visualReportHelper.ts create mode 100644 kibana-reports/server/utils/validationHelper.ts diff --git a/kibana-reports/public/components/context_menu/context_menu.js b/kibana-reports/public/components/context_menu/context_menu.js index 0e980e58..9a4b634e 100644 --- a/kibana-reports/public/components/context_menu/context_menu.js +++ b/kibana-reports/public/components/context_menu/context_menu.js @@ -24,19 +24,20 @@ import { addSuccessOrFailureToast, contextMenuViewReports, } from './context_menu_helpers'; -import { popoverMenu, popoverMenuDiscover, getMenuItem } from './context_menu_ui'; +import { + popoverMenu, + popoverMenuDiscover, + getMenuItem, +} from './context_menu_ui'; const replaceQueryURL = () => { - let url = window.location.href; + let url = location.pathname + location.hash; let timeString = url.substring( url.lastIndexOf('time:'), url.lastIndexOf('))') ); - if (url.includes("visualize") || url.includes("discover")) { - timeString = url.substring( - url.lastIndexOf("time:"), - url.indexOf("))") - ); + if (url.includes('visualize') || url.includes('discover')) { + timeString = url.substring(url.lastIndexOf('time:'), url.indexOf('))')); } let fromDateString = timeString.substring( @@ -68,18 +69,20 @@ const replaceQueryURL = () => { return url; }; -const generateInContextReport = (timeRanges, queryUrl, fileFormat, rest = {}) => { +const generateInContextReport = ( + timeRanges, + queryUrl, + fileFormat, + rest = {} +) => { displayLoadingModal(); - let baseUrl = window.location.href.substr( - 0, - window.location.href.indexOf('?') - ); + const baseUrl = queryUrl.substr(0, queryUrl.indexOf('?')); let reportSource = ''; - if (window.location.href.includes('dashboard')) { + if (baseUrl.includes('dashboard')) { reportSource = 'Dashboard'; - } else if (window.location.href.includes('visualize')) { + } else if (baseUrl.includes('visualize')) { reportSource = 'Visualization'; - } else if (window.location.href.includes('discover')) { + } else if (baseUrl.includes('discover')) { reportSource = 'Saved search'; } @@ -150,7 +153,9 @@ $(function () { if (popoverScreen) { try { const reportPopover = document.createElement('div'); - reportPopover.innerHTML = isDiscover() ? popoverMenuDiscover(getUuidFromUrl()) : popoverMenu(); + reportPopover.innerHTML = isDiscover() + ? popoverMenuDiscover(getUuidFromUrl()) + : popoverMenu(); popoverScreen[0].appendChild(reportPopover.children[0]); $('#reportPopover').show(); } catch (e) { @@ -254,7 +259,10 @@ function locationHashChanged() { } // try to match uuid followed by '?' in URL, which would be the saved search id for discover URL -const getUuidFromUrl = () => window.location.href.match(/(\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)\?/); +const getUuidFromUrl = () => + window.location.href.match( + /(\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)\?/ + ); const isDiscover = () => window.location.href.includes('discover'); window.onhashchange = function () { diff --git a/kibana-reports/public/components/context_menu/context_menu_helpers.js b/kibana-reports/public/components/context_menu/context_menu_helpers.js index 963f849d..b7447ac2 100644 --- a/kibana-reports/public/components/context_menu/context_menu_helpers.js +++ b/kibana-reports/public/components/context_menu/context_menu_helpers.js @@ -18,67 +18,63 @@ import moment from 'moment'; import { reportGenerationInProgressModal, reportGenerationSuccess, - reportGenerationFailure + reportGenerationFailure, } from './context_menu_ui'; const getReportSourceURL = (baseURI) => { - let url = baseURI.substr(0, baseURI.indexOf("?")); - const reportSourceId = url.substr(url.lastIndexOf("/") + 1, url.length); + let url = baseURI.substr(0, baseURI.indexOf('?')); + const reportSourceId = url.substr(url.lastIndexOf('/') + 1, url.length); return reportSourceId; -} +}; export const contextMenuCreateReportDefinition = (baseURI) => { const reportSourceId = getReportSourceURL(baseURI); - let reportSource = ""; + let reportSource = ''; let timeRanges = getTimeFieldsFromUrl(); // check report source if (baseURI.includes('dashboard')) { reportSource = 'dashboard:'; - } - else if (baseURI.includes('visualize')) { - reportSource = 'visualize:' - } - else if (baseURI.includes('discover')) { + } else if (baseURI.includes('visualize')) { + reportSource = 'visualize:'; + } else if (baseURI.includes('discover')) { reportSource = 'discover:'; } reportSource += reportSourceId.toString(); window.location.assign( `opendistro_kibana_reports#/create?previous=${reportSource}?timeFrom=${timeRanges.time_from.toISOString()}?timeTo=${timeRanges.time_to.toISOString()}` - ) -} + ); +}; -export const contextMenuViewReports = () => window.location.assign('opendistro_kibana_reports#/'); +export const contextMenuViewReports = () => + window.location.assign('opendistro_kibana_reports#/'); export const getTimeFieldsFromUrl = () => { let url = window.location.href; let timeString = url.substring( - url.lastIndexOf("time:"), - url.lastIndexOf("))") - ); - if (url.includes("visualize") || url.includes("discover")) { - timeString = url.substring( - url.lastIndexOf("time:"), - url.indexOf("))") - ); + url.lastIndexOf('time:'), + url.lastIndexOf('))') + ); + if (url.includes('visualize') || url.includes('discover')) { + timeString = url.substring(url.lastIndexOf('time:'), url.indexOf('))')); } let fromDateString = timeString.substring( - timeString.lastIndexOf("from:") + 5, - timeString.lastIndexOf(",") + timeString.lastIndexOf('from:') + 5, + timeString.lastIndexOf(',') ); - + // remove extra quotes if the 'from' date is absolute time fromDateString = fromDateString.replace(/[']+/g, ''); - + // convert time range to from date format in case time range is relative let fromDateFormat = dateMath.parse(fromDateString); let toDateString = timeString.substring( - timeString.lastIndexOf("to:") + 3, + timeString.lastIndexOf('to:') + 3, timeString.length ); - + toDateString = toDateString.replace(/[']+/g, ''); let toDateFormat = dateMath.parse(toDateString); @@ -89,39 +85,42 @@ export const getTimeFieldsFromUrl = () => { return { time_from: fromDateFormat, time_to: toDateFormat, - time_duration: timeDuration.toISOString() - } -} + time_duration: timeDuration.toISOString(), + }; +}; export const displayLoadingModal = () => { - const kibanaBody = document.getElementById("kibana-body"); + const kibanaBody = document.getElementById('kibana-body'); if (kibanaBody) { try { - const loadingModal = document.createElement("div"); + const loadingModal = document.createElement('div'); loadingModal.innerHTML = reportGenerationInProgressModal(); kibanaBody.appendChild(loadingModal.children[0]); } catch (e) { - console.log("error displaying loading modal:", e); + console.log('error displaying loading modal:', e); } } -} +}; export const addSuccessOrFailureToast = (status) => { - const generateToast = document.querySelectorAll(".euiGlobalToastList"); + const generateToast = document.querySelectorAll('.euiGlobalToastList'); if (generateToast) { try { - const generateInProgressToast = document.createElement("div"); - if (status === "success") { + const generateInProgressToast = document.createElement('div'); + if (status === 'success') { generateInProgressToast.innerHTML = reportGenerationSuccess(); - setTimeout(function () {document.getElementById('reportSuccessToast').style.display='none'}, 6000); // closes toast automatically after 6s - } - else if (status === "failure") { + setTimeout(function () { + document.getElementById('reportSuccessToast').style.display = 'none'; + }, 6000); // closes toast automatically after 6s + } else if (status === 'failure') { generateInProgressToast.innerHTML = reportGenerationFailure(); - setTimeout(function () {document.getElementById('reportFailureToast').style.display='none'}, 6000); + setTimeout(function () { + document.getElementById('reportFailureToast').style.display = 'none'; + }, 6000); } generateToast[0].appendChild(generateInProgressToast.children[0]); } catch (e) { - console.log("error displaying toast", e); + console.log('error displaying toast', e); } } -} \ No newline at end of file +}; diff --git a/kibana-reports/public/components/main/main_utils.tsx b/kibana-reports/public/components/main/main_utils.tsx index f1828031..ed7854be 100644 --- a/kibana-reports/public/components/main/main_utils.tsx +++ b/kibana-reports/public/components/main/main_utils.tsx @@ -63,7 +63,7 @@ export const addReportsTableContent = (data) => { //TODO: wrong name timeCreated: report.time_created, state: report.state, - url: report.query_url, + url: `${location.host}${report.query_url}`, format: reportParams.core_params.report_format, }; reportsTableItems.push(reportsTableEntry); @@ -85,7 +85,7 @@ export const addReportDefinitionsTableContent = (data: any) => { type: trigger.trigger_type, owner: `\u2014`, // Todo: replace source: reportParams.report_source, - baseUrl: reportParams.core_params.base_url, + baseUrl: `${location.host}${reportParams.core_params.base_url}`, lastUpdated: reportDefinition.last_updated, details: trigger.trigger_type === 'On demand' diff --git a/kibana-reports/public/components/main/report_definition_details/report_definition_details.tsx b/kibana-reports/public/components/main/report_definition_details/report_definition_details.tsx index d9398ea4..2262be4e 100644 --- a/kibana-reports/public/components/main/report_definition_details/report_definition_details.tsx +++ b/kibana-reports/public/components/main/report_definition_details/report_definition_details.tsx @@ -283,7 +283,7 @@ export function ReportDefinitionDetails(props) { const sourceURL = (data) => { return ( - + {data['source']} ); diff --git a/kibana-reports/public/components/main/report_details/report_details.tsx b/kibana-reports/public/components/main/report_details/report_details.tsx index 48ba9d63..00f96e09 100644 --- a/kibana-reports/public/components/main/report_details/report_details.tsx +++ b/kibana-reports/public/components/main/report_details/report_details.tsx @@ -189,7 +189,7 @@ export function ReportDetails(props) { const sourceURL = (data) => { return ( - + {data['source']} ); diff --git a/kibana-reports/public/components/report_definitions/report_settings/report_settings_helpers.tsx b/kibana-reports/public/components/report_definitions/report_settings/report_settings_helpers.tsx index 99f47dbf..189320a1 100644 --- a/kibana-reports/public/components/report_definitions/report_settings/report_settings_helpers.tsx +++ b/kibana-reports/public/components/report_definitions/report_settings/report_settings_helpers.tsx @@ -29,7 +29,7 @@ export const getDashboardBaseUrlCreate = ( edit: boolean, editDefinitionId: string ) => { - let baseUrl = window.location.href; + let baseUrl = location.pathname + location.hash; if (edit) { return baseUrl.replace( `opendistro_kibana_reports#/edit/${editDefinitionId}`, @@ -43,7 +43,7 @@ export const getDashboardBaseUrlCreate = ( }; export const getVisualizationBaseUrlCreate = (edit: boolean) => { - let baseUrl = window.location.href; + let baseUrl = location.pathname + location.hash; if (edit) { return baseUrl.replace( 'opendistro_kibana_reports#/edit', @@ -57,7 +57,7 @@ export const getVisualizationBaseUrlCreate = (edit: boolean) => { }; export const getSavedSearchBaseUrlCreate = (edit: boolean) => { - let baseUrl = window.location.href; + let baseUrl = location.pathname + location.hash; if (edit) { return baseUrl.replace( 'opendistro_kibana_reports#/edit', diff --git a/kibana-reports/server/model/index.ts b/kibana-reports/server/model/index.ts index 373982b3..8d91f70a 100644 --- a/kibana-reports/server/model/index.ts +++ b/kibana-reports/server/model/index.ts @@ -14,6 +14,8 @@ */ import { schema, TypeOf } from '@kbn/config-schema'; +import { isValidDuration, isValidRelativeUrl } from '../utils/validationHelper'; +import { isValidCron } from 'cron-validator'; import { REPORT_TYPE, TRIGGER_TYPE, @@ -27,32 +29,56 @@ import { } from '../routes/utils/constants'; export const dataReportSchema = schema.object({ - base_url: schema.uri(), + base_url: schema.string({ + validate(value) { + if (!isValidRelativeUrl(value)) { + return `invalid relative url: ${value}`; + } + }, + }), saved_search_id: schema.string(), //ISO duration format. 'PT10M' means 10 min - time_duration: schema.string(), + time_duration: schema.string({ + validate(value) { + if (!isValidDuration(value)) { + return `invalid time duration: ${value}`; + } + }, + }), //TODO: future support schema.literal('xlsx') report_format: schema.oneOf([schema.literal(FORMAT.csv)]), - limit: schema.number({ defaultValue: DEFAULT_MAX_SIZE }), + limit: schema.number({ defaultValue: DEFAULT_MAX_SIZE, min: 0 }), excel: schema.boolean({ defaultValue: true }), }); const visualReportSchema = schema.object({ - base_url: schema.uri(), - window_width: schema.number({ defaultValue: 1200 }), - window_height: schema.number({ defaultValue: 800 }), + base_url: schema.string({ + validate(value) { + if (!isValidRelativeUrl(value)) { + return `invalid relative url: ${value}`; + } + }, + }), + window_width: schema.number({ defaultValue: 1200, min: 0 }), + window_height: schema.number({ defaultValue: 800, min: 0 }), report_format: schema.oneOf([ schema.literal(FORMAT.pdf), schema.literal(FORMAT.png), ]), header: schema.maybe(schema.string()), footer: schema.maybe(schema.string()), - time_duration: schema.string(), + time_duration: schema.string({ + validate(value) { + if (!isValidDuration(value)) { + return `invalid time duration: ${value}`; + } + }, + }), }); export const intervalSchema = schema.object({ interval: schema.object({ - period: schema.number(), + period: schema.number({ min: 0 }), // Refer to job scheduler SPI https://github.com/opendistro-for-elasticsearch/job-scheduler/blob/b333469c183a15ddbf496a690300cc9e34d937fb/spi/src/main/java/com/amazon/opendistroforelasticsearch/jobscheduler/spi/schedule/IntervalSchedule.java unit: schema.oneOf([ schema.literal('MINUTES'), @@ -66,7 +92,14 @@ export const intervalSchema = schema.object({ export const cronSchema = schema.object({ cron: schema.object({ - expression: schema.string(), + expression: schema.string({ + validate(value) { + if (!isValidCron(value)) { + return `invalid cron expression: ${value}`; + } + }, + }), + //TODO: add more validation once we add full support of timezone timezone: schema.string(), }), }); @@ -74,7 +107,7 @@ export const cronSchema = schema.object({ export const scheduleSchema = schema.object({ schedule_type: schema.oneOf([ /* - TODO: Future Date will be added in the future. + TODO: Future Date option will be added in the future. Currently @kbn/config-schema has no support for more than 2 conditions, keep an eye on library update */ schema.literal(SCHEDULE_TYPE.recurring), @@ -96,9 +129,9 @@ export const kibanaUserSchema = schema.object({ export const channelSchema = schema.object({ recipients: schema.arrayOf(schema.string(), { minSize: 1 }), - title: schema.string(), - textDescription: schema.string(), - htmlDescription: schema.maybe(schema.string()), + title: schema.string({ minLength: 1 }), + textDescription: schema.string({ minLength: 1 }), + htmlDescription: schema.maybe(schema.string({ minLength: 1 })), email_format: schema.oneOf([ schema.literal(EMAIL_FORMAT.attachment), schema.literal(EMAIL_FORMAT.embeddedHtml), @@ -121,13 +154,13 @@ export const deliverySchema = schema.object({ export const reportDefinitionSchema = schema.object({ report_params: schema.object({ - report_name: schema.string(), + report_name: schema.string({ minLength: 1 }), report_source: schema.oneOf([ schema.literal(REPORT_TYPE.dashboard), schema.literal(REPORT_TYPE.visualization), schema.literal(REPORT_TYPE.savedSearch), ]), - description: schema.string(), + description: schema.string({ minLength: 1 }), core_params: schema.conditional( schema.siblingRef('report_source'), REPORT_TYPE.savedSearch, @@ -135,9 +168,7 @@ export const reportDefinitionSchema = schema.object({ visualReportSchema ), }), - delivery: deliverySchema, - trigger: schema.object({ trigger_type: schema.oneOf([ /* @@ -166,8 +197,14 @@ export const reportDefinitionSchema = schema.object({ }); export const reportSchema = schema.object({ - query_url: schema.uri(), - time_from: schema.number(), + query_url: schema.string({ + validate(value) { + if (!isValidRelativeUrl(value)) { + return `this is invalid relative url: ${value}`; + } + }, + }), + time_from: schema.number({ min: 1 }), time_to: schema.number(), report_definition: reportDefinitionSchema, diff --git a/kibana-reports/server/routes/report.ts b/kibana-reports/server/routes/report.ts index 3ddef29f..67a500f8 100644 --- a/kibana-reports/server/routes/report.ts +++ b/kibana-reports/server/routes/report.ts @@ -30,6 +30,7 @@ import { CONFIG_INDEX_NAME, DEFAULT_MAX_SIZE, DELIVERY_TYPE, + LOCAL_HOST, } from './utils/constants'; export default function (router: IRouter) { diff --git a/kibana-reports/server/routes/utils/__tests__/reportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/reportHelper.test.ts index 08d6384a..73cf1619 100644 --- a/kibana-reports/server/routes/utils/__tests__/reportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/reportHelper.test.ts @@ -14,7 +14,7 @@ */ import 'regenerator-runtime/runtime'; -import { createVisualReport } from '../reportHelper'; +import { createVisualReport } from '../visualReportHelper'; describe('test create visual report', () => { test('create png report', async () => { diff --git a/kibana-reports/server/routes/utils/helpers.ts b/kibana-reports/server/routes/utils/helpers.ts index 4a21fc6c..6e27821e 100644 --- a/kibana-reports/server/routes/utils/helpers.ts +++ b/kibana-reports/server/routes/utils/helpers.ts @@ -19,6 +19,10 @@ import { ILegacyClusterClient, ILegacyScopedClusterClient, } from '../../../../../src/core/server'; +import { RequestParams } from '@elastic/elasticsearch'; +import { CONFIG_INDEX_NAME, REPORT_STATE } from './constants'; +import { CreateReportResultType } from './types'; +import { ReportSchemaType } from 'server/model'; export function parseEsErrorResponse(error: any) { if (error.response) { @@ -68,3 +72,60 @@ export const callCluster = async ( } return esResp; }; + +export const saveToES = async ( + isScheduledTask: boolean, + report: ReportSchemaType, + esClient: ILegacyClusterClient | ILegacyScopedClusterClient +) => { + const timePending = Date.now(); + const saveParams: RequestParams.Index = { + index: CONFIG_INDEX_NAME.report, + body: { + ...report, + state: REPORT_STATE.pending, + last_updated: timePending, + time_created: timePending, + }, + }; + const esResp = await callCluster( + esClient, + 'index', + saveParams, + isScheduledTask + ); + + return esResp; +}; + +export const updateToES = async ( + isScheduledTask: boolean, + reportId: string, + esClient: ILegacyClusterClient | ILegacyScopedClusterClient, + state: string, + createReportResult?: CreateReportResultType +) => { + const timeStamp = createReportResult + ? createReportResult.timeCreated + : Date.now(); + // update report document with state "created" or "error" + const updateParams: RequestParams.Update = { + id: reportId, + index: CONFIG_INDEX_NAME.report, + body: { + doc: { + state: state, + last_updated: timeStamp, + time_created: timeStamp, + }, + }, + }; + const esResp = await callCluster( + esClient, + 'update', + updateParams, + isScheduledTask + ); + + return esResp; +}; diff --git a/kibana-reports/server/routes/utils/reportHelper.ts b/kibana-reports/server/routes/utils/reportHelper.ts index 7c172150..40ab77be 100644 --- a/kibana-reports/server/routes/utils/reportHelper.ts +++ b/kibana-reports/server/routes/utils/reportHelper.ts @@ -13,22 +13,15 @@ * permissions and limitations under the License. */ -import puppeteer from 'puppeteer'; -import createDOMPurify from 'dompurify'; -import { JSDOM } from 'jsdom'; import { FORMAT, REPORT_TYPE, REPORT_STATE, - CONFIG_INDEX_NAME, LOCAL_HOST, DELIVERY_TYPE, EMAIL_FORMAT, - DEFAULT_REPORT_FOOTER, - DEFAULT_REPORT_HEADER, } from './constants'; -import { RequestParams } from '@elastic/elasticsearch'; -import { getFileName, callCluster } from './helpers'; +import { callCluster, updateToES, saveToES } from './helpers'; import { ILegacyClusterClient, ILegacyScopedClusterClient, @@ -37,123 +30,7 @@ import { import { createSavedSearchReport } from './savedSearchReportHelper'; import { ReportSchemaType } from '../../model'; import { CreateReportResultType } from './types'; - -const window = new JSDOM('').window; -const DOMPurify = createDOMPurify(window); - -export const createVisualReport = async ( - reportParams: any, - queryUrl: string, - logger: Logger -): Promise => { - const coreParams = reportParams.core_params; - // parse params - const reportSource = reportParams.report_source; - const reportName = reportParams.report_name; - const windowWidth = coreParams.window_width; - const windowHeight = coreParams.window_height; - const reportFormat = coreParams.report_format; - - // TODO: polish default header, maybe add a logo, depends on UX design - const header = coreParams.header ? DOMPurify.sanitize(coreParams.header) : DEFAULT_REPORT_HEADER; - const footer = coreParams.footer ? DOMPurify.sanitize(coreParams.footer) : DEFAULT_REPORT_FOOTER; - // set up puppeteer - const browser = await puppeteer.launch({ - headless: true, - /** - * TODO: temp fix to disable sandbox when launching chromium on Linux instance - * https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#setting-up-chrome-linux-sandbox - */ - args: ['--no-sandbox', '--disable-setuid-sandbox'], - }); - const page = await browser.newPage(); - page.setDefaultNavigationTimeout(0); - page.setDefaultTimeout(60000); // use 60s timeout instead of default 30s - logger.info(`original queryUrl ${queryUrl}`); - await page.goto(queryUrl, { waitUntil: 'networkidle0' }); - logger.info(`page url ${page.url()}`); - logger.info(`page url includes login? ${page.url().includes('login')}`); - - /** - * TODO: This is a workaround to simulate a login to security enabled domain. - * Need better handle. - */ - if (page.url().includes('login')) { - logger.info( - 'domain enables security, redirecting to login page, start simulating login' - ); - await page.type('[placeholder=Username]', 'admin', { delay: 30 }); - await page.type('[placeholder=Password]', 'admin', { delay: 30 }); - await page.click("[type='submit']"); - await page.waitForNavigation(); - logger.info( - `Done logging in, currently at page: ${page.url()} \nGo to queryUrl again` - ); - await page.goto(queryUrl, { waitUntil: 'networkidle0' }); - logger.info(`wait for network idle, the current page url: ${page.url()}`); - } - - await page.setViewport({ - width: windowWidth, - height: windowHeight, - }); - - let buffer; - let element; - // crop content - if (reportSource === REPORT_TYPE.dashboard) { - await page.waitForSelector('#dashboardViewport'); - element = await page.$('#dashboardViewport'); - } else if (reportSource === REPORT_TYPE.visualization) { - await page.waitForSelector('.visChart'); - element = await page.$('.visChart'); - } - - const screenshot = await element.screenshot({ fullPage: false }); - - /** - * Sets the content of the page to have the header be above the trimmed screenshot - * and the footer be below it - */ - // TODO: need to convert header from markdown to html, either do it on server side, or on client side. - // Email body conversion is done from client side - await page.setContent(` - - -
- ${header} - - ${footer} -
- - `); - - // create pdf or png accordingly - if (reportFormat === FORMAT.pdf) { - const scrollHeight = await page.evaluate( - () => document.documentElement.scrollHeight - ); - - buffer = await page.pdf({ - margin: undefined, - width: windowWidth, - height: scrollHeight + 'px', - printBackground: true, - pageRanges: '1', - }); - } else if (reportFormat === FORMAT.png) { - buffer = await page.screenshot({ - fullPage: true, - }); - } - - const curTime = new Date(); - const timeCreated = curTime.valueOf(); - const fileName = `${getFileName(reportName, curTime)}.${reportFormat}`; - await browser.close(); - - return { timeCreated, dataUrl: buffer.toString('base64'), fileName }; -}; +import { createVisualReport } from './visualReportHelper'; export const createReport = async ( isScheduledTask: boolean, @@ -177,20 +54,8 @@ export const createReport = async ( const reportParams = reportDefinition.report_params; const reportSource = reportParams.report_source; - // validate url - let queryUrl = report.query_url; - const { origin } = new URL(report.query_url); - /** - * Only URL in WHATWG format is accepted. - * e.g http://dasd@test.com/random?size=50 will be considered as invalid and throw error, due to the "@" symbol - * is not part of the url origin according to https://nodejs.org/api/url.html#url_url_strings_and_url_objects - */ - if (report.query_url.search(origin) >= 0) { - queryUrl = report.query_url.replace(origin, LOCAL_HOST); - } else { - throw Error(`query url is not valid: ${queryUrl}`); - } - + // compose url + const queryUrl = `${LOCAL_HOST}${report.query_url}`; try { // generate report if (reportSource === REPORT_TYPE.savedSearch) { @@ -241,63 +106,6 @@ export const createReport = async ( return createReportResult; }; -export const saveToES = async ( - isScheduledTask: boolean, - report: ReportSchemaType, - esClient: ILegacyClusterClient | ILegacyScopedClusterClient -) => { - const timePending = Date.now(); - const saveParams: RequestParams.Index = { - index: CONFIG_INDEX_NAME.report, - body: { - ...report, - state: REPORT_STATE.pending, - last_updated: timePending, - time_created: timePending, - }, - }; - const esResp = await callCluster( - esClient, - 'index', - saveParams, - isScheduledTask - ); - - return esResp; -}; - -export const updateToES = async ( - isScheduledTask: boolean, - reportId: string, - esClient: ILegacyClusterClient | ILegacyScopedClusterClient, - state: string, - createReportResult?: CreateReportResultType -) => { - const timeStamp = createReportResult - ? createReportResult.timeCreated - : Date.now(); - // update report document with state "created" or "error" - const updateParams: RequestParams.Update = { - id: reportId, - index: CONFIG_INDEX_NAME.report, - body: { - doc: { - state: state, - last_updated: timeStamp, - time_created: timeStamp, - }, - }, - }; - const esResp = await callCluster( - esClient, - 'update', - updateParams, - isScheduledTask - ); - - return esResp; -}; - export const deliverReport = async ( report: ReportSchemaType, reportData: CreateReportResultType, diff --git a/kibana-reports/server/routes/utils/types.ts b/kibana-reports/server/routes/utils/types.ts index 88d9617b..1f9226a2 100644 --- a/kibana-reports/server/routes/utils/types.ts +++ b/kibana-reports/server/routes/utils/types.ts @@ -1,3 +1,18 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + export interface CreateReportResultType { timeCreated: number; dataUrl: string; diff --git a/kibana-reports/server/routes/utils/visualReportHelper.ts b/kibana-reports/server/routes/utils/visualReportHelper.ts new file mode 100644 index 00000000..47c7ddc9 --- /dev/null +++ b/kibana-reports/server/routes/utils/visualReportHelper.ts @@ -0,0 +1,149 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import puppeteer from 'puppeteer'; +import createDOMPurify from 'dompurify'; +import { JSDOM } from 'jsdom'; +import { Logger } from '../../../../../src/core/server'; +import { + DEFAULT_REPORT_HEADER, + DEFAULT_REPORT_FOOTER, + REPORT_TYPE, + FORMAT, +} from './constants'; +import { getFileName } from './helpers'; +import { CreateReportResultType } from './types'; + +export const createVisualReport = async ( + reportParams: any, + queryUrl: string, + logger: Logger +): Promise => { + const coreParams = reportParams.core_params; + // parse params + const reportSource = reportParams.report_source; + const reportName = reportParams.report_name; + const windowWidth = coreParams.window_width; + const windowHeight = coreParams.window_height; + const reportFormat = coreParams.report_format; + + // TODO: polish default header, maybe add a logo, depends on UX design + const window = new JSDOM('').window; + const DOMPurify = createDOMPurify(window); + + const header = coreParams.header + ? DOMPurify.sanitize(coreParams.header) + : DEFAULT_REPORT_HEADER; + const footer = coreParams.footer + ? DOMPurify.sanitize(coreParams.footer) + : DEFAULT_REPORT_FOOTER; + + // set up puppeteer + const browser = await puppeteer.launch({ + headless: true, + /** + * TODO: temp fix to disable sandbox when launching chromium on Linux instance + * https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#setting-up-chrome-linux-sandbox + */ + args: ['--no-sandbox', '--disable-setuid-sandbox'], + }); + const page = await browser.newPage(); + page.setDefaultNavigationTimeout(0); + page.setDefaultTimeout(60000); // use 60s timeout instead of default 30s + logger.info(`original queryUrl ${queryUrl}`); + await page.goto(queryUrl, { waitUntil: 'networkidle0' }); + logger.info(`page url ${page.url()}`); + logger.info(`page url includes login? ${page.url().includes('login')}`); + + /** + * TODO: This is a workaround to simulate a login to security enabled domain. + * Need better handle. + */ + if (page.url().includes('login')) { + logger.info( + 'domain enables security, redirecting to login page, start simulating login' + ); + await page.type('[placeholder=Username]', 'admin', { delay: 30 }); + await page.type('[placeholder=Password]', 'admin', { delay: 30 }); + await page.click("[type='submit']"); + await page.waitForNavigation(); + logger.info( + `Done logging in, currently at page: ${page.url()} \nGo to queryUrl again` + ); + await page.goto(queryUrl, { waitUntil: 'networkidle0' }); + logger.info(`wait for network idle, the current page url: ${page.url()}`); + } + + await page.setViewport({ + width: windowWidth, + height: windowHeight, + }); + + let buffer; + let element; + // crop content + if (reportSource === REPORT_TYPE.dashboard) { + await page.waitForSelector('#dashboardViewport'); + element = await page.$('#dashboardViewport'); + } else if (reportSource === REPORT_TYPE.visualization) { + await page.waitForSelector('.visChart'); + element = await page.$('.visChart'); + } + + const screenshot = await element.screenshot({ fullPage: false }); + + /** + * Sets the content of the page to have the header be above the trimmed screenshot + * and the footer be below it + */ + // TODO: need to convert header from markdown to html, either do it on server side, or on client side. + // Email body conversion is done from client side + await page.setContent(` + + +
+ ${header} + + ${footer} +
+ + `); + + // create pdf or png accordingly + if (reportFormat === FORMAT.pdf) { + const scrollHeight = await page.evaluate( + () => document.documentElement.scrollHeight + ); + + buffer = await page.pdf({ + margin: undefined, + width: windowWidth, + height: scrollHeight + 'px', + printBackground: true, + pageRanges: '1', + }); + } else if (reportFormat === FORMAT.png) { + buffer = await page.screenshot({ + fullPage: true, + }); + } + + const curTime = new Date(); + const timeCreated = curTime.valueOf(); + const fileName = `${getFileName(reportName, curTime)}.${reportFormat}`; + await browser.close(); + + return { timeCreated, dataUrl: buffer.toString('base64'), fileName }; +}; diff --git a/kibana-reports/server/utils/constants.ts b/kibana-reports/server/utils/constants.ts index a6858ad3..d81dbdbf 100644 --- a/kibana-reports/server/utils/constants.ts +++ b/kibana-reports/server/utils/constants.ts @@ -13,4 +13,4 @@ * permissions and limitations under the License. */ -export const POLL_INTERVAL = 1000 * 60 * 5; // in milliseconds (5 min) +export const POLL_INTERVAL = 1000 * 60 * 0.5; // in milliseconds (5 min) diff --git a/kibana-reports/server/utils/executor.ts b/kibana-reports/server/utils/executor.ts index 30c1bd18..621a1f92 100644 --- a/kibana-reports/server/utils/executor.ts +++ b/kibana-reports/server/utils/executor.ts @@ -23,6 +23,7 @@ import { } from '../model'; import moment from 'moment'; import { CONFIG_INDEX_NAME } from '../routes/utils/constants'; +import { parseEsErrorResponse } from '../routes/utils/helpers'; async function pollAndExecuteJob( schedulerClient: ILegacyClusterClient, @@ -136,16 +137,4 @@ function createReportMetaData( return report; } -function parseEsErrorResponse(error: any) { - if (error.response) { - try { - const esErrorResponse = JSON.parse(error.response); - return esErrorResponse.reason || error.response; - } catch (parsingError) { - return error.response; - } - } - return error.message; -} - export { pollAndExecuteJob }; diff --git a/kibana-reports/server/utils/validationHelper.ts b/kibana-reports/server/utils/validationHelper.ts new file mode 100644 index 00000000..0f20973b --- /dev/null +++ b/kibana-reports/server/utils/validationHelper.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { LOCAL_HOST } from '../routes/utils/constants'; + +export const isValidRelativeUrl = (relativeUrl: string) => { + try { + new URL(`${LOCAL_HOST}${relativeUrl}`); + } catch (_) { + return false; + } + return true; +}; + +/** + * moment.js isValid() API fails to validate time duration, so use regex + * https://github.com/moment/moment/issues/1805 + **/ + +export const isValidDuration = (duration: string) => { + return !!duration.match( + /^(-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$/ + ); +}; From 77a4a11d80f0ef82e7394d19f13be39d040a6062 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Mon, 19 Oct 2020 12:19:00 -0700 Subject: [PATCH 2/4] clean up --- kibana-reports/server/routes/report.ts | 1 - kibana-reports/server/utils/constants.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/kibana-reports/server/routes/report.ts b/kibana-reports/server/routes/report.ts index 67a500f8..3ddef29f 100644 --- a/kibana-reports/server/routes/report.ts +++ b/kibana-reports/server/routes/report.ts @@ -30,7 +30,6 @@ import { CONFIG_INDEX_NAME, DEFAULT_MAX_SIZE, DELIVERY_TYPE, - LOCAL_HOST, } from './utils/constants'; export default function (router: IRouter) { diff --git a/kibana-reports/server/utils/constants.ts b/kibana-reports/server/utils/constants.ts index d81dbdbf..a6858ad3 100644 --- a/kibana-reports/server/utils/constants.ts +++ b/kibana-reports/server/utils/constants.ts @@ -13,4 +13,4 @@ * permissions and limitations under the License. */ -export const POLL_INTERVAL = 1000 * 60 * 0.5; // in milliseconds (5 min) +export const POLL_INTERVAL = 1000 * 60 * 5; // in milliseconds (5 min) From 751d61d34b6fcbdb159fe7776cc36393c9a24bc4 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Mon, 19 Oct 2020 16:26:08 -0700 Subject: [PATCH 3/4] validate report name and email address --- kibana-reports/server/model/index.ts | 32 +++++++++++++++---- .../server/utils/validationHelper.ts | 9 ++---- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/kibana-reports/server/model/index.ts b/kibana-reports/server/model/index.ts index 8d91f70a..eb4707b1 100644 --- a/kibana-reports/server/model/index.ts +++ b/kibana-reports/server/model/index.ts @@ -14,7 +14,12 @@ */ import { schema, TypeOf } from '@kbn/config-schema'; -import { isValidDuration, isValidRelativeUrl } from '../utils/validationHelper'; +import { + isValidRelativeUrl, + regexDuration, + regexEmailAddress, + regexReportName, +} from '../utils/validationHelper'; import { isValidCron } from 'cron-validator'; import { REPORT_TYPE, @@ -40,7 +45,7 @@ export const dataReportSchema = schema.object({ //ISO duration format. 'PT10M' means 10 min time_duration: schema.string({ validate(value) { - if (!isValidDuration(value)) { + if (!regexDuration.test(value)) { return `invalid time duration: ${value}`; } }, @@ -69,7 +74,7 @@ const visualReportSchema = schema.object({ footer: schema.maybe(schema.string()), time_duration: schema.string({ validate(value) { - if (!isValidDuration(value)) { + if (!regexDuration.test(value)) { return `invalid time duration: ${value}`; } }, @@ -128,7 +133,16 @@ export const kibanaUserSchema = schema.object({ }); export const channelSchema = schema.object({ - recipients: schema.arrayOf(schema.string(), { minSize: 1 }), + recipients: schema.arrayOf( + schema.string({ + validate(value) { + if (!regexEmailAddress.test(value)) { + return `invalid email address ${value}`; + } + }, + }), + { minSize: 1 } + ), title: schema.string({ minLength: 1 }), textDescription: schema.string({ minLength: 1 }), htmlDescription: schema.maybe(schema.string({ minLength: 1 })), @@ -154,7 +168,13 @@ export const deliverySchema = schema.object({ export const reportDefinitionSchema = schema.object({ report_params: schema.object({ - report_name: schema.string({ minLength: 1 }), + report_name: schema.string({ + validate(value) { + if (!regexReportName.test(value)) { + return `invald report name ${value}.\nMust be non-empty, allow a-z, A-Z, 0-9, (), [], ',' - and _ and spaces`; + } + }, + }), report_source: schema.oneOf([ schema.literal(REPORT_TYPE.dashboard), schema.literal(REPORT_TYPE.visualization), @@ -200,7 +220,7 @@ export const reportSchema = schema.object({ query_url: schema.string({ validate(value) { if (!isValidRelativeUrl(value)) { - return `this is invalid relative url: ${value}`; + return `invalid relative url: ${value}`; } }, }), diff --git a/kibana-reports/server/utils/validationHelper.ts b/kibana-reports/server/utils/validationHelper.ts index 0f20973b..28605009 100644 --- a/kibana-reports/server/utils/validationHelper.ts +++ b/kibana-reports/server/utils/validationHelper.ts @@ -28,9 +28,6 @@ export const isValidRelativeUrl = (relativeUrl: string) => { * moment.js isValid() API fails to validate time duration, so use regex * https://github.com/moment/moment/issues/1805 **/ - -export const isValidDuration = (duration: string) => { - return !!duration.match( - /^(-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$/ - ); -}; +export const regexDuration = /^(-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$/; +export const regexEmailAddress = /\S+@\S+\.\S+/; +export const regexReportName = /^[\w\-\s\(\)\[\]\,\_\-+]+$/; From f5f3389f2a90bd0d7571e63dd0af4b5a92024a59 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Mon, 19 Oct 2020 21:35:59 -0700 Subject: [PATCH 4/4] fix --- kibana-reports/server/model/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kibana-reports/server/model/index.ts b/kibana-reports/server/model/index.ts index eb4707b1..f613362e 100644 --- a/kibana-reports/server/model/index.ts +++ b/kibana-reports/server/model/index.ts @@ -143,9 +143,9 @@ export const channelSchema = schema.object({ }), { minSize: 1 } ), - title: schema.string({ minLength: 1 }), - textDescription: schema.string({ minLength: 1 }), - htmlDescription: schema.maybe(schema.string({ minLength: 1 })), + title: schema.string(), + textDescription: schema.string(), + htmlDescription: schema.maybe(schema.string()), email_format: schema.oneOf([ schema.literal(EMAIL_FORMAT.attachment), schema.literal(EMAIL_FORMAT.embeddedHtml), @@ -180,7 +180,7 @@ export const reportDefinitionSchema = schema.object({ schema.literal(REPORT_TYPE.visualization), schema.literal(REPORT_TYPE.savedSearch), ]), - description: schema.string({ minLength: 1 }), + description: schema.string(), core_params: schema.conditional( schema.siblingRef('report_source'), REPORT_TYPE.savedSearch,