Skip to content

Commit

Permalink
[Console] Console with better SQL support (#51446) (#54091)
Browse files Browse the repository at this point in the history
* First iteration of PoC for SQL highlighting and better completion

* Alternate implementation (no """sql markers)

* xLang markers

* - Remove 'start-sql'
- no flow :c

* Revert "- Remove 'start-sql'"

This reverts commit 2d585ee.

* Revert "xLang markers"

This reverts commit f019616.

* Revert "xLang markers" and add comment

This reverts commit f019616.

* Add elasticsearch sql highlight rules

* Add links to sources of data

* Update colors

* Redo built in functions

# Conflicts:
#	src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/input_highlight_rules.js
#	src/legacy/core_plugins/console/public/np_ready/application/models/legacy_core_editor/mode/x_json_highlight_rules.js
  • Loading branch information
jloleysens authored Jan 7, 2020
1 parent 1d6896f commit 24eb8b3
Show file tree
Hide file tree
Showing 8 changed files with 218 additions and 19 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import ace from 'brace';

const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');
const oop = ace.acequire('ace/lib/oop');

export const ElasticsearchSqlHighlightRules = function(this: any) {
// See https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-commands.html
const keywords =
'describe|between|in|like|not|and|or|desc|select|from|where|having|group|by|order' +
'asc|desc|pivot|for|in|as|show|columns|include|frozen|tables|escape|limit|rlike|all|distinct|is';

const builtinConstants = 'true|false';

// See https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-syntax-show-functions.html
const builtinFunctions =
'avg|count|first|first_value|last|last_value|max|min|sum|kurtosis|mad|percentile|percentile_rank|skewness' +
'|stddev_pop|sum_of_squares|var_pop|histogram|case|coalesce|greatest|ifnull|iif|isnull|least|nullif|nvl' +
'|curdate|current_date|current_time|current_timestamp|curtime|dateadd|datediff|datepart|datetrunc|date_add' +
'|date_diff|date_part|date_trunc|day|dayname|dayofmonth|dayofweek|dayofyear|day_name|day_of_month|day_of_week' +
'|day_of_year|dom|dow|doy|hour|hour_of_day|idow|isodayofweek|isodow|isoweek|isoweekofyear|iso_day_of_week|iso_week_of_year' +
'|iw|iwoy|minute|minute_of_day|minute_of_hour|month|monthname|month_name|month_of_year|now|quarter|second|second_of_minute' +
'|timestampadd|timestampdiff|timestamp_add|timestamp_diff|today|week|week_of_year|year|abs|acos|asin|atan|atan2|cbrt' +
'|ceil|ceiling|cos|cosh|cot|degrees|e|exp|expm1|floor|log|log10|mod|pi|power|radians|rand|random|round|sign|signum|sin' +
'|sinh|sqrt|tan|truncate|ascii|bit_length|char|character_length|char_length|concat|insert|lcase|left|length|locate' +
'|ltrim|octet_length|position|repeat|replace|right|rtrim|space|substring|ucase|cast|convert|database|user|st_astext|st_aswkt' +
'|st_distance|st_geometrytype|st_geomfromtext|st_wkttosql|st_x|st_y|st_z|score';

// See https://www.elastic.co/guide/en/elasticsearch/reference/current/sql-data-types.html
const dataTypes =
'null|boolean|byte|short|integer|long|double|float|half_float|scaled_float|keyword|text|binary|date|ip|object|nested|time' +
'|interval_year|interval_month|interval_day|interval_hour|interval_minute|interval_second|interval_year_to_month' +
'inteval_day_to_hour|interval_day_to_minute|interval_day_to_second|interval_hour_to_minute|interval_hour_to_second' +
'interval_minute_to_second|geo_point|geo_shape|shape';

const keywordMapper = this.createKeywordMapper(
{
keyword: [keywords, builtinFunctions, builtinConstants, dataTypes].join('|'),
},
'identifier',
true
);

this.$rules = {
start: [
{
token: 'comment',
regex: '--.*$',
},
{
token: 'comment',
start: '/\\*',
end: '\\*/',
},
{
token: 'string', // " string
regex: '".*?"',
},
{
token: 'constant', // ' string
regex: "'.*?'",
},
{
token: 'string', // ` string (apache drill)
regex: '`.*?`',
},
{
token: 'entity.name.function', // float
regex: '[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b',
},
{
token: keywordMapper,
regex: '[a-zA-Z_$][a-zA-Z0-9_$]*\\b',
},
{
token: 'keyword.operator',
regex: '⇐|<⇒|\\*|\\.|\\:\\:|\\+|\\-|\\/|\\/\\/|%|&|\\^|~|<|>|<=|=>|==|!=|<>|=',
},
{
token: 'paren.lparen',
regex: '[\\(]',
},
{
token: 'paren.rparen',
regex: '[\\)]',
},
{
token: 'text',
regex: '\\s+',
},
],
};
this.normalizeRules();
};

oop.inherits(ElasticsearchSqlHighlightRules, TextHighlightRules);
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,33 @@
const ace = require('brace');
import { addToRules } from './x_json_highlight_rules';

const oop = ace.acequire('ace/lib/oop');
const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');
export function InputHighlightRules() {
function mergeTokens(/* ... */) {
return [].concat.apply([], arguments);
export function addEOL(tokens, reg, nextIfEOL, normalNext) {
if (typeof reg === 'object') {
reg = reg.source;
}
return [
{ token: tokens.concat(['whitespace']), regex: reg + '(\\s*)$', next: nextIfEOL },
{ token: tokens, regex: reg, next: normalNext },
];
}

function addEOL(tokens, reg, nextIfEOL, normalNext) {
if (typeof reg === 'object') {
reg = reg.source;
}
return [
{ token: tokens.concat(['whitespace']), regex: reg + '(\\s*)$', next: nextIfEOL },
{ token: tokens, regex: reg, next: normalNext },
];
}
export function mergeTokens(/* ... */) {
return [].concat.apply([], arguments);
}

const oop = ace.acequire('ace/lib/oop');
const { TextHighlightRules } = ace.acequire('ace/mode/text_highlight_rules');

export function InputHighlightRules() {
// regexp must not have capturing parentheses. Use (?:) instead.
// regexps are ordered -> the first match is used
/*jshint -W015 */
this.$rules = {
'start-sql': [
{ token: 'whitespace', regex: '\\s+' },
{ token: 'paren.lparen', regex: '{', next: 'json-sql', push: true },
{ regex: '', next: 'start' },
],
start: mergeTokens(
[
{ token: 'warning', regex: '#!.*$' },
Expand Down Expand Up @@ -71,6 +77,7 @@ export function InputHighlightRules() {
addEOL(['whitespace'], /(\s+)/, 'start', 'url')
),
url: mergeTokens(
addEOL(['url.part'], /(_sql)/, 'start-sql', 'url-sql'),
addEOL(['url.part'], /([^?\/,\s]+)/, 'start'),
addEOL(['url.comma'], /(,)/, 'start'),
addEOL(['url.slash'], /(\/)/, 'start'),
Expand All @@ -81,6 +88,16 @@ export function InputHighlightRules() {
addEOL(['url.param'], /([^&=]+)/, 'start'),
addEOL(['url.amp'], /(&)/, 'start')
),
'url-sql': mergeTokens(
addEOL(['url.comma'], /(,)/, 'start-sql'),
addEOL(['url.slash'], /(\/)/, 'start-sql'),
addEOL(['url.questionmark'], /(\?)/, 'start-sql', 'urlParams-sql')
),
'urlParams-sql': mergeTokens(
addEOL(['url.param', 'url.equal', 'url.value'], /([^&=]+)(=)([^&]*)/, 'start-sql'),
addEOL(['url.param'], /([^&=]+)/, 'start-sql'),
addEOL(['url.amp'], /(&)/, 'start-sql')
),
};

addToRules(this);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@
*/

const _ = require('lodash');
const ScriptHighlightRules = require('./script_highlight_rules').ScriptHighlightRules;

import { ElasticsearchSqlHighlightRules } from './elasticsearch_sql_highlight_rules';
const { ScriptHighlightRules } = require('./script_highlight_rules');

const jsonRules = function(root) {
root = root ? root : 'json';
const rules = {};
rules[root] = [
const xJsonRules = [
{
token: [
'variable',
Expand Down Expand Up @@ -112,6 +114,24 @@ const jsonRules = function(root) {
regex: '.+?',
},
];

rules[root] = xJsonRules;
rules[root + '-sql'] = [
{
token: [
'variable',
'whitespace',
'ace.punctuation.colon',
'whitespace',
'punctuation.start_triple_quote',
],
regex: '("query")(\\s*?)(:)(\\s*?)(""")',
next: 'sql-start',
merge: false,
push: true,
},
].concat(xJsonRules);

rules.string_literal = [
{
token: 'punctuation.end_triple_quote',
Expand All @@ -135,4 +155,11 @@ export function addToRules(otherRules, embedUnder) {
next: 'pop',
},
]);
otherRules.embedRules(ElasticsearchSqlHighlightRules, 'sql-', [
{
token: 'punctuation.end_triple_quote',
regex: '"""',
next: 'pop',
},
]);
}
Original file line number Diff line number Diff line change
Expand Up @@ -990,7 +990,7 @@ export default function({ coreEditor: editor, parser }: { coreEditor: CoreEditor
score: 0,
context,
};
// we only need out custom insertMatch behavior for the body
// we only need our custom insertMatch behavior for the body
if (context.autoCompleteType === 'body') {
defaults.completer = {
insertMatch() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

// @ts-ignore
import { ConstantComponent } from './constant_component';

export class FullRequestComponent extends ConstantComponent {
private readonly name: string;
constructor(name: string, parent: any, private readonly template: string) {
super(name, parent);
this.name = name;
}

getTerms() {
return [{ name: this.name, snippet: this.template }];
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import {
SimpleParamComponent,
} from './index';

import { FullRequestComponent } from './full_request_component';

/**
* @param parametrizedComponentFactories a dict of the following structure
* that will be used as a fall back for pattern parameters (i.e.: {indices})
Expand Down Expand Up @@ -54,6 +56,9 @@ export class UrlPatternMatcher {
endpoint.methods.forEach(method => {
let c;
let activeComponent = this[method].rootComponent;
if (endpoint.template) {
new FullRequestComponent(pattern + '[body]', activeComponent, endpoint.template);
}
const endpointComponents = endpoint.url_components || {};
const partList = pattern.split('/');
_.each(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,10 @@ export default class RowParser {
return MODE.BETWEEN_REQUESTS;
} // shouldn't really happen

if (mode !== 'start') {
// If another "start" mode is added here because we want to allow for new language highlighting
// please see https://github.com/elastic/kibana/pull/51446 for a discussion on why
// should consider a different approach.
if (mode !== 'start' && mode !== 'start-sql') {
return MODE.IN_REQUEST;
}
let line = (this.editor.getLineValue(lineNumber) || '').trim();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"cbor",
"smile"
]
}
},
"template": "_sql?format=json\n{\n \"query\": \"\"\"\n SELECT * FROM \"${1:TABLE}\"\n \"\"\"\n}\n"
}
}

0 comments on commit 24eb8b3

Please sign in to comment.