Skip to content

Commit

Permalink
Introducing transforms (opensearch-project#16)
Browse files Browse the repository at this point in the history
  • Loading branch information
thalurur authored May 25, 2021
1 parent 3a1b603 commit 21177ab
Show file tree
Hide file tree
Showing 93 changed files with 7,044 additions and 282 deletions.
1 change: 1 addition & 0 deletions .github/workflows/cypress-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ on:
push:
branches:
- main
- development-*

jobs:
tests:
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/unit-tests-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ on:
push:
branches:
- main
- development-*

jobs:
tests:
Expand Down
81 changes: 81 additions & 0 deletions models/interfaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,14 @@ export interface DocumentRollup {
metadata: any;
}

export interface DocumentTransform {
_id: string;
_seqNo: number;
_primaryTerm: number;
transform: Transform;
metadata: any;
}

// TODO: Fill out when needed
// TODO: separate a frontend Policy from backendPolicy
export interface Policy {
Expand Down Expand Up @@ -136,6 +144,44 @@ export interface RollupMetadata {
};
}

export interface Transform {
description: string;
groups: RollupDimensionItem[];
enabled: boolean;
enabled_at: number | null;
updated_at: number;
metadata_id: string | null;
aggregations: Map<String, any>;
page_size: number;
schedule: IntervalSchedule | CronSchedule;
schema_version: number;
source_index: string;
target_index: string;
roles: String[];
data_selection_query: Map<String, any>;
}

export interface TransformMetadata {
metadata_id: string;
transform_metadata: {
id: string;
seq_no: number;
primary_term: number;
transform_id: string;
after_key: Map<string, any> | null;
last_updated_at: number;
status: string;
failure_reason: string | null;
stats: {
pages_processed: number | null;
documents_processed: number | null;
documents_indexed: number | null;
index_time_in_millis: number | null;
search_time_in_millis: number | null;
};
};
}

export interface IntervalSchedule {
interval: {
startTime: number | null;
Expand Down Expand Up @@ -215,3 +261,38 @@ export interface RollupMetricItem {
}
];
}

export type TransformGroupItem = DateHistogramItem | TermsItem | HistogramItem;

export enum GROUP_TYPES {
histogram = "histogram",
dateHistogram = "date_histogram",
terms = "terms",
}

export interface TransformAggItem {
type: TRANSFORM_AGG_TYPE;
name: string;
item: any | DateHistogramItem | TermsItem | HistogramItem;
percents?: number[];
sum?: { field: string };
max?: { field: string };
min?: { field: string };
avg?: { field: string };
value_count?: { field: string };
percentiles?: { field: string; percents: number[] };
scripted_metric?: object;
}

export enum TRANSFORM_AGG_TYPE {
sum = "sum",
max = "max",
min = "min",
avg = "avg",
value_count = "value_count",
percentiles = "percentiles",
scripted_metric = "scripted_metric",
terms = "terms",
histogram = "histogram",
date_histogram = "date_histogram",
}
5 changes: 3 additions & 2 deletions public/index_management_app.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import { CoreStart, AppMountParameters } from "opensearch-dashboards/public";
import React from "react";
import ReactDOM from "react-dom";
import { HashRouter as Router, Route } from "react-router-dom";
import { IndexService, ManagedIndexService, PolicyService, RollupService, ServicesContext } from "./services";
import { IndexService, ManagedIndexService, PolicyService, RollupService, TransformService, ServicesContext } from "./services";
import { DarkModeContext } from "./components/DarkMode";
import Main from "./pages/Main";
import { CoreServicesContext } from "./components/core_services";
Expand All @@ -40,7 +40,8 @@ export function renderApp(coreStart: CoreStart, params: AppMountParameters) {
const managedIndexService = new ManagedIndexService(http);
const policyService = new PolicyService(http);
const rollupService = new RollupService(http);
const services = { indexService, managedIndexService, policyService, rollupService };
const transformService = new TransformService(http);
const services = { indexService, managedIndexService, policyService, rollupService, transformService };

const isDarkMode = coreStart.uiSettings.get("theme:darkMode") || false;

Expand Down
3 changes: 2 additions & 1 deletion public/models/interfaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@
* permissions and limitations under the License.
*/

import { IndexService, ManagedIndexService, PolicyService, RollupService } from "../services";
import { IndexService, ManagedIndexService, PolicyService, RollupService, TransformService } from "../services";

export interface BrowserServices {
indexService: IndexService;
managedIndexService: ManagedIndexService;
policyService: PolicyService;
rollupService: RollupService;
transformService: TransformService;
}
182 changes: 182 additions & 0 deletions public/pages/Commons/BaseAggregationAndMetricSettings.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

/*
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

import React, { Fragment } from "react";

import { EuiFlexItem, EuiText, EuiBasicTable, EuiTableFieldDataColumnType, EuiPanel, EuiFlexGroup, EuiIcon } from "@elastic/eui";

import { DimensionItem, MetricItem } from "../../../models/interfaces";

export const AGGREGATION_AND_METRIC_SETTINGS = "Aggregation and metrics settings";

export interface BaseAggregationAndMetricsState {
from: number;
size: number;
sortField: string;
sortDirection: string;
dimensionFrom: number;
dimensionSize: number;
dimensionSortField: string;
dimensionSortDirection: string;
}

export const BaseAggregationColumns: Readonly<EuiTableFieldDataColumnType<DimensionItem>>[] = [
{
field: "sequence",
name: "Sequence",
sortable: true,
align: "left",
dataType: "number",
},
{
field: "field.label",
name: "Field name",
align: "left",
},
{
field: "aggregationMethod",
name: "Aggregation method",
align: "left",
},
{
field: "interval",
name: "Interval",
dataType: "number",
align: "left",
render: (interval: null | number) => {
if (interval == null) return "-";
else return `${interval}`;
},
},
];

export const BaseMetricsColumns: Readonly<EuiTableFieldDataColumnType<MetricItem>>[] = [
{
field: "source_field",
name: "Field Name",
},
{
field: "min",
name: "Min",
align: "center",
render: (min: boolean) => min && <EuiIcon type="check" />,
},
{
field: "max",
name: "Max",
align: "center",
render: (max: boolean) => max && <EuiIcon type="check" />,
},
{
field: "sum",
name: "Sum",
align: "center",
render: (sum: boolean) => sum && <EuiIcon type="check" />,
},
{
field: "avg",
name: "Avg",
align: "center",
render: (avg: boolean) => avg && <EuiIcon type="check" />,
},
{
field: "value_count",
name: "Value count",
align: "center",
render: (value_count: boolean) => value_count && <EuiIcon type="check" />,
},
];

export function sequenceTableComponents(selectedDimensionField, items, columns, pagination, sorting, onChange) {
if (selectedDimensionField.length == 0) {
return (
<EuiText>
<dd>No fields added for aggregation</dd>
</EuiText>
);
}

return (
<Fragment>
<EuiPanel>
<EuiBasicTable
items={items}
rowHeader="sequence"
columns={columns}
tableLayout="auto"
noItemsMessage="No fields added for aggregations"
pagination={pagination}
sorting={sorting}
onChange={onChange}
/>
</EuiPanel>
</Fragment>
);
}

export function additionalMetricsComponent(selectedMetrics) {
return (
<EuiFlexGroup gutterSize="xs">
<EuiFlexItem grow={false}>
<EuiText>
<h3>Additional metrics</h3>
</EuiText>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText color="subdued" textAlign="left">
<h3>{`(${selectedMetrics.length})`}</h3>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
);
}

export function sourceFieldComponents(selectedMetrics, items, columns, pagination, sorting, onChange) {
if (selectedMetrics.length == 0) {
return (
<EuiText>
<dd>No fields added for metrics</dd>
</EuiText>
);
}

return (
<Fragment>
<EuiPanel>
<EuiBasicTable
items={items}
rowHeader="source_field"
columns={columns}
tableLayout="auto"
pagination={pagination}
sorting={sorting}
onChange={onChange}
noItemsMessage="No fields added for metrics"
/>
</EuiPanel>
</Fragment>
);
}
Loading

0 comments on commit 21177ab

Please sign in to comment.