Skip to content

Commit

Permalink
Revert "[1.x] Backport commits from main (opensearch-project#212)"
Browse files Browse the repository at this point in the history
This reverts commit 3762023.
  • Loading branch information
vmmusings committed Mar 8, 2022
1 parent 3762023 commit 43655fb
Show file tree
Hide file tree
Showing 91 changed files with 385 additions and 1,507 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/draft-release-notes-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: Release Drafter
on:
push:
branches:
- main
- develop

jobs:
update_release_draft:
Expand All @@ -16,6 +16,6 @@ jobs:
with:
config-name: draft-release-notes-config.yml
tag: (None)
version: 1.1.0.0
version: 1.0.0.0-rc1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2 changes: 1 addition & 1 deletion .github/workflows/link-checker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
id: lychee
uses: lycheeverse/lychee-action@master
with:
args: --accept=200,403,429,999 "**/*.html" "**/*.md" "**/*.txt" --exclude "http://localhost*" "https://localhost" "https://odfe-node1:9200/" "https://community.tableau.com/docs/DOC-17978" ".*family.zzz" "https://pypi.python.org/pypi/opensearch-sql-cli/" "opensearch*" ".*@amazon.com" ".*email.com" "[email protected]" "http://timestamp.verisign.com/scripts/timstamp.dll"
args: --accept=200,403,429 "**/*.html" "**/*.md" "**/*.txt" --exclude "http://localhost*" "https://localhost" "https://odfe-node1:9200/" "https://community.tableau.com/docs/DOC-17978" ".*family.zzz" "https://pypi.python.org/pypi/opensearch-sql-cli/" "opensearch*" ".*@amazon.com" ".*email.com" "[email protected]" "http://timestamp.verisign.com/scripts/timstamp.dll"
env:
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
- name: Fail if there were link errors
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sql-cli-release-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
jobs:
build:

runs-on: ubuntu-latest
runs-on: [ubuntu-16.04]
defaults:
run:
working-directory: sql-cli
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sql-cli-test-and-build-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on: [pull_request, push]
jobs:
build:

runs-on: ubuntu-latest
runs-on: [ubuntu-16.04]
defaults:
run:
working-directory: sql-cli
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sql-jdbc-push-jdbc-maven.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:

jobs:
upload-jdbc-jar:
runs-on: ubuntu-latest
runs-on: [ubuntu-16.04]
defaults:
run:
working-directory: sql-jdbc
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sql-odbc-release-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ env:
ODBC_BUILD_PATH: "./build/odbc/build"
AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install"
PLUGIN_NAME: opensearch-sql-odbc
OD_VERSION: 1.1.0.0
OD_VERSION: 1.0.0.0

jobs:
build-mac:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sql-odbc-rename-and-release-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
- rename*

env:
OD_VERSION: 1.1.0.0
OD_VERSION: 1.0.0.0

jobs:
upload-odbc:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sql-release-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
java: [14]

name: Build and Release SQL Plugin
runs-on: ubuntu-latest
runs-on: [ubuntu-16.04]

steps:
- name: Checkout SQL
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/sql-test-and-build-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@ jobs:
with:
repository: 'opensearch-project/OpenSearch'
path: OpenSearch
ref: '1.1'
ref: '1.0'

- name: Build OpenSearch
working-directory: ./OpenSearch
run: ./gradlew publishToMavenLocal
run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=rc1 -Dbuild.snapshot=false

- name: Build with Gradle
run: ./gradlew build assemble -Dopensearch.version=1.1.0-SNAPSHOT
run: ./gradlew build assemble

- name: Create Artifact Path
run: |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/sql-workbench-release-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
env:
PLUGIN_NAME: query-workbench-dashboards
OPENSEARCH_VERSION: '1.0'
OPENSEARCH_PLUGIN_VERSION: 1.0.0.0
OPENSEARCH_PLUGIN_VERSION: 1.0.0.0-rc1

jobs:

Expand Down Expand Up @@ -38,7 +38,7 @@ jobs:
- name: Setup Node
uses: actions/setup-node@v1
with:
node-version: '10.24.1'
node-version: '10.23.1'

- name: Move Workbench to Plugins Dir
run: |
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/sql-workbench-test-and-build-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ on: [pull_request, push]

env:
PLUGIN_NAME: query-workbench-dashboards
OPENSEARCH_VERSION: '1.x'
OPENSEARCH_PLUGIN_VERSION: 1.1.0.0
OPENSEARCH_VERSION: '1.0'
OPENSEARCH_PLUGIN_VERSION: 1.0.0.0-rc1

jobs:

Expand All @@ -27,7 +27,7 @@ jobs:
- name: Setup Node
uses: actions/setup-node@v1
with:
node-version: '10.24.1'
node-version: '10.23.1'

- name: Move Workbench to Plugins Dir
run: |
Expand Down
193 changes: 171 additions & 22 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,13 @@
[![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://discuss.opendistrocommunity.dev/c/sql/)
![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success)

<img src="https://opensearch.org/assets/brand/SVG/Logo/opensearch_logo_default.svg" height="64px"/>
# OpenSearch SQL

- [OpenSearch SQL](#opensearch-sql)
- [Highlights](#highlights)
- [Documentation](#documentation)
- [Contributing](#contributing)
- [Attribution](#attribution)
- [Code of Conduct](#code-of-conduct)
- [Security](#security)
- [License](#license)
- [Copyright](#copyright)

OpenSearch enables you to extract insights out of OpenSearch using the familiar SQL query syntax. Use aggregations, group by, and where clauses to investigate your data. Read your data as JSON documents or CSV tables so you have the flexibility to use the format that works best for you.

# OpenSearch SQL

OpenSearch enables you to extract insights out of OpenSearch using the familiar SQL or Piped Processing Language (PPL) query syntax. Use aggregations, group by, and where clauses to investigate your data. Read your data as JSON documents or CSV tables so you have the flexibility to use the format that works best for you.
## SQL Related Projects

The following projects have been merged into this repository as separate folders as of July 9, 2020. Please refer to links below for details. This document will focus on the SQL plugin for OpenSearch.

Expand All @@ -30,39 +21,197 @@ The following projects have been merged into this repository as separate folders
* [Query Workbench](https://github.com/opensearch-project/sql/tree/main/workbench)


## Highlights
## Documentation

Besides basic filtering and aggregation, OpenSearch SQL also supports complex queries, such as querying semi-structured data, JOINs, set operations, sub-queries etc. Beyond the standard functions, OpenSearch functions are provided for better analytics and visualization. Please check our [documentation](#documentation) for more details.
Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://docs-beta.opensearch.org/) for detailed information on installing and configuring plugin. Looking to contribute? Read the instructions on [Developer Guide](./DEVELOPER_GUIDE.rst) and then submit a patch!

Recently we have been actively improving our query engine primarily for better correctness and extensibility. Behind the scene, the new enhanced engine has already supported both SQL and Piped Processing Language. Please find more details in [SQL Engine V2 - Release Notes](./docs/dev/NewSQLEngine.md).
## SQL Engine V2

Recently we have been actively improving our query engine primarily for better correctness and extensibility. Behind the scene, the new enhanced engine has already supported the new released Piped Processing Language. However, it was experimental and disabled by default for SQL query processing. With most important features and full testing complete, now we're ready to promote it as our default SQL query engine. Please find more details in [SQL Engine V2 - Release Notes](/docs/dev/NewSQLEngine.md).

## Documentation

Please refer to the [SQL Language Reference Manual](./docs/user/index.rst), [Piped Processing Language (PPL) Reference Manual](./docs/user/ppl/index.rst) and [Technical Documentation](https://docs-beta.opensearch.org/) for detailed information on installing and configuring plugin.
## Setup

Install as plugin: build plugin from source code by following the instruction in Build section and install it to your OpenSearch.

After doing this, you need to restart the OpenSearch server. Otherwise you may get errors like `Invalid index name [sql], must not start with '']; ","status":400}`.


## Build

The package uses the [Gradle](https://docs.gradle.org/4.10.2/userguide/userguide.html) build system.

1. Checkout this package from version control.
2. To build from command line set `JAVA_HOME` to point to a JDK >=14
3. Run `./gradlew build`


## Basic Usage

To use the feature, send requests to the `_plugins/_sql` URI. You can use a request parameter or the request body (recommended). Note that for backward compatibility, old `_opendistro/_sql` endpoint is still available, though any future API will be only accessible by new OpenSearch endpoint.

* Simple query

```
POST https://<host>:<port>/_plugins/_sql
{
"query": "SELECT * FROM my-index LIMIT 50"
}
```

* Explain SQL to OpenSearch query DSL
```
POST _plugins/_sql/_explain
{
"query": "SELECT * FROM my-index LIMIT 50"
}
```

* For a sample curl command with the OpenSearch Security plugin, try:
```
curl -XPOST https://localhost:9200/_plugins/_sql -u admin:admin -k -d '{"query": "SELECT * FROM my-index LIMIT 10"}' -H 'Content-Type: application/json'
```


## SQL Usage

* Query

SELECT * FROM bank WHERE age >30 AND gender = 'm'

* Aggregation

SELECT COUNT(*),SUM(age),MIN(age) as m, MAX(age),AVG(age)
FROM bank
GROUP BY gender
HAVING m >= 20
ORDER BY SUM(age), m DESC

* Join

SELECT b1.firstname, b1.lastname, b2.age
FROM bank b1
LEFT JOIN bank b2
ON b1.age = b2.age AND b1.state = b2.state

* Show

## Contributing
SHOW TABLES LIKE ban%
DESCRIBE TABLES LIKE bank

See [developer guide](DEVELOPER_GUIDE.rst) and [how to contribute to this project](CONTRIBUTING.md).
* Delete

DELETE FROM bank WHERE age >30 AND gender = 'm'


## Beyond SQL

* Search

SELECT address FROM bank WHERE address = matchQuery('880 Holmes Lane') ORDER BY _score DESC LIMIT 3

* Nested Field

+
SELECT address FROM bank b, b.nestedField e WHERE b.state = 'WA' and e.name = 'test'
+
SELECT address, nested(nestedField.name)
FROM bank
WHERE nested(nestedField, nestedField.state = 'WA' AND nestedField.name = 'test')
OR nested(nestedField.state) = 'CA'

* Aggregations

+ range age group 20-25,25-30,30-35,35-40

SELECT COUNT(age) FROM bank GROUP BY range(age, 20,25,30,35,40)

+ range date group by day

SELECT online FROM online GROUP BY date_histogram(field='insert_time','interval'='1d')

+ range date group by your config

SELECT online FROM online GROUP BY date_range(field='insert_time','format'='yyyy-MM-dd' ,'2014-08-18','2014-08-17','now-8d','now-7d','now-6d','now')

* OpenSearch Geographic

SELECT * FROM locations WHERE GEO_BOUNDING_BOX(fieldname,100.0,1.0,101,0.0)

* Select type or pattern

SELECT * FROM indexName/type
SELECT * FROM index*


## SQL Features

* SQL Select
* SQL Delete
* SQL Where
* SQL Order By
* SQL Group By
* SQL Having
* SQL Inner Join
* SQL Left Join
* SQL Show
* SQL Describe
* SQL AND & OR
* SQL Like
* SQL COUNT distinct
* SQL In
* SQL Between
* SQL Aliases
* SQL Not Null
* SQL(OpenSearch) Date
* SQL avg()
* SQL count()
* SQL max()
* SQL min()
* SQL sum()
* SQL Nulls
* SQL isnull()
* SQL floor
* SQL trim
* SQL log
* SQL log10
* SQL substring
* SQL round
* SQL sqrt
* SQL concat_ws
* SQL union and minus

## JDBC Support

Please check out JDBC driver repository for more details.

## Beyond SQL features

* OpenSearch TopHits
* OpenSearch MISSING
* OpenSearch STATS
* OpenSearch GEO_INTERSECTS
* OpenSearch GEO_BOUNDING_BOX
* OpenSearch GEO_DISTANCE
* OpenSearch GEOHASH_GRID aggregation

## Attribution

This project is based on the Apache 2.0-licensed [elasticsearch-sql](https://github.com/NLPchina/elasticsearch-sql) project. Thank you [eliranmoyal](https://github.com/eliranmoyal), [shi-yuan](https://github.com/shi-yuan), [ansjsun](https://github.com/ansjsun) and everyone else who contributed great code to that project. Read this for more details [Attributions](./docs/attributions.md).


## Code of Conduct

This project has adopted an [Open Source Code of Conduct](./CODE_OF_CONDUCT.md).


## Security
## Security issue notifications

If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue.


## License
## Licensing

See the [LICENSE](./LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution.

Expand Down
8 changes: 3 additions & 5 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

buildscript {
ext {
opensearch_version = System.getProperty("opensearch.version", "1.1.0-SNAPSHOT")
opensearch_version = "1.0.0-rc1"
}

repositories {
Expand Down Expand Up @@ -55,14 +55,12 @@ repositories {
}

ext {
opensearchVersion = '1.0.0'
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
}

allprojects {
version = opensearch_version - "-SNAPSHOT" + ".0"
if (isSnapshot) {
version += "-SNAPSHOT"
}
version = "${opensearchVersion}.0-rc1"

plugins.withId('java') {
sourceCompatibility = targetCompatibility = "1.8"
Expand Down
Loading

0 comments on commit 43655fb

Please sign in to comment.