fix/issue 105/support additional operators (#112)

* fix: add support for more operators, closes #105 * refactor: clean up node and comment imports * fix: add support for snowflake's minus keyword * fix: add grouping operators
tconbeer · Feb 2, 2022 · e2e9286 · e2e9286
1 parent 2cc8825
commit e2e9286
Show file tree

Hide file tree

Showing 13 changed files with 121 additions and 21 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
 ### Fixes
 
 -   adds "cross join" to list of supported join types. No longer merges the "cross" keyword with the previous statement ([#110](https://github.com/tconbeer/sqlfmt/issues/110) - thank you [@rdeese](https://github.com/rdeese)!)
+-   add support for every valid operator in postgresql, even the weird ones, like `@>`, `||/`, `?-|` ([#105](https://github.com/tconbeer/sqlfmt/issues/105))
 
 ## [0.4.3] - 2022-01-31
 

diff --git a/src/sqlfmt/actions.py b/src/sqlfmt/actions.py
@@ -2,8 +2,10 @@
 from typing import List, Optional
 
 from sqlfmt.analyzer import MAYBE_WHITESPACES, Analyzer, group
+from sqlfmt.comment import Comment
 from sqlfmt.exception import SqlfmtBracketError, StopJinjaLexing
-from sqlfmt.line import Comment, Line, Node
+from sqlfmt.line import Line
+from sqlfmt.node import Node
 from sqlfmt.token import Token, TokenType
 
 

diff --git a/src/sqlfmt/analyzer.py b/src/sqlfmt/analyzer.py
@@ -2,8 +2,10 @@
 from dataclasses import dataclass, field
 from typing import Callable, Dict, List, Optional
 
+from sqlfmt.comment import Comment
 from sqlfmt.exception import SqlfmtBracketError, SqlfmtParsingError
-from sqlfmt.line import Comment, Line, Node
+from sqlfmt.line import Line
+from sqlfmt.node import Node
 from sqlfmt.query import Query
 
 

diff --git a/src/sqlfmt/dialect.py b/src/sqlfmt/dialect.py
@@ -220,12 +220,33 @@ def __init__(self) -> None:
                     name="operator",
                     priority=910,
                     pattern=group(
+                        r"\|\|?\/",  # square or cube root ||/
+                        r"~=",  # geo compare
+                        r"!?~\*?",  # posix like/not like
+                        r"\?(=|!|<=|<!)",  # regex lookahead/behind
+                        r"\?(-\||\|\||-|\|)",  # regex lookahead/behind
+                        r"@-@",  # length operator
+                        r"@@@?",  # center point operator; also text match
+                        r"##",  # closest point
+                        r"<->",  # distance operator
+                        r"@>",  # contains
+                        r"<@",  # contained by
                         r"<>",
-                        r"!=",
+                        r"\|?>>=?",
+                        r"<<(=|\|)?",
                         r"=>",
+                        r"(-|#)>>?",  # json extraction
+                        r"&&",
+                        r"&<\|?",  # not extends
+                        r"\|?&>",  # not extends
+                        r"<\^",  # below
+                        r">\^",  # above
+                        r"\?#",  # intersect
                         r"\|\|",
-                        r"[+\-*/%&@|^=<>:]=?",
-                        r"~",
+                        r"-\|-",
+                        r"[*+?]?\?",  # regex greedy/non-greedy, also ?
+                        r"!!",  # negate text match
+                        r"[+\-*/%&|^=<>:#!]=?",  # singles
                     ),
                     action=partial(
                         actions.add_node_to_buffer, token_type=TokenType.OPERATOR
@@ -235,16 +256,24 @@ def __init__(self) -> None:
                     name="word_operator",
                     priority=920,
                     pattern=group(
+                        r"all",
+                        r"any",
                         r"between",
+                        r"cube",
+                        r"exists",
+                        r"grouping sets",
                         r"ilike",
                         r"in",
                         r"is",
                         r"isnull",
-                        r"like",
-                        r"not",
+                        r"like(\s+any)?",
                         r"notnull",
+                        r"not",
                         r"over",
-                        r"similar",
+                        r"rollup",
+                        r"rlike",
+                        r"some",
+                        r"similar\s+to",
                     )
                     + group(r"\W", r"$"),
                     action=partial(
@@ -311,7 +340,7 @@ def __init__(self) -> None:
                         r"having",
                         r"qualify",
                         r"window",
-                        r"(union|intersect|except)(\s+all|distinct)?",
+                        r"(union|intersect|except|minus)(\s+all|distinct)?",
                         r"order\s+by",
                         r"limit",
                         r"offset",

diff --git a/src/sqlfmt/formatter.py b/src/sqlfmt/formatter.py
@@ -2,9 +2,10 @@
 from typing import List, Optional
 
 from sqlfmt.jinjafmt import JinjaFormatter
-from sqlfmt.line import Line, Node
+from sqlfmt.line import Line
 from sqlfmt.merger import LineMerger
 from sqlfmt.mode import Mode
+from sqlfmt.node import Node
 from sqlfmt.query import Query
 from sqlfmt.splitter import LineSplitter
 

diff --git a/src/sqlfmt/jinjafmt.py b/src/sqlfmt/jinjafmt.py
@@ -4,8 +4,9 @@
 from types import ModuleType
 from typing import Optional
 
-from sqlfmt.line import Line, Node
+from sqlfmt.line import Line
 from sqlfmt.mode import Mode
+from sqlfmt.node import Node
 
 
 class BlackWrapper:

diff --git a/src/sqlfmt/merger.py b/src/sqlfmt/merger.py
@@ -1,9 +1,11 @@
 from dataclasses import dataclass
 from typing import List
 
+from sqlfmt.comment import Comment
 from sqlfmt.exception import CannotMergeException
-from sqlfmt.line import Comment, Line, Node
+from sqlfmt.line import Line
 from sqlfmt.mode import Mode
+from sqlfmt.node import Node
 from sqlfmt.token import TokenType
 
 

diff --git a/src/sqlfmt/query.py b/src/sqlfmt/query.py
@@ -1,7 +1,8 @@
 from dataclasses import dataclass, field
 from typing import List
 
-from sqlfmt.line import Line, Node
+from sqlfmt.line import Line
+from sqlfmt.node import Node
 from sqlfmt.token import Token
 
 

diff --git a/tests/data/errors/900_bad_token.sql b/tests/data/errors/900_bad_token.sql
@@ -1,3 +1 @@
-select ?
-from my_table
-where no_question_mark is true
+select $
diff --git a/tests/unit_tests/test_analyzer.py b/tests/unit_tests/test_analyzer.py
@@ -1,8 +1,8 @@
 import pytest
 
 from sqlfmt.analyzer import Analyzer, SqlfmtParsingError
+from sqlfmt.comment import Comment
 from sqlfmt.exception import SqlfmtBracketError
-from sqlfmt.line import Comment
 from sqlfmt.mode import Mode
 from sqlfmt.token import Token, TokenType
 from tests.util import read_test_data
@@ -88,7 +88,7 @@ def test_simple_query_parsing(all_output_modes: Mode) -> None:
 
 
 def test_parsing_error(default_analyzer: Analyzer) -> None:
-    source_string = "select !"
+    source_string = "select $"
     with pytest.raises(SqlfmtParsingError):
         _ = default_analyzer.parse_query(source_string=source_string)
 

diff --git a/tests/unit_tests/test_api.py b/tests/unit_tests/test_api.py
@@ -5,7 +5,6 @@
 
 import pytest
 
-from sqlfmt.analyzer import SqlfmtParsingError
 from sqlfmt.api import (
     _format_many,
     _generate_matched_paths,
@@ -51,7 +50,6 @@ def test_format_empty_string(all_output_modes: Mode) -> None:
 @pytest.mark.parametrize(
     "source,exception",
     [
-        ("?\n", SqlfmtParsingError),
         ("select )\n", SqlfmtBracketError),
         ("{{\n", SqlfmtBracketError),
     ],

diff --git a/tests/unit_tests/test_dialect.py b/tests/unit_tests/test_dialect.py
@@ -82,11 +82,70 @@ def test_rule_props_are_unique(self, polyglot: Polyglot) -> None:
             ("main", "double_colon", "::"),
             ("main", "colon", ":"),
             ("main", "semicolon", ";"),
+            ("main", "operator", "+"),
+            ("main", "operator", "-"),
+            ("main", "operator", "/"),
             ("main", "operator", "<>"),
             ("main", "operator", "||"),
             ("main", "operator", "=>"),
+            ("main", "operator", "||/"),
+            ("main", "operator", "|/"),
+            ("main", "operator", "#"),
+            ("main", "operator", ">>"),
+            ("main", "operator", "<<"),
+            ("main", "operator", "!"),
+            ("main", "operator", "!="),
+            # posix like/ not like
+            ("main", "operator", "~"),
+            ("main", "operator", "!~"),
+            ("main", "operator", "~*"),
+            ("main", "operator", "!~*"),
+            # postgresql geo operators
+            # see: https://www.postgresql.org/docs/current/functions-geometry.html
+            ("main", "operator", "@-@"),
+            ("main", "operator", "@@"),
+            ("main", "operator", "##"),
+            ("main", "operator", "<->"),
+            ("main", "operator", "<@"),
+            ("main", "operator", "@>"),
+            ("main", "operator", "&&"),
+            ("main", "operator", "&<"),
+            ("main", "operator", "&>"),
+            ("main", "operator", "<<|"),
+            ("main", "operator", "|>>"),
+            ("main", "operator", "&<|"),
+            ("main", "operator", "|&>"),
+            ("main", "operator", "<^"),
+            ("main", "operator", ">^"),
+            ("main", "operator", "?#"),
+            ("main", "operator", "?-"),
+            ("main", "operator", "?|"),
+            ("main", "operator", "?-|"),
+            ("main", "operator", "?||"),
+            ("main", "operator", "~="),
+            # network operators
+            # see https://www.postgresql.org/docs/current/functions-net.html
+            ("main", "operator", "<<="),
+            ("main", "operator", ">>="),
+            # json operators
+            # see https://www.postgresql.org/docs/current/functions-json.html
+            ("main", "operator", "->"),
+            ("main", "operator", "->>"),
+            ("main", "operator", "#>"),
+            ("main", "operator", "#>>"),
+            ("main", "operator", "-|-"),  # range adjacency
             ("main", "word_operator", "is"),
             ("main", "word_operator", "in"),
+            ("main", "word_operator", "like"),
+            ("main", "word_operator", "ilike"),
+            ("main", "word_operator", "like any"),
+            ("main", "word_operator", "any"),
+            ("main", "word_operator", "some"),
+            ("main", "word_operator", "exists"),
+            ("main", "word_operator", "all"),
+            ("main", "word_operator", "grouping sets"),
+            ("main", "word_operator", "cube"),
+            ("main", "word_operator", "rollup"),
             ("main", "as", "as"),
             ("main", "on", "on"),
             ("main", "boolean_operator", "AND"),
@@ -101,6 +160,11 @@ def test_rule_props_are_unique(self, polyglot: Polyglot) -> None:
             ("main", "unterm_keyword", "left join"),
             ("main", "unterm_keyword", "cross join"),
             ("main", "unterm_keyword", "join"),
+            ("main", "unterm_keyword", "union"),
+            ("main", "unterm_keyword", "union all"),
+            ("main", "unterm_keyword", "intersect"),
+            ("main", "unterm_keyword", "minus"),
+            ("main", "unterm_keyword", "except"),
             ("main", "name", "my_table_45"),
             ("main", "newline", "\n"),
             ("jinja", "jinja_comment", "{# my comment #}"),

diff --git a/tests/unit_tests/test_jinjafmt.py b/tests/unit_tests/test_jinjafmt.py
@@ -4,8 +4,9 @@
 import pytest
 
 from sqlfmt.jinjafmt import JinjaFormatter, JinjaTag
-from sqlfmt.line import Line, Node
+from sqlfmt.line import Line
 from sqlfmt.mode import Mode
+from sqlfmt.node import Node
 from sqlfmt.token import Token, TokenType