From 32e57818ec9d6229b345c44423c76bd37f73109e Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 9 Jan 2024 13:21:24 -0500 Subject: [PATCH 01/55] empty --- .../src/Common_Table_Operations/Nothing_Spec.enso | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso new file mode 100644 index 000000000000..3239bf2accc4 --- /dev/null +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -0,0 +1,15 @@ +from Standard.Base import all + +from Standard.Test import Test +import Standard.Test.Extensions + +from project.Common_Table_Operations.Util import run_default_backend + +main = run_default_backend spec + +spec setup = + prefix = setup.prefix + table_builder = setup.table_builder + + Test.group prefix+"Equality" <| + Test.specify "== Respects Nothing" <| From d76d2abed5242ed9160585710f31851f6ee3764f Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 9 Jan 2024 16:35:29 -0500 Subject: [PATCH 02/55] == --- .../lib/Standard/Table/0.0.0-dev/src/Data/Column.enso | 1 + test/Table_Tests/src/Common_Table_Operations/Main.enso | 2 ++ .../src/Common_Table_Operations/Nothing_Spec.enso | 10 +++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso index 66461c78d524..2421a5f8c447 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso @@ -212,6 +212,7 @@ type Column == self other = new_name = naming_helper.binary_operation_name "==" self other fallback problem_builder a b = + IO.println 'AAA == fallback' if (a.is_a Float) || (b.is_a Float) then problem_builder.reportFloatingPointEquality -1 a == b diff --git a/test/Table_Tests/src/Common_Table_Operations/Main.enso b/test/Table_Tests/src/Common_Table_Operations/Main.enso index 9cbbe384d962..65f19c9ebf25 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Main.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Main.enso @@ -22,6 +22,7 @@ import project.Common_Table_Operations.Join.Union_Spec import project.Common_Table_Operations.Join.Lookup_Spec import project.Common_Table_Operations.Map_Spec import project.Common_Table_Operations.Missing_Values_Spec +import project.Common_Table_Operations.Nothing_Spec import project.Common_Table_Operations.Order_By_Spec import project.Common_Table_Operations.Select_Columns_Spec import project.Common_Table_Operations.Take_Drop_Spec @@ -139,5 +140,6 @@ spec setup = Add_Row_Number_Spec.spec setup Integration_Tests.spec setup Temp_Column_Spec.spec setup + Nothing_Spec.spec setup main = run_default_backend spec diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 3239bf2accc4..2438fb8f0a8a 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -10,6 +10,14 @@ main = run_default_backend spec spec setup = prefix = setup.prefix table_builder = setup.table_builder + IO.println 'AAA NS' - Test.group prefix+"Equality" <| + Test.group prefix+"Equalityasdfasdf" <| Test.specify "== Respects Nothing" <| + scalars = [12, 12.3, "twelve"] + scalars.map x-> + t0 = table_builder [["x", [x, Nothing]]] + c0 = t0.at "x" == Nothing + IO.println 'AAA' + IO.println c0 + c0.to_vector . should_equal [Nothing, Nothing] From f7b18e9988188491917aa9a530138ef46e5098b3 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 12 Jan 2024 15:00:32 -0500 Subject: [PATCH 03/55] comparisons --- .../Common_Table_Operations/Nothing_Spec.enso | 51 +++++++++++++++---- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 2438fb8f0a8a..30a8a1557632 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -1,5 +1,7 @@ from Standard.Base import all +from Standard.Table import all + from Standard.Test import Test import Standard.Test.Extensions @@ -7,17 +9,44 @@ from project.Common_Table_Operations.Util import run_default_backend main = run_default_backend spec +type My_Type + Value x + spec setup = prefix = setup.prefix table_builder = setup.table_builder - IO.println 'AAA NS' - - Test.group prefix+"Equalityasdfasdf" <| - Test.specify "== Respects Nothing" <| - scalars = [12, 12.3, "twelve"] - scalars.map x-> - t0 = table_builder [["x", [x, Nothing]]] - c0 = t0.at "x" == Nothing - IO.println 'AAA' - IO.println c0 - c0.to_vector . should_equal [Nothing, Nothing] + + Test.group prefix+"Comparisons" <| + js_object = JS_Object.from_pairs [["a", 10], ["b", 20]] + values = [[True, Value_Type.Boolean], [My_Type.Value "1", Value_Type.Mixed], [js_object, Value_Type.Mixed]] + + [["twelve", Value_Type.Char], [12, Value_Type.Integer], [9223372036854775807, Value_Type.Integer], [12.3, Value_Type.Float]] + + [[Date.new 2024 1 3, Value_Type.Date], [Date_Time.new 2024 1 3 2 30 10, Value_Type.Date_Time]] + + [[Time_Of_Day.new 2 30 10, Value_Type.Time], [[1], Value_Type.Mixed]] + + [[Nothing, Value_Type.Mixed]] + operations = [["==", ==], ["!=", !=], ["<", <], ["<=", <=], [">", >], [">=", >=]] + + operations.map pair-> + operation_name = pair.at 0 + operation = pair.at 1 + + values.map pair-> + value = pair.at 0 + expected_value_type = pair.at 1 + + Test.specify "Binary op: "+expected_value_type.to_text+" "+operation_name+" Nothing value" <| + t0 = table_builder [["x", [value, Nothing]]] + t0.at "x" . value_type . should_equal expected_value_type + c0 = operation (t0.at "x") Nothing + c0.to_vector . should_equal [Nothing, Nothing] + + Test.specify "Binary op: "+expected_value_type.to_text+" "+operation_name+" Nothing column" <| + t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] + t0.at "x" . value_type . should_equal expected_value_type + c0 = operation (t0.at "x") (t0.at "n") + c0.to_vector . should_equal [Nothing, Nothing] + + Test.specify "Binary op: Nothing column "+expected_value_type.to_text+" "+operation_name <| + t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] + t0.at "x" . value_type . should_equal expected_value_type + c0 = operation (t0.at "n") (t0.at "x") + c0.to_vector . should_equal [Nothing, Nothing] From 9b67c3412f14b72d7427c8d98439b5bf5580f8dd Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 12 Jan 2024 15:07:22 -0500 Subject: [PATCH 04/55] is_nothing --- .../Common_Table_Operations/Nothing_Spec.enso | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 30a8a1557632..057e69643100 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -16,13 +16,13 @@ spec setup = prefix = setup.prefix table_builder = setup.table_builder + js_object = JS_Object.from_pairs [["a", 10], ["b", 20]] + values = [[True, Value_Type.Boolean], [My_Type.Value "1", Value_Type.Mixed], [js_object, Value_Type.Mixed]] + + [["twelve", Value_Type.Char], [12, Value_Type.Integer], [9223372036854775807, Value_Type.Integer], [12.3, Value_Type.Float]] + + [[Date.new 2024 1 3, Value_Type.Date], [Date_Time.new 2024 1 3 2 30 10, Value_Type.Date_Time]] + + [[Time_Of_Day.new 2 30 10, Value_Type.Time], [[1], Value_Type.Mixed]] + Test.group prefix+"Comparisons" <| - js_object = JS_Object.from_pairs [["a", 10], ["b", 20]] - values = [[True, Value_Type.Boolean], [My_Type.Value "1", Value_Type.Mixed], [js_object, Value_Type.Mixed]] - + [["twelve", Value_Type.Char], [12, Value_Type.Integer], [9223372036854775807, Value_Type.Integer], [12.3, Value_Type.Float]] - + [[Date.new 2024 1 3, Value_Type.Date], [Date_Time.new 2024 1 3 2 30 10, Value_Type.Date_Time]] - + [[Time_Of_Day.new 2 30 10, Value_Type.Time], [[1], Value_Type.Mixed]] - + [[Nothing, Value_Type.Mixed]] operations = [["==", ==], ["!=", !=], ["<", <], ["<=", <=], [">", >], [">=", >=]] operations.map pair-> @@ -50,3 +50,14 @@ spec setup = t0.at "x" . value_type . should_equal expected_value_type c0 = operation (t0.at "n") (t0.at "x") c0.to_vector . should_equal [Nothing, Nothing] + + Test.group prefix+"is_nothing" <| + values.map pair-> + value = pair.at 0 + expected_value_type = pair.at 1 + + Test.specify expected_value_type.to_text+".is_nothing" <| + t0 = table_builder [["x", [value, Nothing]]] + t0.at "x" . value_type . should_equal expected_value_type + c0 = t0.at "x" . is_nothing + c0.to_vector . should_equal [False, True] From 2543611ab17715e4d291bec6d4dfdc3b1fc1d8da Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 12 Jan 2024 15:09:59 -0500 Subject: [PATCH 05/55] column --- .../src/Common_Table_Operations/Nothing_Spec.enso | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 057e69643100..04c098dd4289 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -17,10 +17,17 @@ spec setup = table_builder = setup.table_builder js_object = JS_Object.from_pairs [["a", 10], ["b", 20]] - values = [[True, Value_Type.Boolean], [My_Type.Value "1", Value_Type.Mixed], [js_object, Value_Type.Mixed]] - + [["twelve", Value_Type.Char], [12, Value_Type.Integer], [9223372036854775807, Value_Type.Integer], [12.3, Value_Type.Float]] - + [[Date.new 2024 1 3, Value_Type.Date], [Date_Time.new 2024 1 3 2 30 10, Value_Type.Date_Time]] - + [[Time_Of_Day.new 2 30 10, Value_Type.Time], [[1], Value_Type.Mixed]] + values = [[True, Value_Type.Boolean]] + + [[My_Type.Value "1", Value_Type.Mixed]] + + [[js_object, Value_Type.Mixed]] + + [["twelve", Value_Type.Char]] + + [[12, Value_Type.Integer]] + + [[9223372036854775807, Value_Type.Integer]] + + [[12.3, Value_Type.Float]] + + [[Date.new 2024 1 3, Value_Type.Date]] + + [[Date_Time.new 2024 1 3 2 30 10, Value_Type.Date_Time]] + + [[Time_Of_Day.new 2 30 10, Value_Type.Time]] + + [[[1], Value_Type.Mixed]] Test.group prefix+"Comparisons" <| operations = [["==", ==], ["!=", !=], ["<", <], ["<=", <=], [">", >], [">=", >=]] From 0e6422b29ce1e55781b6d370e7112335ded94826 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 12 Jan 2024 15:24:17 -0500 Subject: [PATCH 06/55] between --- .../Common_Table_Operations/Nothing_Spec.enso | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 04c098dd4289..eaaba286283b 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -32,6 +32,12 @@ spec setup = Test.group prefix+"Comparisons" <| operations = [["==", ==], ["!=", !=], ["<", <], ["<=", <=], [">", >], [">=", >=]] + Test.specify "Confirm expected types" <| + values.map pair-> + value = pair.at 0 + expected_value_type = pair.at 1 + table_builder [["x", [value, Nothing]]] . at "x" . value_type . should_equal expected_value_type + operations.map pair-> operation_name = pair.at 0 operation = pair.at 1 @@ -42,22 +48,45 @@ spec setup = Test.specify "Binary op: "+expected_value_type.to_text+" "+operation_name+" Nothing value" <| t0 = table_builder [["x", [value, Nothing]]] - t0.at "x" . value_type . should_equal expected_value_type c0 = operation (t0.at "x") Nothing c0.to_vector . should_equal [Nothing, Nothing] Test.specify "Binary op: "+expected_value_type.to_text+" "+operation_name+" Nothing column" <| t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] - t0.at "x" . value_type . should_equal expected_value_type c0 = operation (t0.at "x") (t0.at "n") c0.to_vector . should_equal [Nothing, Nothing] Test.specify "Binary op: Nothing column "+expected_value_type.to_text+" "+operation_name <| t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] - t0.at "x" . value_type . should_equal expected_value_type c0 = operation (t0.at "n") (t0.at "x") c0.to_vector . should_equal [Nothing, Nothing] + Test.group prefix+"between" <| + values.map pair-> + value = pair.at 0 + expected_value_type = pair.at 1 + t0 = table_builder [["x", [value, Nothing]], ["y", [value, Nothing]], ["n", [Nothing, Nothing]]] + + Test.specify "Nothing column between "+expected_value_type.to_text+" and "+expected_value_type.to_text <| + c0 = t0.at "n" . between (t0.at "x") (t0.at "y") + c0.to_vector . should_equal [Nothing, Nothing] + + Test.specify expected_value_type.to_text+" between Nothing column and "+expected_value_type.to_text <| + c0 = t0.at "x" . between (t0.at "n") (t0.at "y") + c0.to_vector . should_equal [Nothing, Nothing] + + Test.specify expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing column" <| + c0 = t0.at "x" . between (t0.at "y") (t0.at "n") + c0.to_vector . should_equal [Nothing, Nothing] + + Test.specify expected_value_type.to_text+" between Nothing value and "+expected_value_type.to_text <| + c0 = t0.at "x" . between Nothing (t0.at "y") + c0.to_vector . should_equal [Nothing, Nothing] + + Test.specify expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing value" <| + c0 = t0.at "x" . between (t0.at "y") Nothing + c0.to_vector . should_equal [Nothing, Nothing] + Test.group prefix+"is_nothing" <| values.map pair-> value = pair.at 0 @@ -65,6 +94,5 @@ spec setup = Test.specify expected_value_type.to_text+".is_nothing" <| t0 = table_builder [["x", [value, Nothing]]] - t0.at "x" . value_type . should_equal expected_value_type c0 = t0.at "x" . is_nothing c0.to_vector . should_equal [False, True] From 85ca4bfd381784be831ae2089b07824833f9d163 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 12 Jan 2024 15:40:43 -0500 Subject: [PATCH 07/55] is_in --- .../Common_Table_Operations/Nothing_Spec.enso | 66 ++++++++++++------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index eaaba286283b..679434d36069 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -17,34 +17,35 @@ spec setup = table_builder = setup.table_builder js_object = JS_Object.from_pairs [["a", 10], ["b", 20]] - values = [[True, Value_Type.Boolean]] - + [[My_Type.Value "1", Value_Type.Mixed]] - + [[js_object, Value_Type.Mixed]] - + [["twelve", Value_Type.Char]] - + [[12, Value_Type.Integer]] - + [[9223372036854775807, Value_Type.Integer]] - + [[12.3, Value_Type.Float]] - + [[Date.new 2024 1 3, Value_Type.Date]] - + [[Date_Time.new 2024 1 3 2 30 10, Value_Type.Date_Time]] - + [[Time_Of_Day.new 2 30 10, Value_Type.Time]] - + [[[1], Value_Type.Mixed]] + js_object2 = JS_Object.from_pairs [["a", 100], ["b", 200]] + values = [[True, False, Value_Type.Boolean]] + + [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] + + [[js_object, js_object2, Value_Type.Mixed]] + + [["twelve", "thirteen", Value_Type.Char]] + + [[12, 13, Value_Type.Integer]] + + [[9223372036854775807, 13, Value_Type.Integer]] + + [[12.3, 23.4, Value_Type.Float]] + + [[Date.new 2024 1 3, Date.new 2024 1 4, Value_Type.Date]] + + [[Date_Time.new 2024 1 3 2 30 10, Date_Time.new 2024 1 3 2 30 11, Value_Type.Date_Time]] + + [[Time_Of_Day.new 2 30 10, Time_Of_Day.new 2 30 11, Value_Type.Time]] + + [[[1], [2], Value_Type.Mixed]] Test.group prefix+"Comparisons" <| operations = [["==", ==], ["!=", !=], ["<", <], ["<=", <=], [">", >], [">=", >=]] Test.specify "Confirm expected types" <| - values.map pair-> - value = pair.at 0 - expected_value_type = pair.at 1 + values.map triple-> + value = triple.at 0 + expected_value_type = triple.at 2 table_builder [["x", [value, Nothing]]] . at "x" . value_type . should_equal expected_value_type operations.map pair-> operation_name = pair.at 0 operation = pair.at 1 - values.map pair-> - value = pair.at 0 - expected_value_type = pair.at 1 + values.map triple-> + value = triple.at 0 + expected_value_type = triple.at 2 Test.specify "Binary op: "+expected_value_type.to_text+" "+operation_name+" Nothing value" <| t0 = table_builder [["x", [value, Nothing]]] @@ -62,9 +63,9 @@ spec setup = c0.to_vector . should_equal [Nothing, Nothing] Test.group prefix+"between" <| - values.map pair-> - value = pair.at 0 - expected_value_type = pair.at 1 + values.map triple-> + value = triple.at 0 + expected_value_type = triple.at 2 t0 = table_builder [["x", [value, Nothing]], ["y", [value, Nothing]], ["n", [Nothing, Nothing]]] Test.specify "Nothing column between "+expected_value_type.to_text+" and "+expected_value_type.to_text <| @@ -88,11 +89,30 @@ spec setup = c0.to_vector . should_equal [Nothing, Nothing] Test.group prefix+"is_nothing" <| - values.map pair-> - value = pair.at 0 - expected_value_type = pair.at 1 + values.map triple-> + value = triple.at 0 + expected_value_type = triple.at 2 Test.specify expected_value_type.to_text+".is_nothing" <| t0 = table_builder [["x", [value, Nothing]]] c0 = t0.at "x" . is_nothing c0.to_vector . should_equal [False, True] + + Test.group prefix+"is_in" <| + values.map triple-> + value = triple.at 0 + other_value = triple.at 1 + expected_value_type = triple.at 2 + t0 = table_builder [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["n", [Nothing, Nothing]]] + + Test.specify expected_value_type.to_text+".is_in Column (returning True)" <| + t0.at "x" . is_in (t0.at "x") . to_vector . should_equal [True, Nothing] + + Test.specify expected_value_type.to_text+".is_in Column (returning Nothing)" <| + t0.at "x" . is_in (t0.at "y") . to_vector . should_equal [Nothing, Nothing] + + Test.specify expected_value_type.to_text+".is_in Vector (returning True)" <| + t0.at "x" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [True, Nothing] + + Test.specify expected_value_type.to_text+".is_in Vector (returning Nothing)" <| + t0.at "x" . is_in (t0.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] From f4e89ec878d29571dd62b61276a8b9fb78b669b0 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 12 Jan 2024 15:50:27 -0500 Subject: [PATCH 08/55] not --- .../Common_Table_Operations/Nothing_Spec.enso | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 679434d36069..8d8c9b017fed 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -47,17 +47,17 @@ spec setup = value = triple.at 0 expected_value_type = triple.at 2 - Test.specify "Binary op: "+expected_value_type.to_text+" "+operation_name+" Nothing value" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+operation_name+" Nothing value" <| t0 = table_builder [["x", [value, Nothing]]] c0 = operation (t0.at "x") Nothing c0.to_vector . should_equal [Nothing, Nothing] - Test.specify "Binary op: "+expected_value_type.to_text+" "+operation_name+" Nothing column" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+operation_name+" Nothing column" <| t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] c0 = operation (t0.at "x") (t0.at "n") c0.to_vector . should_equal [Nothing, Nothing] - Test.specify "Binary op: Nothing column "+expected_value_type.to_text+" "+operation_name <| + Test.specify "Correctly handle Nothing in: Nothing column "+operation_name+" "+expected_value_type.to_text <| t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] c0 = operation (t0.at "n") (t0.at "x") c0.to_vector . should_equal [Nothing, Nothing] @@ -68,23 +68,23 @@ spec setup = expected_value_type = triple.at 2 t0 = table_builder [["x", [value, Nothing]], ["y", [value, Nothing]], ["n", [Nothing, Nothing]]] - Test.specify "Nothing column between "+expected_value_type.to_text+" and "+expected_value_type.to_text <| + Test.specify "Correctly handle Nothing in: Nothing column between "+expected_value_type.to_text+" and "+expected_value_type.to_text <| c0 = t0.at "n" . between (t0.at "x") (t0.at "y") c0.to_vector . should_equal [Nothing, Nothing] - Test.specify expected_value_type.to_text+" between Nothing column and "+expected_value_type.to_text <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between Nothing column and "+expected_value_type.to_text <| c0 = t0.at "x" . between (t0.at "n") (t0.at "y") c0.to_vector . should_equal [Nothing, Nothing] - Test.specify expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing column" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing column" <| c0 = t0.at "x" . between (t0.at "y") (t0.at "n") c0.to_vector . should_equal [Nothing, Nothing] - Test.specify expected_value_type.to_text+" between Nothing value and "+expected_value_type.to_text <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between Nothing value and "+expected_value_type.to_text <| c0 = t0.at "x" . between Nothing (t0.at "y") c0.to_vector . should_equal [Nothing, Nothing] - Test.specify expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing value" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing value" <| c0 = t0.at "x" . between (t0.at "y") Nothing c0.to_vector . should_equal [Nothing, Nothing] @@ -93,11 +93,16 @@ spec setup = value = triple.at 0 expected_value_type = triple.at 2 - Test.specify expected_value_type.to_text+".is_nothing" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_nothing" <| t0 = table_builder [["x", [value, Nothing]]] c0 = t0.at "x" . is_nothing c0.to_vector . should_equal [False, True] + Test.group prefix+"not" <| + Test.specify "Correctly handle Nothing in .not" <| + t0 = table_builder [["x", [True, False, Nothing]]] + t0.at "x" . not . to_vector . should_equal [False, True, Nothing] + Test.group prefix+"is_in" <| values.map triple-> value = triple.at 0 @@ -105,14 +110,14 @@ spec setup = expected_value_type = triple.at 2 t0 = table_builder [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["n", [Nothing, Nothing]]] - Test.specify expected_value_type.to_text+".is_in Column (returning True)" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Column (returning True)" <| t0.at "x" . is_in (t0.at "x") . to_vector . should_equal [True, Nothing] - Test.specify expected_value_type.to_text+".is_in Column (returning Nothing)" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Column (returning Nothing)" <| t0.at "x" . is_in (t0.at "y") . to_vector . should_equal [Nothing, Nothing] - Test.specify expected_value_type.to_text+".is_in Vector (returning True)" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Vector (returning True)" <| t0.at "x" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [True, Nothing] - Test.specify expected_value_type.to_text+".is_in Vector (returning Nothing)" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Vector (returning Nothing)" <| t0.at "x" . is_in (t0.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] From 9a7b0477fc96c828416ad612fb3c06fcbf569d84 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 12 Jan 2024 16:08:14 -0500 Subject: [PATCH 09/55] distinct, remove js_object --- .../src/Common_Table_Operations/Nothing_Spec.enso | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 8d8c9b017fed..fa3640ee34cd 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -16,11 +16,8 @@ spec setup = prefix = setup.prefix table_builder = setup.table_builder - js_object = JS_Object.from_pairs [["a", 10], ["b", 20]] - js_object2 = JS_Object.from_pairs [["a", 100], ["b", 200]] values = [[True, False, Value_Type.Boolean]] + [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] - + [[js_object, js_object2, Value_Type.Mixed]] + [["twelve", "thirteen", Value_Type.Char]] + [[12, 13, Value_Type.Integer]] + [[9223372036854775807, 13, Value_Type.Integer]] @@ -121,3 +118,14 @@ spec setup = Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Vector (returning Nothing)" <| t0.at "x" . is_in (t0.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] + + Test.group prefix+"distinct" <| + values.map triple-> + value = triple.at 0 + other_value = triple.at 1 + expected_value_type = triple.at 2 + + Test.specify "Correctly handle Nothing in .distinct for "+expected_value_type.to_text <| + t0 = table_builder [["x", [value, other_value, other_value, Nothing, value, Nothing]]] + t1 = t0 . distinct + t1.at "x" . to_vector . should_contain_the_same_elements_as [value, other_value, Nothing] From bba6daa243c20d35a21a3149d524e5a190d39cd2 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 12 Jan 2024 16:17:53 -0500 Subject: [PATCH 10/55] order_by --- .../Common_Table_Operations/Nothing_Spec.enso | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index fa3640ee34cd..42dc910b0ba9 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -18,9 +18,9 @@ spec setup = values = [[True, False, Value_Type.Boolean]] + [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] - + [["twelve", "thirteen", Value_Type.Char]] + + [["abcd", "efgh", Value_Type.Char]] + [[12, 13, Value_Type.Integer]] - + [[9223372036854775807, 13, Value_Type.Integer]] + + [[9223372036854775806, 9223372036854775807, Value_Type.Integer]] + [[12.3, 23.4, Value_Type.Float]] + [[Date.new 2024 1 3, Date.new 2024 1 4, Value_Type.Date]] + [[Date_Time.new 2024 1 3 2 30 10, Date_Time.new 2024 1 3 2 30 11, Value_Type.Date_Time]] @@ -127,5 +127,21 @@ spec setup = Test.specify "Correctly handle Nothing in .distinct for "+expected_value_type.to_text <| t0 = table_builder [["x", [value, other_value, other_value, Nothing, value, Nothing]]] - t1 = t0 . distinct + t1 = t0 . distinct ["x"] t1.at "x" . to_vector . should_contain_the_same_elements_as [value, other_value, Nothing] + + Test.group prefix+"order_by" <| + values.map triple-> + value = triple.at 0 + other_value = triple.at 1 + expected_value_type = triple.at 2 + + t0 = table_builder [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] + + Test.specify "Correctly handle Nothing in .order_by (asc) for "+expected_value_type.to_text <| + t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Ascending] + t1.at "x" . to_vector . should_equal [Nothing, Nothing, Nothing, value, value, other_value, other_value] + + Test.specify "Correctly handle Nothing in .order_by (desc) for "+expected_value_type.to_text <| + t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Descending] + t1.at "x" . to_vector . should_equal [other_value, other_value, value, value, Nothing, Nothing, Nothing] From e35bfdaa0dc56fc64d3eb50b2a4f994e9c4be37b Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 15 Jan 2024 12:49:00 -0500 Subject: [PATCH 11/55] rename to comparisons --- .../Common_Table_Operations/Nothing_Spec.enso | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 42dc910b0ba9..5870d98e0ca2 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -28,7 +28,7 @@ spec setup = + [[[1], [2], Value_Type.Mixed]] Test.group prefix+"Comparisons" <| - operations = [["==", ==], ["!=", !=], ["<", <], ["<=", <=], [">", >], [">=", >=]] + comparisons = [["==", ==], ["!=", !=], ["<", <], ["<=", <=], [">", >], [">=", >=]] Test.specify "Confirm expected types" <| values.map triple-> @@ -36,27 +36,27 @@ spec setup = expected_value_type = triple.at 2 table_builder [["x", [value, Nothing]]] . at "x" . value_type . should_equal expected_value_type - operations.map pair-> - operation_name = pair.at 0 - operation = pair.at 1 + comparisons.map pair-> + comparison_name = pair.at 0 + comparison = pair.at 1 values.map triple-> value = triple.at 0 expected_value_type = triple.at 2 - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+operation_name+" Nothing value" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+comparison_name+" Nothing value" <| t0 = table_builder [["x", [value, Nothing]]] - c0 = operation (t0.at "x") Nothing + c0 = comparison (t0.at "x") Nothing c0.to_vector . should_equal [Nothing, Nothing] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+operation_name+" Nothing column" <| + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+comparison_name+" Nothing column" <| t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] - c0 = operation (t0.at "x") (t0.at "n") + c0 = comparison (t0.at "x") (t0.at "n") c0.to_vector . should_equal [Nothing, Nothing] - Test.specify "Correctly handle Nothing in: Nothing column "+operation_name+" "+expected_value_type.to_text <| + Test.specify "Correctly handle Nothing in: Nothing column "+comparison_name+" "+expected_value_type.to_text <| t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] - c0 = operation (t0.at "n") (t0.at "x") + c0 = comparison (t0.at "n") (t0.at "x") c0.to_vector . should_equal [Nothing, Nothing] Test.group prefix+"between" <| From ad4badaeab4413a616848234219f814167812fdc Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 15 Jan 2024 13:23:53 -0500 Subject: [PATCH 12/55] add Nothing value --- .../Common_Table_Operations/Nothing_Spec.enso | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 5870d98e0ca2..2d7a89bdfefb 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -16,7 +16,7 @@ spec setup = prefix = setup.prefix table_builder = setup.table_builder - values = [[True, False, Value_Type.Boolean]] + values_without_nothing = [[True, False, Value_Type.Boolean]] + [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] + [["abcd", "efgh", Value_Type.Char]] + [[12, 13, Value_Type.Integer]] @@ -26,12 +26,13 @@ spec setup = + [[Date_Time.new 2024 1 3 2 30 10, Date_Time.new 2024 1 3 2 30 11, Value_Type.Date_Time]] + [[Time_Of_Day.new 2 30 10, Time_Of_Day.new 2 30 11, Value_Type.Time]] + [[[1], [2], Value_Type.Mixed]] + values_with_nothing = values_without_nothing + [[Nothing, Nothing, Value_Type.Mixed]] Test.group prefix+"Comparisons" <| comparisons = [["==", ==], ["!=", !=], ["<", <], ["<=", <=], [">", >], [">=", >=]] Test.specify "Confirm expected types" <| - values.map triple-> + values_with_nothing.map triple-> value = triple.at 0 expected_value_type = triple.at 2 table_builder [["x", [value, Nothing]]] . at "x" . value_type . should_equal expected_value_type @@ -40,60 +41,60 @@ spec setup = comparison_name = pair.at 0 comparison = pair.at 1 - values.map triple-> + values_with_nothing.map triple-> value = triple.at 0 expected_value_type = triple.at 2 Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+comparison_name+" Nothing value" <| - t0 = table_builder [["x", [value, Nothing]]] + t0 = table_builder [["x", [value]]] c0 = comparison (t0.at "x") Nothing - c0.to_vector . should_equal [Nothing, Nothing] + c0.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+comparison_name+" Nothing column" <| - t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] + t0 = table_builder [["x", [value]], ["n", [Nothing]]] c0 = comparison (t0.at "x") (t0.at "n") - c0.to_vector . should_equal [Nothing, Nothing] + c0.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: Nothing column "+comparison_name+" "+expected_value_type.to_text <| - t0 = table_builder [["x", [value, Nothing]], ["n", [Nothing, Nothing]]] + t0 = table_builder [["x", [value]], ["n", [Nothing]]] c0 = comparison (t0.at "n") (t0.at "x") - c0.to_vector . should_equal [Nothing, Nothing] + c0.to_vector . should_equal [Nothing] Test.group prefix+"between" <| - values.map triple-> + values_with_nothing.map triple-> value = triple.at 0 expected_value_type = triple.at 2 - t0 = table_builder [["x", [value, Nothing]], ["y", [value, Nothing]], ["n", [Nothing, Nothing]]] + t0 = table_builder [["x", [value]], ["y", [value]], ["n", [Nothing]]] Test.specify "Correctly handle Nothing in: Nothing column between "+expected_value_type.to_text+" and "+expected_value_type.to_text <| c0 = t0.at "n" . between (t0.at "x") (t0.at "y") - c0.to_vector . should_equal [Nothing, Nothing] + c0.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between Nothing column and "+expected_value_type.to_text <| c0 = t0.at "x" . between (t0.at "n") (t0.at "y") - c0.to_vector . should_equal [Nothing, Nothing] + c0.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing column" <| c0 = t0.at "x" . between (t0.at "y") (t0.at "n") - c0.to_vector . should_equal [Nothing, Nothing] + c0.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between Nothing value and "+expected_value_type.to_text <| c0 = t0.at "x" . between Nothing (t0.at "y") - c0.to_vector . should_equal [Nothing, Nothing] + c0.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing value" <| c0 = t0.at "x" . between (t0.at "y") Nothing - c0.to_vector . should_equal [Nothing, Nothing] + c0.to_vector . should_equal [Nothing] Test.group prefix+"is_nothing" <| - values.map triple-> + values_with_nothing.map triple-> value = triple.at 0 expected_value_type = triple.at 2 Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_nothing" <| - t0 = table_builder [["x", [value, Nothing]]] + t0 = table_builder [["x", [value]]] c0 = t0.at "x" . is_nothing - c0.to_vector . should_equal [False, True] + c0.to_vector . should_equal [value == Nothing] Test.group prefix+"not" <| Test.specify "Correctly handle Nothing in .not" <| @@ -101,7 +102,7 @@ spec setup = t0.at "x" . not . to_vector . should_equal [False, True, Nothing] Test.group prefix+"is_in" <| - values.map triple-> + values_without_nothing.map triple-> value = triple.at 0 other_value = triple.at 1 expected_value_type = triple.at 2 @@ -120,7 +121,7 @@ spec setup = t0.at "x" . is_in (t0.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] Test.group prefix+"distinct" <| - values.map triple-> + values_without_nothing.map triple-> value = triple.at 0 other_value = triple.at 1 expected_value_type = triple.at 2 @@ -128,10 +129,18 @@ spec setup = Test.specify "Correctly handle Nothing in .distinct for "+expected_value_type.to_text <| t0 = table_builder [["x", [value, other_value, other_value, Nothing, value, Nothing]]] t1 = t0 . distinct ["x"] - t1.at "x" . to_vector . should_contain_the_same_elements_as [value, other_value, Nothing] + v0 = t1.at "x" . to_vector + v0 . length . should_equal 3 + v0 . should_contain_the_same_elements_as [value, other_value, Nothing] + + Test.specify "Correctly handle Nothing in .distinct for Nothing" <| + t0 = table_builder [["x", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing]]] + t1 = t0 . distinct ["x"] + v0 = t1.at "x" . to_vector + v0 . should_equal [Nothing] Test.group prefix+"order_by" <| - values.map triple-> + values_with_nothing.map triple-> value = triple.at 0 other_value = triple.at 1 expected_value_type = triple.at 2 From 9ef1d053ad9aa5d90c5906143b650ba2a82e401c Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 15 Jan 2024 16:13:43 -0500 Subject: [PATCH 13/55] skip comparables --- .../Common_Table_Operations/Nothing_Spec.enso | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 2d7a89bdfefb..698dfac8938b 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -10,13 +10,14 @@ from project.Common_Table_Operations.Util import run_default_backend main = run_default_backend spec type My_Type - Value x + Value x:Text + spec setup = prefix = setup.prefix table_builder = setup.table_builder - values_without_nothing = [[True, False, Value_Type.Boolean]] + values_without_nothing = [[False, True, Value_Type.Boolean]] + [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] + [["abcd", "efgh", Value_Type.Char]] + [[12, 13, Value_Type.Integer]] @@ -145,12 +146,17 @@ spec setup = other_value = triple.at 1 expected_value_type = triple.at 2 - t0 = table_builder [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] + is_comparable = Default_Comparator.compare value other_value != Nothing + + if is_comparable then + t0 = table_builder [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] - Test.specify "Correctly handle Nothing in .order_by (asc) for "+expected_value_type.to_text <| - t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Ascending] - t1.at "x" . to_vector . should_equal [Nothing, Nothing, Nothing, value, value, other_value, other_value] + Test.specify "Correctly handle Nothing in .order_by (asc) for "+expected_value_type.to_text <| + #IO.println (Nothing < Nothing) + #IO.println (My_Type.Value "1" < My_Type.Value "2") + t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Ascending] + t1.at "x" . to_vector . should_equal [Nothing, Nothing, Nothing, value, value, other_value, other_value] - Test.specify "Correctly handle Nothing in .order_by (desc) for "+expected_value_type.to_text <| - t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Descending] - t1.at "x" . to_vector . should_equal [other_value, other_value, value, value, Nothing, Nothing, Nothing] + Test.specify "Correctly handle Nothing in .order_by (desc) for "+expected_value_type.to_text <| + t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Descending] + t1.at "x" . to_vector . should_equal [other_value, other_value, value, value, Nothing, Nothing, Nothing] From 42b7762e46449b6674946f68bc1cfa0086119998 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 16 Jan 2024 12:33:12 -0500 Subject: [PATCH 14/55] Two more is_in cases, no mixed columns in db test --- .../Common_Table_Operations/Nothing_Spec.enso | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 698dfac8938b..7a647bd54041 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -14,19 +14,20 @@ type My_Type spec setup = - prefix = setup.prefix + prefix = setup.prefix+"asdfasdf" table_builder = setup.table_builder + mixed_values = if setup.is_database then [] else + [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] + [[[1], [2], Value_Type.Mixed]] values_without_nothing = [[False, True, Value_Type.Boolean]] - + [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] - + [["abcd", "efgh", Value_Type.Char]] - + [[12, 13, Value_Type.Integer]] - + [[9223372036854775806, 9223372036854775807, Value_Type.Integer]] - + [[12.3, 23.4, Value_Type.Float]] - + [[Date.new 2024 1 3, Date.new 2024 1 4, Value_Type.Date]] - + [[Date_Time.new 2024 1 3 2 30 10, Date_Time.new 2024 1 3 2 30 11, Value_Type.Date_Time]] - + [[Time_Of_Day.new 2 30 10, Time_Of_Day.new 2 30 11, Value_Type.Time]] - + [[[1], [2], Value_Type.Mixed]] + + [["abcd", "efgh", Value_Type.Char]] + + [[12, 13, Value_Type.Integer]] + + [[9223372036854775806, 9223372036854775807, Value_Type.Integer]] + + [[12.3, 23.4, Value_Type.Float]] + + [[Date.new 2024 1 3, Date.new 2024 1 4, Value_Type.Date]] + + [[Date_Time.new 2024 1 3 2 30 10, Date_Time.new 2024 1 3 2 30 11, Value_Type.Date_Time]] + + [[Time_Of_Day.new 2 30 10, Time_Of_Day.new 2 30 11, Value_Type.Time]] + + mixed_values values_with_nothing = values_without_nothing + [[Nothing, Nothing, Value_Type.Mixed]] Test.group prefix+"Comparisons" <| @@ -115,12 +116,18 @@ spec setup = Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Column (returning Nothing)" <| t0.at "x" . is_in (t0.at "y") . to_vector . should_equal [Nothing, Nothing] + Test.specify "Correctly handle Nothing in: Nothing.is_in Column (returning Nothing)" <| + t0.at "n" . is_in (t0.at "x") . to_vector . should_equal [Nothing, Nothing] + Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Vector (returning True)" <| t0.at "x" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [True, Nothing] Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Vector (returning Nothing)" <| t0.at "x" . is_in (t0.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] + Test.specify "Correctly handle Nothing in: Nothing.is_in Vector (returning Nothing)" <| + t0.at "n" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [Nothing, Nothing] + Test.group prefix+"distinct" <| values_without_nothing.map triple-> value = triple.at 0 @@ -152,8 +159,6 @@ spec setup = t0 = table_builder [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] Test.specify "Correctly handle Nothing in .order_by (asc) for "+expected_value_type.to_text <| - #IO.println (Nothing < Nothing) - #IO.println (My_Type.Value "1" < My_Type.Value "2") t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Ascending] t1.at "x" . to_vector . should_equal [Nothing, Nothing, Nothing, value, value, other_value, other_value] From ef83197e2b96e1e204ce94fb13eb7309a319c4c6 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 16 Jan 2024 12:58:15 -0500 Subject: [PATCH 15/55] cast columns for db backends --- .../Common_Table_Operations/Nothing_Spec.enso | 87 ++++++++++--------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 7a647bd54041..b152a58b9c21 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -16,6 +16,15 @@ type My_Type spec setup = prefix = setup.prefix+"asdfasdf" table_builder = setup.table_builder + table_builder_typed columns types = + Runtime.assert (columns.length == types.length) + cast_columns = columns.map_with_index i-> c-> + value_type = types.at i + Column.from_vector (c.at 0) (c.at 1) . cast value_type + setup.table_builder cast_columns + + # We cannot create a column of Nothing/NULL in the database without casting it to a non-mixed type. + type_for_nothing_column = if setup.is_database then Value_Type.Char else Value_Type.Mixed mixed_values = if setup.is_database then [] else [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] + [[[1], [2], Value_Type.Mixed]] @@ -28,73 +37,67 @@ spec setup = + [[Date_Time.new 2024 1 3 2 30 10, Date_Time.new 2024 1 3 2 30 11, Value_Type.Date_Time]] + [[Time_Of_Day.new 2 30 10, Time_Of_Day.new 2 30 11, Value_Type.Time]] + mixed_values - values_with_nothing = values_without_nothing + [[Nothing, Nothing, Value_Type.Mixed]] + values_with_nothing = values_without_nothing + [[Nothing, Nothing, type_for_nothing_column]] Test.group prefix+"Comparisons" <| comparisons = [["==", ==], ["!=", !=], ["<", <], ["<=", <=], [">", >], [">=", >=]] - Test.specify "Confirm expected types" <| - values_with_nothing.map triple-> - value = triple.at 0 - expected_value_type = triple.at 2 - table_builder [["x", [value, Nothing]]] . at "x" . value_type . should_equal expected_value_type - comparisons.map pair-> comparison_name = pair.at 0 comparison = pair.at 1 values_with_nothing.map triple-> value = triple.at 0 - expected_value_type = triple.at 2 + value_type = triple.at 2 + + t0 = table_builder_typed [["x", [value]], ["n", [Nothing]]] [value_type, value_type] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+comparison_name+" Nothing value" <| - t0 = table_builder [["x", [value]]] + Test.specify "Correctly handle Nothing in: "+value_type.to_text+" "+comparison_name+" Nothing value" <| c0 = comparison (t0.at "x") Nothing c0.to_vector . should_equal [Nothing] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" "+comparison_name+" Nothing column" <| - t0 = table_builder [["x", [value]], ["n", [Nothing]]] + Test.specify "Correctly handle Nothing in: "+value_type.to_text+" "+comparison_name+" Nothing column" <| c0 = comparison (t0.at "x") (t0.at "n") c0.to_vector . should_equal [Nothing] - Test.specify "Correctly handle Nothing in: Nothing column "+comparison_name+" "+expected_value_type.to_text <| - t0 = table_builder [["x", [value]], ["n", [Nothing]]] + Test.specify "Correctly handle Nothing in: Nothing column "+comparison_name+" "+value_type.to_text <| c0 = comparison (t0.at "n") (t0.at "x") c0.to_vector . should_equal [Nothing] Test.group prefix+"between" <| values_with_nothing.map triple-> value = triple.at 0 - expected_value_type = triple.at 2 - t0 = table_builder [["x", [value]], ["y", [value]], ["n", [Nothing]]] + value_type = triple.at 2 - Test.specify "Correctly handle Nothing in: Nothing column between "+expected_value_type.to_text+" and "+expected_value_type.to_text <| + t0 = table_builder_typed [["x", [value]], ["y", [value]], ["n", [Nothing]]] [value_type, value_type, value_type] + + Test.specify "Correctly handle Nothing in: Nothing column between "+value_type.to_text+" and "+value_type.to_text <| c0 = t0.at "n" . between (t0.at "x") (t0.at "y") c0.to_vector . should_equal [Nothing] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between Nothing column and "+expected_value_type.to_text <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+" between Nothing column and "+value_type.to_text <| c0 = t0.at "x" . between (t0.at "n") (t0.at "y") c0.to_vector . should_equal [Nothing] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing column" <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+" between "+value_type.to_text+" and Nothing column" <| c0 = t0.at "x" . between (t0.at "y") (t0.at "n") c0.to_vector . should_equal [Nothing] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between Nothing value and "+expected_value_type.to_text <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+" between Nothing value and "+value_type.to_text <| c0 = t0.at "x" . between Nothing (t0.at "y") c0.to_vector . should_equal [Nothing] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+" between "+expected_value_type.to_text+" and Nothing value" <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+" between "+value_type.to_text+" and Nothing value" <| c0 = t0.at "x" . between (t0.at "y") Nothing c0.to_vector . should_equal [Nothing] Test.group prefix+"is_nothing" <| values_with_nothing.map triple-> value = triple.at 0 - expected_value_type = triple.at 2 + value_type = triple.at 2 - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_nothing" <| - t0 = table_builder [["x", [value]]] + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_nothing" <| + t0 = table_builder_typed [["x", [value]]] [value_type] c0 = t0.at "x" . is_nothing c0.to_vector . should_equal [value == Nothing] @@ -104,25 +107,25 @@ spec setup = t0.at "x" . not . to_vector . should_equal [False, True, Nothing] Test.group prefix+"is_in" <| - values_without_nothing.map triple-> + values_with_nothing.map triple-> value = triple.at 0 other_value = triple.at 1 - expected_value_type = triple.at 2 - t0 = table_builder [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["n", [Nothing, Nothing]]] + value_type = triple.at 2 + t0 = table_builder_typed [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["n", [Nothing, Nothing]]] [value_type, value_type, value_type] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Column (returning True)" <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column (returning True)" <| t0.at "x" . is_in (t0.at "x") . to_vector . should_equal [True, Nothing] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Column (returning Nothing)" <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column (returning Nothing)" <| t0.at "x" . is_in (t0.at "y") . to_vector . should_equal [Nothing, Nothing] Test.specify "Correctly handle Nothing in: Nothing.is_in Column (returning Nothing)" <| t0.at "n" . is_in (t0.at "x") . to_vector . should_equal [Nothing, Nothing] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Vector (returning True)" <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector (returning True)" <| t0.at "x" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [True, Nothing] - Test.specify "Correctly handle Nothing in: "+expected_value_type.to_text+".is_in Vector (returning Nothing)" <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector (returning Nothing)" <| t0.at "x" . is_in (t0.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] Test.specify "Correctly handle Nothing in: Nothing.is_in Vector (returning Nothing)" <| @@ -132,36 +135,36 @@ spec setup = values_without_nothing.map triple-> value = triple.at 0 other_value = triple.at 1 - expected_value_type = triple.at 2 + value_type = triple.at 2 - Test.specify "Correctly handle Nothing in .distinct for "+expected_value_type.to_text <| + Test.specify "Correctly handle Nothing in .distinct for "+value_type.to_text <| t0 = table_builder [["x", [value, other_value, other_value, Nothing, value, Nothing]]] t1 = t0 . distinct ["x"] v0 = t1.at "x" . to_vector v0 . length . should_equal 3 v0 . should_contain_the_same_elements_as [value, other_value, Nothing] - Test.specify "Correctly handle Nothing in .distinct for Nothing" <| - t0 = table_builder [["x", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing]]] - t1 = t0 . distinct ["x"] - v0 = t1.at "x" . to_vector - v0 . should_equal [Nothing] + Test.specify "Correctly handle Nothing in .distinct for Nothing" <| + t0 = table_builder_typed [["x", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing]]] [Value_Type.Char] + t1 = t0 . distinct ["x"] + v0 = t1.at "x" . to_vector + v0 . should_equal [Nothing] Test.group prefix+"order_by" <| values_with_nothing.map triple-> value = triple.at 0 other_value = triple.at 1 - expected_value_type = triple.at 2 + value_type = triple.at 2 is_comparable = Default_Comparator.compare value other_value != Nothing if is_comparable then - t0 = table_builder [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] + t0 = table_builder_typed [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] [value_type] - Test.specify "Correctly handle Nothing in .order_by (asc) for "+expected_value_type.to_text <| + Test.specify "Correctly handle Nothing in .order_by (asc) for "+value_type.to_text <| t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Ascending] t1.at "x" . to_vector . should_equal [Nothing, Nothing, Nothing, value, value, other_value, other_value] - Test.specify "Correctly handle Nothing in .order_by (desc) for "+expected_value_type.to_text <| + Test.specify "Correctly handle Nothing in .order_by (desc) for "+value_type.to_text <| t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Descending] t1.at "x" . to_vector . should_equal [other_value, other_value, value, value, Nothing, Nothing, Nothing] From 0a62d1d3eac4fc8c4f6d954b34b35433e414bbd3 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 16 Jan 2024 13:00:02 -0500 Subject: [PATCH 16/55] no date_time for sqlite --- .../src/Common_Table_Operations/Nothing_Spec.enso | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index b152a58b9c21..7dc5bd57a0fb 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -26,16 +26,17 @@ spec setup = # We cannot create a column of Nothing/NULL in the database without casting it to a non-mixed type. type_for_nothing_column = if setup.is_database then Value_Type.Char else Value_Type.Mixed - mixed_values = if setup.is_database then [] else - [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] + [[[1], [2], Value_Type.Mixed]] + mixed_values = if setup.is_database then [] else [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] + + [[[1], [2], Value_Type.Mixed]] + date_time_values = if setup.test_selection.date_time.not then [] else [[Date.new 2024 1 3, Date.new 2024 1 4, Value_Type.Date]] + + [[Date_Time.new 2024 1 3 2 30 10, Date_Time.new 2024 1 3 2 30 11, Value_Type.Date_Time]] + + [[Time_Of_Day.new 2 30 10, Time_Of_Day.new 2 30 11, Value_Type.Time]] values_without_nothing = [[False, True, Value_Type.Boolean]] + [["abcd", "efgh", Value_Type.Char]] + [[12, 13, Value_Type.Integer]] + [[9223372036854775806, 9223372036854775807, Value_Type.Integer]] + [[12.3, 23.4, Value_Type.Float]] - + [[Date.new 2024 1 3, Date.new 2024 1 4, Value_Type.Date]] - + [[Date_Time.new 2024 1 3 2 30 10, Date_Time.new 2024 1 3 2 30 11, Value_Type.Date_Time]] - + [[Time_Of_Day.new 2 30 10, Time_Of_Day.new 2 30 11, Value_Type.Time]] + + date_time_values + mixed_values values_with_nothing = values_without_nothing + [[Nothing, Nothing, type_for_nothing_column]] From 9046cba2db3247b44170fb6db03f8104950ca625 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 16 Jan 2024 15:26:47 -0500 Subject: [PATCH 17/55] remove special null handling for is_in Vector --- .../Database/0.0.0-dev/src/Data/Column.enso | 14 +------------- .../0.0.0-dev/src/Internal/Base_Generator.enso | 5 +---- .../src/Common_Table_Operations/Nothing_Spec.enso | 6 ++++-- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 2a5f687856b9..c5df08a2ed9e 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1536,19 +1536,7 @@ type Column fix it later) and setting up the query - but at the set up this only applies to adding nulls - setting any other object does not check the type at this level anyway. - partitioned = vector.partition .is_nothing - nulls = partitioned.first - non_nulls = partitioned.second - ## Since SQL `NULL IN (NULL)` yields `NULL`, we need to handle this case - separately. So we handle all non-null values using `IS_IN` and then - `OR` that with a null check (if the vector contained any nulls to - begin with). The implementation also ensures that even - `NULL IN (...)` is coalesced to False, so that negation works as - expected. - is_in_not_null = self.make_op "IS_IN" operands=non_nulls new_name=new_name - result = case nulls.not_empty of - True -> is_in_not_null || self.is_nothing - False -> is_in_not_null + result = self.make_op "IS_IN" operands=vector new_name=new_name result.rename new_name _ : Array -> self.is_in (Vector.from_polyglot_array vector) column : Column -> if Helpers.check_connection self column . not then (Error.throw (Integrity_Error.Error "Column "+column.name)) else diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index bb16dc733587..9e1701b788b6 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -255,10 +255,7 @@ make_is_in arguments = case arguments.length of _ -> expr = arguments.first list = arguments.drop 1 - is_in = expr ++ " IN (" ++ (Builder.join ", " list) ++ ")" - ## We ensure that even `NULL IN (...)` is coalesced to False, so that - negation will work as expected. - Builder.code "COALESCE(" ++ is_in ++ ", FALSE)" + expr ++ " IN (" ++ (Builder.join ", " list) ++ ")" ## PRIVATE make_is_in_column : Vector Builder -> Builder diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 7dc5bd57a0fb..8ff5ce124d79 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -114,8 +114,10 @@ spec setup = value_type = triple.at 2 t0 = table_builder_typed [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["n", [Nothing, Nothing]]] [value_type, value_type, value_type] + true_if_not_nothing = if value == Nothing then Nothing else True + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column (returning True)" <| - t0.at "x" . is_in (t0.at "x") . to_vector . should_equal [True, Nothing] + t0.at "x" . is_in (t0.at "x") . to_vector . should_equal [true_if_not_nothing, Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column (returning Nothing)" <| t0.at "x" . is_in (t0.at "y") . to_vector . should_equal [Nothing, Nothing] @@ -124,7 +126,7 @@ spec setup = t0.at "n" . is_in (t0.at "x") . to_vector . should_equal [Nothing, Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector (returning True)" <| - t0.at "x" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [True, Nothing] + t0.at "x" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [true_if_not_nothing, Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector (returning Nothing)" <| t0.at "x" . is_in (t0.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] From 49c54e62652c3dc0bc892724640d536d71f20f6e Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 16 Jan 2024 15:39:33 -0500 Subject: [PATCH 18/55] remove special null handling for is_in Column --- .../Standard/Database/0.0.0-dev/src/Data/Column.enso | 12 +----------- .../0.0.0-dev/src/Internal/Base_Generator.enso | 9 +++------ 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index c5df08a2ed9e..37a9c3463fdc 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1540,18 +1540,8 @@ type Column result.rename new_name _ : Array -> self.is_in (Vector.from_polyglot_array vector) column : Column -> if Helpers.check_connection self column . not then (Error.throw (Integrity_Error.Error "Column "+column.name)) else - ## We slightly abuse the expression syntax putting a Query as one of - the sub-expressions. Once type-checking is added, we may need to - amend the signature of `SQL_Expression.Operation` to account for - this. Also, unfortunately as `NULL IN (...)` is `NULL` in SQL, we - need to do separate handling of nulls - we check if the target - column has any nulls and if so, we will do `IS NULL` checks for - our columns too. That is because, we want the containment check - for `NULL` to work the same way as for any other value. in_subquery = Query.Select [Pair.new column.name column.expression] column.context - has_nulls_expression = SQL_Expression.Operation "BOOL_OR" [column.is_nothing.expression] - has_nulls_subquery = Query.Select [Pair.new "has_nulls" has_nulls_expression] column.context - new_expr = SQL_Expression.Operation "IS_IN_COLUMN" [self.expression, in_subquery, has_nulls_subquery] + new_expr = SQL_Expression.Operation "IS_IN_COLUMN" [self.expression, in_subquery] # This mapping should never be imprecise, if there are errors we need to amend the implementation. sql_type = self.connection.dialect.get_type_mapping.value_type_to_sql Value_Type.Boolean Problem_Behavior.Report_Error new_type_ref = SQL_Type_Reference.from_constant sql_type . catch Inexact_Type_Coercion _-> diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index 9e1701b788b6..33015e7c7405 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -260,14 +260,11 @@ make_is_in arguments = case arguments.length of ## PRIVATE make_is_in_column : Vector Builder -> Builder make_is_in_column arguments = case arguments.length of - 3 -> + 2 -> expr = arguments.at 0 in_query = arguments.at 1 - has_nulls_query = arguments.at 2 - is_in = Builder.code "COALESCE(" ++ expr ++ " IN (" ++ in_query ++ "), FALSE)" - has_nulls = has_nulls_query.paren ++ " = TRUE" - Builder.code "CASE WHEN " ++ expr ++ " IS NULL THEN " ++ has_nulls ++ " ELSE " ++ is_in ++ " END" - _ -> Error.throw <| Illegal_State.Error ("The operation IS_IN_COLUMN requires at exactly 3 arguments: the expression, the IN subquery, the subquery checking for nulls.") + Builder.code "(" ++ expr ++ " IN (" ++ in_query ++ "))" + _ -> Error.throw <| Illegal_State.Error ("The operation IS_IN_COLUMN requires at exactly 2 arguments: the expression and the IN subquery.") ## PRIVATE make_row_number : Vector Builder -> Row_Number_Metadata -> Builder From db5634f3b5bf08d2b7ef0194aeeb6da8923ce2bb Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 16 Jan 2024 15:52:38 -0500 Subject: [PATCH 19/55] fix time zone test failures in postgres --- .../Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 8ff5ce124d79..bf8573ff5571 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -26,10 +26,12 @@ spec setup = # We cannot create a column of Nothing/NULL in the database without casting it to a non-mixed type. type_for_nothing_column = if setup.is_database then Value_Type.Char else Value_Type.Mixed + zone = Time_Zone.parse "Z" + mixed_values = if setup.is_database then [] else [[My_Type.Value "1", My_Type.Value "2", Value_Type.Mixed]] + [[[1], [2], Value_Type.Mixed]] date_time_values = if setup.test_selection.date_time.not then [] else [[Date.new 2024 1 3, Date.new 2024 1 4, Value_Type.Date]] - + [[Date_Time.new 2024 1 3 2 30 10, Date_Time.new 2024 1 3 2 30 11, Value_Type.Date_Time]] + + [[Date_Time.new 2024 1 3 2 30 10 zone=zone, Date_Time.new 2024 1 3 2 30 11 zone=zone, Value_Type.Date_Time]] + [[Time_Of_Day.new 2 30 10, Time_Of_Day.new 2 30 11, Value_Type.Time]] values_without_nothing = [[False, True, Value_Type.Boolean]] + [["abcd", "efgh", Value_Type.Char]] From 1c94d881ba6a52d535cb9ebb51cee346e9c0d250 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 16 Jan 2024 16:00:41 -0500 Subject: [PATCH 20/55] is_in test with no nothings in the col --- .../Common_Table_Operations/Nothing_Spec.enso | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index bf8573ff5571..e780abdc2ee3 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -114,26 +114,32 @@ spec setup = value = triple.at 0 other_value = triple.at 1 value_type = triple.at 2 - t0 = table_builder_typed [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["n", [Nothing, Nothing]]] [value_type, value_type, value_type] + t0 = table_builder_typed [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["z", [value, other_value]], ["n", [Nothing, Nothing]]] [value_type, value_type, value_type, value_type] true_if_not_nothing = if value == Nothing then Nothing else True Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column (returning True)" <| + t0.at "x" . is_in (t0.at "z") . to_vector . should_equal [true_if_not_nothing, Nothing] + + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column with Nothings (returning True)" <| t0.at "x" . is_in (t0.at "x") . to_vector . should_equal [true_if_not_nothing, Nothing] - Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column (returning Nothing)" <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column with Nothings (returning Nothing)" <| t0.at "x" . is_in (t0.at "y") . to_vector . should_equal [Nothing, Nothing] - Test.specify "Correctly handle Nothing in: Nothing.is_in Column (returning Nothing)" <| + Test.specify "Correctly handle Nothing in: Nothing.is_in Column with Nothings (returning Nothing)" <| t0.at "n" . is_in (t0.at "x") . to_vector . should_equal [Nothing, Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector (returning True)" <| + t0.at "x" . is_in (t0.at "z" . to_vector) . to_vector . should_equal [true_if_not_nothing, Nothing] + + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector with Nothings (returning True)" <| t0.at "x" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [true_if_not_nothing, Nothing] - Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector (returning Nothing)" <| + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector with Nothings (returning Nothing)" <| t0.at "x" . is_in (t0.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] - Test.specify "Correctly handle Nothing in: Nothing.is_in Vector (returning Nothing)" <| + Test.specify "Correctly handle Nothing in: Nothing.is_in Vector with Nothings (returning Nothing)" <| t0.at "n" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [Nothing, Nothing] Test.group prefix+"distinct" <| From 6f53139d1b37a5e959acafdf316d2ee8d8a34629 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 16 Jan 2024 16:54:28 -0500 Subject: [PATCH 21/55] Fix Char==Nothing non-fallback case --- .../lib/Standard/Table/0.0.0-dev/src/Data/Column.enso | 4 ++++ .../org/enso/table/data/column/storage/StringStorage.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso index 55c71b0efffd..a3c97661e1eb 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso @@ -213,6 +213,10 @@ type Column new_name = naming_helper.binary_operation_name "==" self other fallback problem_builder a b = IO.println 'AAA == fallback' + IO.println self.value_type + IO.println other + if other.is_a Column then + IO.println other.value_type if (a.is_a Float) || (b.is_a Float) then problem_builder.reportFloatingPointEquality -1 a == b diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/storage/StringStorage.java b/std-bits/table/src/main/java/org/enso/table/data/column/storage/StringStorage.java index 7fabfada9395..5baefb27e782 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/storage/StringStorage.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/storage/StringStorage.java @@ -59,7 +59,7 @@ public BoolStorage runBinaryMap( BitSet missing = new BitSet(); Context context = Context.getCurrent(); for (int i = 0; i < storage.size(); i++) { - if (storage.getItem(i) == null) { + if (storage.getItem(i) == null || arg == null) { missing.set(i); } else if (arg instanceof String s && Text_Utils.equals(storage.getItem(i), s)) { r.set(i); From 8416e0be5081d09b5ed3d299e2e378dd6adfb702 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 17 Jan 2024 12:00:23 -0500 Subject: [PATCH 22/55] slow bool is_in, full test cases --- .../operation/map/bool/BooleanIsInOp.java | 40 +++++++++++++- .../Common_Table_Operations/Nothing_Spec.enso | 55 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index 95aa7867c378..e501928dd4e0 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -60,7 +60,44 @@ public Storage runZip( } private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, boolean hadFalse) { - BitSet newVals; + BitSet newVals = new BitSet(storage.size()); + BitSet newMissing = new BitSet(storage.size()); + byte[] bits = storage.getValues().toByteArray(); + System.out.println("vals " + bits); + for (int i = 0; i < storage.size(); ++i) { + boolean bitUnNegated = storage.getValues().get(i); + boolean bit = storage.isNegated() ? !bitUnNegated : bitUnNegated; + System.out.println("bits " + bitUnNegated + " " + bit); + boolean inputValue = storage.getItem(i); + boolean inputIsNull = storage.isNa(i); + if (inputIsNull) { + newMissing.set(i, true); + } else if (inputValue) { + if (hadTrue) { + newVals.set(i, true); + } else { + if (hadNull) { + newMissing.set(i, true); + } else { + newVals.set(i, false); + } + } + } else { // inputValue==false + if (hadFalse) { + newVals.set(i, true); + } else { + if (hadNull) { + newMissing.set(i, true); + } else { + newVals.set(i, false); + } + } + } + } + + return new BoolStorage(newVals, newMissing, storage.size(), false); + +/* boolean negated = false; if (hadNull && hadTrue && hadFalse) { @@ -98,5 +135,6 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b } return new BoolStorage(newVals, new BitSet(), storage.size(), negated); + */ } } diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index e780abdc2ee3..dfdb4ece4302 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -142,6 +142,61 @@ spec setup = Test.specify "Correctly handle Nothing in: Nothing.is_in Vector with Nothings (returning Nothing)" <| t0.at "n" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [Nothing, Nothing] + Test.group prefix+"Boolean is_in" <| + make_containing_values had_null had_true had_false = + null_maybe = if had_null then [Nothing] else [] + true_maybe = if had_true then [True] else [] + false_maybe = if had_false then [False] else [] + null_maybe + true_maybe + false_maybe + + cases = [[True, True, True, True, True]] + + [[True, True, True, False, True]] + + [[True, False, True, True, True]] + + [[True, False, True, False, True]] + + [[True, True, False, True, Nothing]] + + [[True, True, False, False, Nothing]] + + [[True, False, False, True, False]] + + [[True, False, False, False, False]] + + [[False, True, True, True, True]] + + [[False, True, False, True, True]] + + [[False, False, True, True, True]] + + [[False, False, False, True, True]] + + [[False, True, True, False, Nothing]] + + [[False, True, False, False, Nothing]] + + [[False, False, True, False, False]] + + [[False, False, False, False, False]] + + [[Nothing, True, True, True, Nothing]] + + [[Nothing, True, False, True, Nothing]] + + [[Nothing, False, True, True, Nothing]] + + [[Nothing, False, False, True, Nothing]] + + [[Nothing, True, True, False, Nothing]] + + [[Nothing, True, False, False, Nothing]] + + [[Nothing, False, True, False, Nothing]] + + [[Nothing, False, False, False, Nothing]] + + cases.map cs-> + input = cs.at 0 + output = cs.at 4 + containing_values = make_containing_values (cs.at 1) (cs.at 2) (cs.at 3) + + Test.specify "Boolean is_in: (Vector) "+cs.to_text <| + input_column = [input] + t = table_builder_typed [["input", input_column]] [Value_Type.Boolean] + + c = t.at "input" . is_in containing_values + + c.to_vector . should_equal [output] + + Test.specify "Boolean is_in: (Column) "+cs.to_text <| + input_column = Vector.fill containing_values.length input + t = table_builder_typed [["input", input_column], ["containing", containing_values]] [Value_Type.Boolean, Value_Type.Boolean] + expected_output = if input_column.is_empty then [] else [output] + + c = t.at "input" . is_in (t.at "containing") + + c.to_vector . length . should_equal input_column.length + c.to_vector.distinct . should_equal expected_output + Test.group prefix+"distinct" <| values_without_nothing.map triple-> value = triple.at 0 From 10ef070d30636539b3dab880f8b0dce017755dc2 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 17 Jan 2024 12:35:15 -0500 Subject: [PATCH 23/55] four negation cases --- .../Common_Table_Operations/Nothing_Spec.enso | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index dfdb4ece4302..03e10ae5dd08 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -149,6 +149,7 @@ spec setup = false_maybe = if had_false then [False] else [] null_maybe + true_maybe + false_maybe + # [input, had_null, had_true, had_false, output] cases = [[True, True, True, True, True]] + [[True, True, True, False, True]] + [[True, False, True, True, True]] @@ -174,28 +175,42 @@ spec setup = + [[Nothing, False, True, False, Nothing]] + [[Nothing, False, False, False, Nothing]] - cases.map cs-> - input = cs.at 0 - output = cs.at 4 - containing_values = make_containing_values (cs.at 1) (cs.at 2) (cs.at 3) + negate : Vector | Column -> Vector | Column + negate vorc = case vorc of + _ : Vector -> vorc.map (x-> x.if_not_nothing x.not) + _ : Column -> vorc.not + negation_cases = [["normal input, normal argument", identity, identity]] + + [["negated input, normal argument", negate, identity]] + + [["normal input, negated argument", identity, negate]] + + [["negated input, negated argument", negate, negate]] - Test.specify "Boolean is_in: (Vector) "+cs.to_text <| - input_column = [input] - t = table_builder_typed [["input", input_column]] [Value_Type.Boolean] + negation_cases.map negation_case-> + negation_desc = negation_case.at 0 + neg_input = negation_case.at 1 + neg_argument = negation_case.at 2 - c = t.at "input" . is_in containing_values + cases.map cs-> + input = cs.at 0 + output = cs.at 4 + containing_values = make_containing_values (cs.at 1) (cs.at 2) (cs.at 3) - c.to_vector . should_equal [output] + Test.specify "Boolean is_in: (Vector), "+negation_desc+" "+cs.to_text <| + input_column = neg_input [input] + t = table_builder_typed [["input", input_column]] [Value_Type.Boolean] - Test.specify "Boolean is_in: (Column) "+cs.to_text <| - input_column = Vector.fill containing_values.length input - t = table_builder_typed [["input", input_column], ["containing", containing_values]] [Value_Type.Boolean, Value_Type.Boolean] - expected_output = if input_column.is_empty then [] else [output] + c = (neg_input (t.at "input")) . is_in containing_values - c = t.at "input" . is_in (t.at "containing") + c.to_vector . should_equal [output] - c.to_vector . length . should_equal input_column.length - c.to_vector.distinct . should_equal expected_output + Test.specify "Boolean is_in: (Column), "+negation_desc+" "+cs.to_text <| + input_column = neg_input (Vector.fill containing_values.length input) + t = table_builder_typed [["input", input_column], ["containing", neg_argument containing_values]] [Value_Type.Boolean, Value_Type.Boolean] + expected_output = if input_column.is_empty then [] else [output] + + c = (neg_input (t.at "input")) . is_in (neg_argument (t.at "containing")) + + c.to_vector . length . should_equal input_column.length + c.to_vector.distinct . should_equal expected_output Test.group prefix+"distinct" <| values_without_nothing.map triple-> From ae8c4c60e350d1632d23d4a68bfec46995a99956 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 17 Jan 2024 14:34:19 -0500 Subject: [PATCH 24/55] optimized --- .../operation/map/bool/BooleanIsInOp.java | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index e501928dd4e0..d0423f1f6189 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -60,6 +60,107 @@ public Storage runZip( } private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, boolean hadFalse) { + BitSet values = storage.getValues(); + BitSet missing = storage.getIsMissing(); + boolean negated = storage.isNegated(); + + BitSet newValues; + BitSet newMissing; + + if (hadNull) { + if (hadTrue) { + if (hadFalse) { + // t t t + newValues = (BitSet) missing.clone(); + newValues.flip(0, storage.size()); + newMissing = missing; + } else { + // t t f + BitSet oldValuesFlipped = (BitSet) values.clone(); + oldValuesFlipped.flip(0, storage.size()); + if (negated) { + newValues = (BitSet) missing.clone(); + newValues.flip(0, storage.size()); + newValues.and(oldValuesFlipped); + newMissing = (BitSet) missing.clone(); + newMissing.or(values); + } else { + newValues = (BitSet) missing.clone(); + newValues.flip(0, storage.size()); + newValues.and(values); + newMissing = (BitSet) missing.clone(); + newMissing.or(oldValuesFlipped); + } + } + } else { + if (hadFalse) { + // t f t + BitSet oldMissingFlipped = (BitSet) missing.clone(); + oldMissingFlipped.flip(0, storage.size()); + BitSet oldValuesFlipped = (BitSet) values.clone(); + oldValuesFlipped.flip(0, storage.size()); + if (negated) { + newValues = oldMissingFlipped; + newValues.and(values); + newMissing = (BitSet) missing.clone(); + newMissing.or(oldValuesFlipped); + } else { + newValues = oldMissingFlipped; + newValues.and(oldValuesFlipped); + newMissing = (BitSet) missing.clone(); + newMissing.or(values); + } + } else { + // t f f + newValues = new BitSet(storage.size()); // Values don't matter + newMissing = new BitSet(storage.size()); + newMissing.flip(0, storage.size()); + } + } + } else { + if (hadTrue) { + if (hadFalse) { + // f t t + newValues = (BitSet) missing.clone(); + newValues.flip(0, storage.size()); + newMissing = (BitSet) missing.clone(); + } else { + // f t f + newValues = (BitSet) missing.clone(); + newValues.flip(0, storage.size()); + if (negated) { + BitSet oldValuesFlipped = (BitSet) values.clone(); + oldValuesFlipped.flip(0, storage.size()); + newValues.and(oldValuesFlipped); + } else { + newValues.and(values); + } + newMissing = missing; + } + } else { + if (hadFalse) { + // f f t + newValues = (BitSet) missing.clone(); + newValues.flip(0, storage.size()); + if (negated) { + newValues.and(values); + } else { + BitSet oldValuesFlipped = (BitSet) values.clone(); + oldValuesFlipped.flip(0, storage.size()); + newValues.and(oldValuesFlipped); + } + newMissing = missing; + } else { + // f f f + newValues = missing; + newMissing = missing; + } + } + } + + return new BoolStorage(newValues, newMissing, storage.size(), false); + + /* BitSet newVals = new BitSet(storage.size()); BitSet newMissing = new BitSet(storage.size()); byte[] bits = storage.getValues().toByteArray(); @@ -96,6 +197,7 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b } return new BoolStorage(newVals, newMissing, storage.size(), false); + */ /* boolean negated = false; From 573de6150fba518a341ca19c975ff598d881a3c4 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 17 Jan 2024 14:39:19 -0500 Subject: [PATCH 25/55] andNot --- .../operation/map/bool/BooleanIsInOp.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index d0423f1f6189..89c320de122a 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -76,15 +76,15 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b newMissing = missing; } else { // t t f - BitSet oldValuesFlipped = (BitSet) values.clone(); - oldValuesFlipped.flip(0, storage.size()); if (negated) { newValues = (BitSet) missing.clone(); newValues.flip(0, storage.size()); - newValues.and(oldValuesFlipped); + newValues.andNot(values); newMissing = (BitSet) missing.clone(); newMissing.or(values); } else { + BitSet oldValuesFlipped = (BitSet) values.clone(); + oldValuesFlipped.flip(0, storage.size()); newValues = (BitSet) missing.clone(); newValues.flip(0, storage.size()); newValues.and(values); @@ -97,16 +97,16 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b // t f t BitSet oldMissingFlipped = (BitSet) missing.clone(); oldMissingFlipped.flip(0, storage.size()); - BitSet oldValuesFlipped = (BitSet) values.clone(); - oldValuesFlipped.flip(0, storage.size()); if (negated) { + BitSet oldValuesFlipped = (BitSet) values.clone(); + oldValuesFlipped.flip(0, storage.size()); newValues = oldMissingFlipped; newValues.and(values); newMissing = (BitSet) missing.clone(); newMissing.or(oldValuesFlipped); } else { newValues = oldMissingFlipped; - newValues.and(oldValuesFlipped); + newValues.andNot(values); newMissing = (BitSet) missing.clone(); newMissing.or(values); } @@ -129,9 +129,7 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b newValues = (BitSet) missing.clone(); newValues.flip(0, storage.size()); if (negated) { - BitSet oldValuesFlipped = (BitSet) values.clone(); - oldValuesFlipped.flip(0, storage.size()); - newValues.and(oldValuesFlipped); + newValues.andNot(values); } else { newValues.and(values); } @@ -145,9 +143,7 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b if (negated) { newValues.and(values); } else { - BitSet oldValuesFlipped = (BitSet) values.clone(); - oldValuesFlipped.flip(0, storage.size()); - newValues.and(oldValuesFlipped); + newValues.andNot(values); } newMissing = missing; } else { From e923c7a51ea525953f58bbdc918a075fded2f31f Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 17 Jan 2024 14:42:45 -0500 Subject: [PATCH 26/55] ttf comments --- .../column/operation/map/bool/BooleanIsInOp.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index 89c320de122a..cb61dcaeb723 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -70,12 +70,12 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b if (hadNull) { if (hadTrue) { if (hadFalse) { - // t t t + // hadNull=t hadTrue=t hadFalse=t newValues = (BitSet) missing.clone(); newValues.flip(0, storage.size()); newMissing = missing; } else { - // t t f + // hadNull=t hadTrue=t hadFalse=f if (negated) { newValues = (BitSet) missing.clone(); newValues.flip(0, storage.size()); @@ -94,7 +94,7 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b } } else { if (hadFalse) { - // t f t + // hadNull=t hadTrue=f hadFalse=t BitSet oldMissingFlipped = (BitSet) missing.clone(); oldMissingFlipped.flip(0, storage.size()); if (negated) { @@ -111,7 +111,7 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b newMissing.or(values); } } else { - // t f f + // hadNull=t hadTrue=f hadFalse=f newValues = new BitSet(storage.size()); // Values don't matter newMissing = new BitSet(storage.size()); newMissing.flip(0, storage.size()); @@ -120,12 +120,12 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b } else { if (hadTrue) { if (hadFalse) { - // f t t + // hadNull=f hadTrue=t hadFalse=t newValues = (BitSet) missing.clone(); newValues.flip(0, storage.size()); newMissing = (BitSet) missing.clone(); } else { - // f t f + // hadNull=f hadTrue=t hadFalse=f newValues = (BitSet) missing.clone(); newValues.flip(0, storage.size()); if (negated) { @@ -137,7 +137,7 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b } } else { if (hadFalse) { - // f f t + // hadNull=f hadTrue=f hadFalse=t newValues = (BitSet) missing.clone(); newValues.flip(0, storage.size()); if (negated) { @@ -147,7 +147,7 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b } newMissing = missing; } else { - // f f f + // hadNull=f hadTrue=f hadFalse=f newValues = missing; newMissing = missing; } From 84259981b373001ca8318050519540abf2e97ea3 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 17 Jan 2024 14:59:31 -0500 Subject: [PATCH 27/55] all is_in but Mixed --- .../data/column/operation/map/SpecializedIsInOp.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java index 2218bd3e3725..16f2e2674ab3 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java @@ -55,16 +55,21 @@ public Storage runMap(S storage, List arg) { Context context = Context.getCurrent(); CompactRepresentation compactRepresentation = prepareList(arg); BitSet newVals = new BitSet(); + BitSet missing = new BitSet(); for (int i = 0; i < storage.size(); i++) { - if (storage.isNa(i) && compactRepresentation.hasNulls) { - newVals.set(i); + if (storage.isNa(i)) { + missing.set(i); } else if (compactRepresentation.coercedValues.contains(storage.getItemBoxed(i))) { newVals.set(i); + } else if (compactRepresentation.hasNulls) { + missing.set(i); + } else { + // Leave as default=false } context.safepoint(); } - return new BoolStorage(newVals, new BitSet(), storage.size(), false); + return new BoolStorage(newVals, missing, storage.size(), false); } @Override From 665a6a8ff3a520f1b72e0c084d179c5005124d77 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 17 Jan 2024 15:13:41 -0500 Subject: [PATCH 28/55] all is_in --- .../lib/Standard/Table/0.0.0-dev/src/Data/Column.enso | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso index a3c97661e1eb..cf5480860bf5 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso @@ -1605,7 +1605,9 @@ type Column run_vectorized_binary_op self op_name as_vector expected_result_type=Value_Type.Boolean skip_nulls=False new_name=result_name False -> set = Set.from_vector as_vector error_on_duplicates=False - run_unary_op self set.contains new_name=result_name skip_nulls=False expected_result_type=Value_Type.Boolean + set_has_nulls = set.contains Nothing + contains x = if set.contains x then True else (if set_has_nulls then Nothing else False) + run_unary_op self contains new_name=result_name skip_nulls=True expected_result_type=Value_Type.Boolean ## GROUP Standard.Base.Conversions ICON convert From 577a9930fd7d6603eece9db2621a547c75556beb Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 17 Jan 2024 15:23:31 -0500 Subject: [PATCH 29/55] handle Database Column in negator --- test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 03e10ae5dd08..a2fe7d6cccdd 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -2,6 +2,8 @@ from Standard.Base import all from Standard.Table import all +import Standard.Database.Data.Column.Column as Database_Column + from Standard.Test import Test import Standard.Test.Extensions @@ -179,6 +181,7 @@ spec setup = negate vorc = case vorc of _ : Vector -> vorc.map (x-> x.if_not_nothing x.not) _ : Column -> vorc.not + _ : Database_Column -> vorc.not negation_cases = [["normal input, normal argument", identity, identity]] + [["negated input, normal argument", negate, identity]] + [["normal input, negated argument", identity, negate]] From a2ef8fd27d0f67bc79c7827638fde7b9df8c3501 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 17 Jan 2024 15:29:37 -0500 Subject: [PATCH 30/55] NULL.is_in empty set is false --- .../table/data/column/operation/map/bool/BooleanIsInOp.java | 5 +++-- .../src/Common_Table_Operations/Nothing_Spec.enso | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index cb61dcaeb723..896739ee0df7 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -148,8 +148,9 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b newMissing = missing; } else { // hadNull=f hadTrue=f hadFalse=f - newValues = missing; - newMissing = missing; + // NULL.is_in([]) is false, not NULL + newValues = new BitSet(); + newMissing = new BitSet(); } } } diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index a2fe7d6cccdd..986770fcc070 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -175,7 +175,7 @@ spec setup = + [[Nothing, True, True, False, Nothing]] + [[Nothing, True, False, False, Nothing]] + [[Nothing, False, True, False, Nothing]] - + [[Nothing, False, False, False, Nothing]] + + [[Nothing, False, False, False, False]] negate : Vector | Column -> Vector | Column negate vorc = case vorc of From dcb3e0aa264c935a51827663a3401b4361c3c6f1 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 18 Jan 2024 11:59:00 -0500 Subject: [PATCH 31/55] pseudocode --- .../operation/map/bool/BooleanIsInOp.java | 64 ++++++++----------- .../org/enso/table/util/ImmutableBitSet.java | 43 +++++++++++++ 2 files changed, 69 insertions(+), 38 deletions(-) create mode 100644 std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index 896739ee0df7..b7247600652b 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -63,6 +63,31 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b BitSet values = storage.getValues(); BitSet missing = storage.getIsMissing(); boolean negated = storage.isNegated(); + int size = values.size(); + + BitSet newValues; + BitSet newMissing; + + BitSet falz = ImmutableBitSet.allFalse(size); + BitSet tru = ImmutableBitSet.allTrue(size); + + if (hasTrue && !hasFalse) { + newValues = storage.isNegated() ? and(not(missing), not(vals)) : and(not(missing), vals); + newMissing = hasNull ? or(missing, not(vals)) : missing; + } else if (!hasTrue && hasFalse) { + newValues = storage.isNegated() ? and(not(missing), vals) : and(not(missing), not(vals)); + newMissing = hasNull ? or(missing, vals) : missing; + } else if (hasTrue && hasFalse) { + newValues = not(missing); + newMissing = missing; + } else { + newValues = falz; newMissing = hasNull ? tru : falz; + } + + /* + BitSet values = storage.getValues(); + BitSet missing = storage.getIsMissing(); + boolean negated = storage.isNegated(); BitSet newValues; BitSet newMissing; @@ -71,69 +96,31 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b if (hadTrue) { if (hadFalse) { // hadNull=t hadTrue=t hadFalse=t - newValues = (BitSet) missing.clone(); - newValues.flip(0, storage.size()); - newMissing = missing; } else { // hadNull=t hadTrue=t hadFalse=f if (negated) { - newValues = (BitSet) missing.clone(); - newValues.flip(0, storage.size()); - newValues.andNot(values); - newMissing = (BitSet) missing.clone(); - newMissing.or(values); } else { - BitSet oldValuesFlipped = (BitSet) values.clone(); - oldValuesFlipped.flip(0, storage.size()); - newValues = (BitSet) missing.clone(); - newValues.flip(0, storage.size()); - newValues.and(values); - newMissing = (BitSet) missing.clone(); - newMissing.or(oldValuesFlipped); } } } else { if (hadFalse) { // hadNull=t hadTrue=f hadFalse=t - BitSet oldMissingFlipped = (BitSet) missing.clone(); - oldMissingFlipped.flip(0, storage.size()); if (negated) { - BitSet oldValuesFlipped = (BitSet) values.clone(); - oldValuesFlipped.flip(0, storage.size()); - newValues = oldMissingFlipped; - newValues.and(values); - newMissing = (BitSet) missing.clone(); - newMissing.or(oldValuesFlipped); } else { - newValues = oldMissingFlipped; - newValues.andNot(values); - newMissing = (BitSet) missing.clone(); - newMissing.or(values); } } else { // hadNull=t hadTrue=f hadFalse=f - newValues = new BitSet(storage.size()); // Values don't matter - newMissing = new BitSet(storage.size()); - newMissing.flip(0, storage.size()); } } } else { if (hadTrue) { if (hadFalse) { // hadNull=f hadTrue=t hadFalse=t - newValues = (BitSet) missing.clone(); - newValues.flip(0, storage.size()); - newMissing = (BitSet) missing.clone(); } else { // hadNull=f hadTrue=t hadFalse=f - newValues = (BitSet) missing.clone(); - newValues.flip(0, storage.size()); if (negated) { - newValues.andNot(values); } else { - newValues.and(values); } - newMissing = missing; } } else { if (hadFalse) { @@ -156,6 +143,7 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b } return new BoolStorage(newValues, newMissing, storage.size(), false); +*/ /* BitSet newVals = new BitSet(storage.size()); diff --git a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java new file mode 100644 index 000000000000..5caa8ff935f3 --- /dev/null +++ b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java @@ -0,0 +1,43 @@ +package org.enso.table.util; + +import java.util.BitSet; +/** + * A wrapper around BitSet that implements boolean operations conveniently. + * Unlike BitSet, ImmutableBitSet takes a size parameter, which allows .not to + * be implemented. + */ +public class ImmutableBitSet { + private BitSet bitSet; + private long size; + + public ImmutableBitSet(BitSet bitSet, long size) { + this.bitSet = bitSet; + this.size = size; + } + + public BitSet and(BitSet other) { + BitSet result = (BitSet) bitSet.clone(); + result.and(other); + return result; + } + + public BitSet or(BitSet other) { + BitSet result = (BitSet) bitSet.clone(); + result.or(other); + return result; + } + + public BitSet not() { + BitSet result = (BitSet) a.clone(); + result.flip(0, size); + return result; + } + + public static BitSet allFalse(long size) { + return new ImmutableBitSet(new BitSet(), size); + } + + public static BitSet allTrue(long size) { + return new ImmutableBitSet(new BitSet(), size).not(); + } +} \ No newline at end of file From fd4952080d0a990be57a28b8cecb0b376f8d9e07 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 18 Jan 2024 12:11:58 -0500 Subject: [PATCH 32/55] builds --- .../operation/map/bool/BooleanIsInOp.java | 38 ++++++++++--------- .../org/enso/table/util/ImmutableBitSet.java | 32 +++++++++------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index b7247600652b..1f3bfc70e23c 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -6,6 +6,7 @@ import org.enso.table.data.column.operation.map.MapOperationProblemAggregator; import org.enso.table.data.column.storage.BoolStorage; import org.enso.table.data.column.storage.Storage; +import org.enso.table.util.ImmutableBitSet; import org.graalvm.polyglot.Context; /** @@ -60,30 +61,33 @@ public Storage runZip( } private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, boolean hadFalse) { - BitSet values = storage.getValues(); - BitSet missing = storage.getIsMissing(); + int size = storage.size(); + ImmutableBitSet values = new ImmutableBitSet(storage.getValues(), size); + ImmutableBitSet missing = new ImmutableBitSet(storage.getIsMissing(), size); boolean negated = storage.isNegated(); - int size = values.size(); - BitSet newValues; - BitSet newMissing; + ImmutableBitSet newValues; + ImmutableBitSet newMissing; + + ImmutableBitSet falz = ImmutableBitSet.allFalse(size); + ImmutableBitSet tru = ImmutableBitSet.allTrue(size); - BitSet falz = ImmutableBitSet.allFalse(size); - BitSet tru = ImmutableBitSet.allTrue(size); - - if (hasTrue && !hasFalse) { - newValues = storage.isNegated() ? and(not(missing), not(vals)) : and(not(missing), vals); - newMissing = hasNull ? or(missing, not(vals)) : missing; - } else if (!hasTrue && hasFalse) { - newValues = storage.isNegated() ? and(not(missing), vals) : and(not(missing), not(vals)); - newMissing = hasNull ? or(missing, vals) : missing; - } else if (hasTrue && hasFalse) { - newValues = not(missing); + if (hadTrue && !hadFalse) { + newValues = storage.isNegated() ? missing.not().and(values.not()) : missing.not().and(values); + newMissing = hadNull ? missing.or(values.not()) : missing; + } else if (!hadTrue && hadFalse) { + newValues = storage.isNegated() ? missing.not().and(values) : missing.not().and(values.not()); + newMissing = hadNull ? missing.or(values) : missing; + } else if (hadTrue && hadFalse) { + newValues = missing.not(); newMissing = missing; } else { - newValues = falz; newMissing = hasNull ? tru : falz; + newValues = falz; + newMissing = hadNull ? tru : falz; } + return new BoolStorage(newValues.toBitSet(), newMissing.toBitSet(), size, false); + /* BitSet values = storage.getValues(); BitSet missing = storage.getIsMissing(); diff --git a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java index 5caa8ff935f3..1ae2ae574b17 100644 --- a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java +++ b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java @@ -8,36 +8,42 @@ */ public class ImmutableBitSet { private BitSet bitSet; - private long size; + private int size; - public ImmutableBitSet(BitSet bitSet, long size) { + public ImmutableBitSet(BitSet bitSet, int size) { this.bitSet = bitSet; this.size = size; } - public BitSet and(BitSet other) { + public BitSet toBitSet() { + return bitSet; + } + + public ImmutableBitSet and(ImmutableBitSet other) { + assert size == other.size; BitSet result = (BitSet) bitSet.clone(); - result.and(other); - return result; + result.and(other.bitSet); + return new ImmutableBitSet(result, size); } - public BitSet or(BitSet other) { + public ImmutableBitSet or(ImmutableBitSet other) { + assert size == other.size; BitSet result = (BitSet) bitSet.clone(); - result.or(other); - return result; + result.or(other.bitSet); + return new ImmutableBitSet(result, size); } - public BitSet not() { - BitSet result = (BitSet) a.clone(); + public ImmutableBitSet not() { + BitSet result = (BitSet) bitSet.clone(); result.flip(0, size); - return result; + return new ImmutableBitSet(result, size); } - public static BitSet allFalse(long size) { + public static ImmutableBitSet allFalse(int size) { return new ImmutableBitSet(new BitSet(), size); } - public static BitSet allTrue(long size) { + public static ImmutableBitSet allTrue(int size) { return new ImmutableBitSet(new BitSet(), size).not(); } } \ No newline at end of file From 4c99f183066de911092d3f0e02e7bf409fb9538f Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 18 Jan 2024 12:32:38 -0500 Subject: [PATCH 33/55] wip --- test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 986770fcc070..4cce8fea32d3 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -145,7 +145,7 @@ spec setup = t0.at "n" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [Nothing, Nothing] Test.group prefix+"Boolean is_in" <| - make_containing_values had_null had_true had_false = + make_containing_values had_null had_true had_false = null_maybe = if had_null then [Nothing] else [] true_maybe = if had_true then [True] else [] false_maybe = if had_false then [False] else [] From 656538c9cb2c539f65061d3027fc49a5abfd4115 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 18 Jan 2024 15:06:56 -0500 Subject: [PATCH 34/55] fixed last cases --- .../table/data/column/operation/map/bool/BooleanIsInOp.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index 1f3bfc70e23c..7cf688b936c7 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -74,10 +74,10 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b if (hadTrue && !hadFalse) { newValues = storage.isNegated() ? missing.not().and(values.not()) : missing.not().and(values); - newMissing = hadNull ? missing.or(values.not()) : missing; + newMissing = hadNull ? (storage.isNegated() ? missing.or(values) : missing.or(values.not())) : missing; } else if (!hadTrue && hadFalse) { newValues = storage.isNegated() ? missing.not().and(values) : missing.not().and(values.not()); - newMissing = hadNull ? missing.or(values) : missing; + newMissing = hadNull ? (storage.isNegated() ? missing.or(values.not()) : missing.or(values)) : missing; } else if (hadTrue && hadFalse) { newValues = missing.not(); newMissing = missing; From 36d9aa73510c2154060675ca9be6e9b9e83d2850 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 18 Jan 2024 15:09:16 -0500 Subject: [PATCH 35/55] andNot --- .../data/column/operation/map/bool/BooleanIsInOp.java | 4 ++-- .../src/main/java/org/enso/table/util/ImmutableBitSet.java | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index 7cf688b936c7..d1a46df82012 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -73,10 +73,10 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b ImmutableBitSet tru = ImmutableBitSet.allTrue(size); if (hadTrue && !hadFalse) { - newValues = storage.isNegated() ? missing.not().and(values.not()) : missing.not().and(values); + newValues = storage.isNegated() ? missing.not().andNot(values) : missing.not().and(values); newMissing = hadNull ? (storage.isNegated() ? missing.or(values) : missing.or(values.not())) : missing; } else if (!hadTrue && hadFalse) { - newValues = storage.isNegated() ? missing.not().and(values) : missing.not().and(values.not()); + newValues = storage.isNegated() ? missing.not().and(values) : missing.not().andNot(values); newMissing = hadNull ? (storage.isNegated() ? missing.or(values.not()) : missing.or(values)) : missing; } else if (hadTrue && hadFalse) { newValues = missing.not(); diff --git a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java index 1ae2ae574b17..0808e7529e20 100644 --- a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java +++ b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java @@ -33,6 +33,13 @@ public ImmutableBitSet or(ImmutableBitSet other) { return new ImmutableBitSet(result, size); } + public ImmutableBitSet andNot(ImmutableBitSet other) { + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + result.andNot(other.bitSet); + return new ImmutableBitSet(result, size); + } + public ImmutableBitSet not() { BitSet result = (BitSet) bitSet.clone(); result.flip(0, size); From 26e537e37a75afa4b9740866cebffcdc4433d343 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 18 Jan 2024 15:11:44 -0500 Subject: [PATCH 36/55] inline t/f all --- .../data/column/operation/map/bool/BooleanIsInOp.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index d1a46df82012..bc67e2ef9a3a 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -69,9 +69,6 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b ImmutableBitSet newValues; ImmutableBitSet newMissing; - ImmutableBitSet falz = ImmutableBitSet.allFalse(size); - ImmutableBitSet tru = ImmutableBitSet.allTrue(size); - if (hadTrue && !hadFalse) { newValues = storage.isNegated() ? missing.not().andNot(values) : missing.not().and(values); newMissing = hadNull ? (storage.isNegated() ? missing.or(values) : missing.or(values.not())) : missing; @@ -82,8 +79,8 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b newValues = missing.not(); newMissing = missing; } else { - newValues = falz; - newMissing = hadNull ? tru : falz; + newValues = ImmutableBitSet.allFalse(size); + newMissing = hadNull ? ImmutableBitSet.allTrue(size) : ImmutableBitSet.allFalse(size); } return new BoolStorage(newValues.toBitSet(), newMissing.toBitSet(), size, false); From 276bf7630a1ddabb1fec39e7fdcb3689aa57e4f2 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 18 Jan 2024 15:27:05 -0500 Subject: [PATCH 37/55] compound operations --- .../operation/map/bool/BooleanIsInOp.java | 8 +++--- .../org/enso/table/util/ImmutableBitSet.java | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index bc67e2ef9a3a..cd61d9a0835e 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -70,11 +70,11 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b ImmutableBitSet newMissing; if (hadTrue && !hadFalse) { - newValues = storage.isNegated() ? missing.not().andNot(values) : missing.not().and(values); - newMissing = hadNull ? (storage.isNegated() ? missing.or(values) : missing.or(values.not())) : missing; + newValues = storage.isNegated() ? missing.notAndNot(values) : missing.notAnd(values); + newMissing = hadNull ? (storage.isNegated() ? missing.or(values) : missing.orNot(values)) : missing; } else if (!hadTrue && hadFalse) { - newValues = storage.isNegated() ? missing.not().and(values) : missing.not().andNot(values); - newMissing = hadNull ? (storage.isNegated() ? missing.or(values.not()) : missing.or(values)) : missing; + newValues = storage.isNegated() ? missing.notAnd(values) : missing.notAndNot(values); + newMissing = hadNull ? (storage.isNegated() ? missing.orNot(values) : missing.or(values)) : missing; } else if (hadTrue && hadFalse) { newValues = missing.not(); newMissing = missing; diff --git a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java index 0808e7529e20..a752e55f148c 100644 --- a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java +++ b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java @@ -46,6 +46,31 @@ public ImmutableBitSet not() { return new ImmutableBitSet(result, size); } + public ImmutableBitSet notAnd(ImmutableBitSet other) { + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + result.flip(0, size); + result.and(other.bitSet); + return new ImmutableBitSet(result, size); + } + + public ImmutableBitSet notAndNot(ImmutableBitSet other) { + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + result.flip(0, size); + result.andNot(other.bitSet); + return new ImmutableBitSet(result, size); + } + + public ImmutableBitSet orNot(ImmutableBitSet other) { + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + BitSet otherNegated = (BitSet) other.bitSet.clone(); + otherNegated.flip(0, size); + result.or(otherNegated); + return new ImmutableBitSet(result, size); + } + public static ImmutableBitSet allFalse(int size) { return new ImmutableBitSet(new BitSet(), size); } From b33e5df587f32139449939f7c189ab913596ecc4 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 18 Jan 2024 15:31:40 -0500 Subject: [PATCH 38/55] Doing an extra operation to avoid doing an extra allocation. --- .../main/java/org/enso/table/util/ImmutableBitSet.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java index a752e55f148c..c5efa83451c3 100644 --- a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java +++ b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java @@ -63,11 +63,13 @@ public ImmutableBitSet notAndNot(ImmutableBitSet other) { } public ImmutableBitSet orNot(ImmutableBitSet other) { + // Doing an extra operation to avoid doing an extra allocation. + // a || !b => !(!a && b) assert size == other.size; BitSet result = (BitSet) bitSet.clone(); - BitSet otherNegated = (BitSet) other.bitSet.clone(); - otherNegated.flip(0, size); - result.or(otherNegated); + result.flip(0, size); + result.and(other.bitSet); + result.flip(0, size); return new ImmutableBitSet(result, size); } From dbf0df7c84c2b3d2724bdb2ddc64444f4476822b Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 18 Jan 2024 15:34:26 -0500 Subject: [PATCH 39/55] update comment --- .../operation/map/bool/BooleanIsInOp.java | 142 +----------------- 1 file changed, 1 insertion(+), 141 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index cd61d9a0835e..a7ec1638e62a 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -12,7 +12,7 @@ /** * A specialized implementation for the IS_IN operation on booleans - since booleans have just three * possible values we can have a highly efficient implementation that does not even rely on hashmap - * and after processing the input vector, performs the checks in constant time. + * and after processing the input vector, performs the checks using only BitSet builtins. */ public class BooleanIsInOp extends BinaryMapOperation { public BooleanIsInOp() { @@ -84,145 +84,5 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b } return new BoolStorage(newValues.toBitSet(), newMissing.toBitSet(), size, false); - - /* - BitSet values = storage.getValues(); - BitSet missing = storage.getIsMissing(); - boolean negated = storage.isNegated(); - - BitSet newValues; - BitSet newMissing; - - if (hadNull) { - if (hadTrue) { - if (hadFalse) { - // hadNull=t hadTrue=t hadFalse=t - } else { - // hadNull=t hadTrue=t hadFalse=f - if (negated) { - } else { - } - } - } else { - if (hadFalse) { - // hadNull=t hadTrue=f hadFalse=t - if (negated) { - } else { - } - } else { - // hadNull=t hadTrue=f hadFalse=f - } - } - } else { - if (hadTrue) { - if (hadFalse) { - // hadNull=f hadTrue=t hadFalse=t - } else { - // hadNull=f hadTrue=t hadFalse=f - if (negated) { - } else { - } - } - } else { - if (hadFalse) { - // hadNull=f hadTrue=f hadFalse=t - newValues = (BitSet) missing.clone(); - newValues.flip(0, storage.size()); - if (negated) { - newValues.and(values); - } else { - newValues.andNot(values); - } - newMissing = missing; - } else { - // hadNull=f hadTrue=f hadFalse=f - // NULL.is_in([]) is false, not NULL - newValues = new BitSet(); - newMissing = new BitSet(); - } - } - } - - return new BoolStorage(newValues, newMissing, storage.size(), false); -*/ - - /* - BitSet newVals = new BitSet(storage.size()); - BitSet newMissing = new BitSet(storage.size()); - byte[] bits = storage.getValues().toByteArray(); - System.out.println("vals " + bits); - for (int i = 0; i < storage.size(); ++i) { - boolean bitUnNegated = storage.getValues().get(i); - boolean bit = storage.isNegated() ? !bitUnNegated : bitUnNegated; - System.out.println("bits " + bitUnNegated + " " + bit); - boolean inputValue = storage.getItem(i); - boolean inputIsNull = storage.isNa(i); - if (inputIsNull) { - newMissing.set(i, true); - } else if (inputValue) { - if (hadTrue) { - newVals.set(i, true); - } else { - if (hadNull) { - newMissing.set(i, true); - } else { - newVals.set(i, false); - } - } - } else { // inputValue==false - if (hadFalse) { - newVals.set(i, true); - } else { - if (hadNull) { - newMissing.set(i, true); - } else { - newVals.set(i, false); - } - } - } - } - - return new BoolStorage(newVals, newMissing, storage.size(), false); - */ - -/* - boolean negated = false; - - if (hadNull && hadTrue && hadFalse) { - // We use empty newVals which has everything set to false and negate it to make all of that - // set to true with zero cost. - newVals = new BitSet(); - negated = true; - } else if (!hadNull && !hadTrue && !hadFalse) { - // No values are present, so the result is to be false everywhere. - newVals = new BitSet(); - } else if (hadNull && !hadTrue && !hadFalse) { - // Only missing values are in the set, so we just return the missing indicator. - newVals = storage.getIsMissing(); - } else if (hadTrue && hadFalse) { // && !hadNull - // All non-missing values are in the set - so we just return the negated missing indicator. - newVals = storage.getIsMissing(); - negated = true; - } else { - // hadTrue != hadFalse - newVals = storage.getValues().get(0, storage.size()); - if (hadTrue) { - if (storage.isNegated()) { - newVals.flip(0, storage.size()); - } - } else { // hadFalse - if (!storage.isNegated()) { - newVals.flip(0, storage.size()); - } - } - newVals.andNot(storage.getIsMissing()); - - if (hadNull) { - newVals.or(storage.getIsMissing()); - } - } - - return new BoolStorage(newVals, new BitSet(), storage.size(), negated); - */ } } From a23b34166d2d8a0a04aa34e89c29f8ea4b1fe010 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 11:46:33 -0500 Subject: [PATCH 40/55] one type for table_builder_typed --- .../Common_Table_Operations/Nothing_Spec.enso | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 4cce8fea32d3..457832193927 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -18,10 +18,8 @@ type My_Type spec setup = prefix = setup.prefix+"asdfasdf" table_builder = setup.table_builder - table_builder_typed columns types = - Runtime.assert (columns.length == types.length) - cast_columns = columns.map_with_index i-> c-> - value_type = types.at i + table_builder_typed columns value_type = + cast_columns = columns.map c-> Column.from_vector (c.at 0) (c.at 1) . cast value_type setup.table_builder cast_columns @@ -55,7 +53,7 @@ spec setup = value = triple.at 0 value_type = triple.at 2 - t0 = table_builder_typed [["x", [value]], ["n", [Nothing]]] [value_type, value_type] + t0 = table_builder_typed [["x", [value]], ["n", [Nothing]]] value_type Test.specify "Correctly handle Nothing in: "+value_type.to_text+" "+comparison_name+" Nothing value" <| c0 = comparison (t0.at "x") Nothing @@ -74,7 +72,7 @@ spec setup = value = triple.at 0 value_type = triple.at 2 - t0 = table_builder_typed [["x", [value]], ["y", [value]], ["n", [Nothing]]] [value_type, value_type, value_type] + t0 = table_builder_typed [["x", [value]], ["y", [value]], ["n", [Nothing]]] value_type Test.specify "Correctly handle Nothing in: Nothing column between "+value_type.to_text+" and "+value_type.to_text <| c0 = t0.at "n" . between (t0.at "x") (t0.at "y") @@ -102,7 +100,7 @@ spec setup = value_type = triple.at 2 Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_nothing" <| - t0 = table_builder_typed [["x", [value]]] [value_type] + t0 = table_builder_typed [["x", [value]]] value_type c0 = t0.at "x" . is_nothing c0.to_vector . should_equal [value == Nothing] @@ -116,7 +114,7 @@ spec setup = value = triple.at 0 other_value = triple.at 1 value_type = triple.at 2 - t0 = table_builder_typed [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["z", [value, other_value]], ["n", [Nothing, Nothing]]] [value_type, value_type, value_type, value_type] + t0 = table_builder_typed [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["z", [value, other_value]], ["n", [Nothing, Nothing]]] value_type true_if_not_nothing = if value == Nothing then Nothing else True @@ -199,7 +197,7 @@ spec setup = Test.specify "Boolean is_in: (Vector), "+negation_desc+" "+cs.to_text <| input_column = neg_input [input] - t = table_builder_typed [["input", input_column]] [Value_Type.Boolean] + t = table_builder_typed [["input", input_column]] Value_Type.Boolean c = (neg_input (t.at "input")) . is_in containing_values @@ -207,7 +205,7 @@ spec setup = Test.specify "Boolean is_in: (Column), "+negation_desc+" "+cs.to_text <| input_column = neg_input (Vector.fill containing_values.length input) - t = table_builder_typed [["input", input_column], ["containing", neg_argument containing_values]] [Value_Type.Boolean, Value_Type.Boolean] + t = table_builder_typed [["input", input_column], ["containing", neg_argument containing_values]] Value_Type.Boolean expected_output = if input_column.is_empty then [] else [output] c = (neg_input (t.at "input")) . is_in (neg_argument (t.at "containing")) @@ -229,7 +227,7 @@ spec setup = v0 . should_contain_the_same_elements_as [value, other_value, Nothing] Test.specify "Correctly handle Nothing in .distinct for Nothing" <| - t0 = table_builder_typed [["x", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing]]] [Value_Type.Char] + t0 = table_builder_typed [["x", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing]]] Value_Type.Char t1 = t0 . distinct ["x"] v0 = t1.at "x" . to_vector v0 . should_equal [Nothing] @@ -243,7 +241,7 @@ spec setup = is_comparable = Default_Comparator.compare value other_value != Nothing if is_comparable then - t0 = table_builder_typed [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] [value_type] + t0 = table_builder_typed [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] value_type Test.specify "Correctly handle Nothing in .order_by (asc) for "+value_type.to_text <| t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Ascending] From aa193f7f517a8e46a6461fa26c29a7d80e853e26 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 11:49:18 -0500 Subject: [PATCH 41/55] rename vars --- .../Common_Table_Operations/Nothing_Spec.enso | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 457832193927..b3507eaade4f 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -53,46 +53,46 @@ spec setup = value = triple.at 0 value_type = triple.at 2 - t0 = table_builder_typed [["x", [value]], ["n", [Nothing]]] value_type + table = table_builder_typed [["x", [value]], ["n", [Nothing]]] value_type Test.specify "Correctly handle Nothing in: "+value_type.to_text+" "+comparison_name+" Nothing value" <| - c0 = comparison (t0.at "x") Nothing - c0.to_vector . should_equal [Nothing] + co = comparison (table.at "x") Nothing + co.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+" "+comparison_name+" Nothing column" <| - c0 = comparison (t0.at "x") (t0.at "n") - c0.to_vector . should_equal [Nothing] + co = comparison (table.at "x") (table.at "n") + co.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: Nothing column "+comparison_name+" "+value_type.to_text <| - c0 = comparison (t0.at "n") (t0.at "x") - c0.to_vector . should_equal [Nothing] + co = comparison (table.at "n") (table.at "x") + co.to_vector . should_equal [Nothing] Test.group prefix+"between" <| values_with_nothing.map triple-> value = triple.at 0 value_type = triple.at 2 - t0 = table_builder_typed [["x", [value]], ["y", [value]], ["n", [Nothing]]] value_type + table = table_builder_typed [["x", [value]], ["y", [value]], ["n", [Nothing]]] value_type Test.specify "Correctly handle Nothing in: Nothing column between "+value_type.to_text+" and "+value_type.to_text <| - c0 = t0.at "n" . between (t0.at "x") (t0.at "y") - c0.to_vector . should_equal [Nothing] + co = table.at "n" . between (table.at "x") (table.at "y") + co.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+" between Nothing column and "+value_type.to_text <| - c0 = t0.at "x" . between (t0.at "n") (t0.at "y") - c0.to_vector . should_equal [Nothing] + co = table.at "x" . between (table.at "n") (table.at "y") + co.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+" between "+value_type.to_text+" and Nothing column" <| - c0 = t0.at "x" . between (t0.at "y") (t0.at "n") - c0.to_vector . should_equal [Nothing] + co = table.at "x" . between (table.at "y") (table.at "n") + co.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+" between Nothing value and "+value_type.to_text <| - c0 = t0.at "x" . between Nothing (t0.at "y") - c0.to_vector . should_equal [Nothing] + co = table.at "x" . between Nothing (table.at "y") + co.to_vector . should_equal [Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+" between "+value_type.to_text+" and Nothing value" <| - c0 = t0.at "x" . between (t0.at "y") Nothing - c0.to_vector . should_equal [Nothing] + co = table.at "x" . between (table.at "y") Nothing + co.to_vector . should_equal [Nothing] Test.group prefix+"is_nothing" <| values_with_nothing.map triple-> @@ -100,47 +100,47 @@ spec setup = value_type = triple.at 2 Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_nothing" <| - t0 = table_builder_typed [["x", [value]]] value_type - c0 = t0.at "x" . is_nothing - c0.to_vector . should_equal [value == Nothing] + table = table_builder_typed [["x", [value]]] value_type + co = table.at "x" . is_nothing + co.to_vector . should_equal [value == Nothing] Test.group prefix+"not" <| Test.specify "Correctly handle Nothing in .not" <| - t0 = table_builder [["x", [True, False, Nothing]]] - t0.at "x" . not . to_vector . should_equal [False, True, Nothing] + table = table_builder [["x", [True, False, Nothing]]] + table.at "x" . not . to_vector . should_equal [False, True, Nothing] Test.group prefix+"is_in" <| values_with_nothing.map triple-> value = triple.at 0 other_value = triple.at 1 value_type = triple.at 2 - t0 = table_builder_typed [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["z", [value, other_value]], ["n", [Nothing, Nothing]]] value_type + table = table_builder_typed [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["z", [value, other_value]], ["n", [Nothing, Nothing]]] value_type true_if_not_nothing = if value == Nothing then Nothing else True Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column (returning True)" <| - t0.at "x" . is_in (t0.at "z") . to_vector . should_equal [true_if_not_nothing, Nothing] + table.at "x" . is_in (table.at "z") . to_vector . should_equal [true_if_not_nothing, Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column with Nothings (returning True)" <| - t0.at "x" . is_in (t0.at "x") . to_vector . should_equal [true_if_not_nothing, Nothing] + table.at "x" . is_in (table.at "x") . to_vector . should_equal [true_if_not_nothing, Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Column with Nothings (returning Nothing)" <| - t0.at "x" . is_in (t0.at "y") . to_vector . should_equal [Nothing, Nothing] + table.at "x" . is_in (table.at "y") . to_vector . should_equal [Nothing, Nothing] Test.specify "Correctly handle Nothing in: Nothing.is_in Column with Nothings (returning Nothing)" <| - t0.at "n" . is_in (t0.at "x") . to_vector . should_equal [Nothing, Nothing] + table.at "n" . is_in (table.at "x") . to_vector . should_equal [Nothing, Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector (returning True)" <| - t0.at "x" . is_in (t0.at "z" . to_vector) . to_vector . should_equal [true_if_not_nothing, Nothing] + table.at "x" . is_in (table.at "z" . to_vector) . to_vector . should_equal [true_if_not_nothing, Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector with Nothings (returning True)" <| - t0.at "x" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [true_if_not_nothing, Nothing] + table.at "x" . is_in (table.at "x" . to_vector) . to_vector . should_equal [true_if_not_nothing, Nothing] Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in Vector with Nothings (returning Nothing)" <| - t0.at "x" . is_in (t0.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] + table.at "x" . is_in (table.at "y" . to_vector) . to_vector . should_equal [Nothing, Nothing] Test.specify "Correctly handle Nothing in: Nothing.is_in Vector with Nothings (returning Nothing)" <| - t0.at "n" . is_in (t0.at "x" . to_vector) . to_vector . should_equal [Nothing, Nothing] + table.at "n" . is_in (table.at "x" . to_vector) . to_vector . should_equal [Nothing, Nothing] Test.group prefix+"Boolean is_in" <| make_containing_values had_null had_true had_false = @@ -220,17 +220,17 @@ spec setup = value_type = triple.at 2 Test.specify "Correctly handle Nothing in .distinct for "+value_type.to_text <| - t0 = table_builder [["x", [value, other_value, other_value, Nothing, value, Nothing]]] - t1 = t0 . distinct ["x"] - v0 = t1.at "x" . to_vector - v0 . length . should_equal 3 - v0 . should_contain_the_same_elements_as [value, other_value, Nothing] + table = table_builder [["x", [value, other_value, other_value, Nothing, value, Nothing]]] + t1 = table . distinct ["x"] + v = t1.at "x" . to_vector + v . length . should_equal 3 + v . should_contain_the_same_elements_as [value, other_value, Nothing] Test.specify "Correctly handle Nothing in .distinct for Nothing" <| - t0 = table_builder_typed [["x", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing]]] Value_Type.Char - t1 = t0 . distinct ["x"] - v0 = t1.at "x" . to_vector - v0 . should_equal [Nothing] + table = table_builder_typed [["x", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing]]] Value_Type.Char + t1 = table . distinct ["x"] + v = t1.at "x" . to_vector + v . should_equal [Nothing] Test.group prefix+"order_by" <| values_with_nothing.map triple-> @@ -241,12 +241,12 @@ spec setup = is_comparable = Default_Comparator.compare value other_value != Nothing if is_comparable then - t0 = table_builder_typed [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] value_type + table = table_builder_typed [["x", [value, Nothing, other_value, other_value, Nothing, value, Nothing]]] value_type Test.specify "Correctly handle Nothing in .order_by (asc) for "+value_type.to_text <| - t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Ascending] + t1 = table . order_by [Sort_Column.Name "x" Sort_Direction.Ascending] t1.at "x" . to_vector . should_equal [Nothing, Nothing, Nothing, value, value, other_value, other_value] Test.specify "Correctly handle Nothing in .order_by (desc) for "+value_type.to_text <| - t1 = t0 . order_by [Sort_Column.Name "x" Sort_Direction.Descending] + t1 = table . order_by [Sort_Column.Name "x" Sort_Direction.Descending] t1.at "x" . to_vector . should_equal [other_value, other_value, value, value, Nothing, Nothing, Nothing] From af922afeba3f9bf9a80d19584c86e76cecd9be32 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 11:50:12 -0500 Subject: [PATCH 42/55] cleanup --- .../lib/Standard/Table/0.0.0-dev/src/Data/Column.enso | 5 ----- .../src/Common_Table_Operations/Nothing_Spec.enso | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso index 4be3cc7233fe..ae396ef9490d 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso @@ -212,11 +212,6 @@ type Column == self other = new_name = naming_helper.binary_operation_name "==" self other fallback problem_builder a b = - IO.println 'AAA == fallback' - IO.println self.value_type - IO.println other - if other.is_a Column then - IO.println other.value_type if (a.is_a Float) || (b.is_a Float) then problem_builder.reportFloatingPointEquality -1 a == b diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index b3507eaade4f..9934db591d24 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -16,7 +16,7 @@ type My_Type spec setup = - prefix = setup.prefix+"asdfasdf" + prefix = setup.prefix table_builder = setup.table_builder table_builder_typed columns value_type = cast_columns = columns.map c-> From 1e8f1d9612c2b568229eefb2f272db742b2ef6ed Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 11:56:37 -0500 Subject: [PATCH 43/55] comments --- .../src/main/java/org/enso/table/util/ImmutableBitSet.java | 2 +- .../src/Common_Table_Operations/Nothing_Spec.enso | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java index c5efa83451c3..3a4e298183c5 100644 --- a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java +++ b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java @@ -80,4 +80,4 @@ public static ImmutableBitSet allFalse(int size) { public static ImmutableBitSet allTrue(int size) { return new ImmutableBitSet(new BitSet(), size).not(); } -} \ No newline at end of file +} diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 9934db591d24..99f33c26b539 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -175,11 +175,12 @@ spec setup = + [[Nothing, False, True, False, Nothing]] + [[Nothing, False, False, False, False]] + # To test negated columns, we invert the values before putting them into the Storage, and set the negated bit. negate : Vector | Column -> Vector | Column negate vorc = case vorc of - _ : Vector -> vorc.map (x-> x.if_not_nothing x.not) - _ : Column -> vorc.not - _ : Database_Column -> vorc.not + _ : Vector -> vorc.map (x-> x.if_not_nothing x.not) # Invert values first + _ : Column -> vorc.not # Set negated bit + _ : Database_Column -> vorc.not # Set negated bit negation_cases = [["normal input, normal argument", identity, identity]] + [["negated input, normal argument", negate, identity]] + [["normal input, negated argument", identity, negate]] From a1c481a7551126621775cfeb5309522d9fd9d609 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 14:45:18 -0500 Subject: [PATCH 44/55] Fix Table_Spec. Fix `is_in` test cases. Added Set.contains_relational. --- .../Standard/Base/0.0.0-dev/src/Data/Set.enso | 21 ++++++++++++ .../Table/0.0.0-dev/src/Data/Column.enso | 4 +-- test/Base_Tests/src/Data/Set_Spec.enso | 10 ++++++ .../Table_Tests/src/In_Memory/Table_Spec.enso | 33 ++++++++++--------- 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Set.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Set.enso index 20ae8472ea74..e789c605611f 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Set.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Set.enso @@ -66,6 +66,27 @@ type Set contains : Any -> Boolean contains self value = self.underlying_map.contains_key value + ## GROUP Logical + Checks if this set contains a given value, treating Nothing as a + relational NULL. + + If the argument is non-Nothing and exists in `value`, return true. + + If the argument is non-Nothing and does not exist in `value`, return + false if `value` does not contain a Nothing, or Nothing if `value` does + contain a Nothing. + + If the argument is Nothing, return Nothing if `value` is non-empty, or + false if `value` is empty. + contains_relational : Any -> Boolean | Nothing + contains_relational self value = + case value of + _ : Nothing -> if self.is_empty then False else Nothing + _ -> + if self.contains value then True else + has_nulls = self.contains Nothing + if has_nulls then Nothing else False + ## ALIAS add GROUP Calculations Adds a value to this set. diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso index ae396ef9490d..d2596f446a02 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso @@ -1600,9 +1600,7 @@ type Column run_vectorized_binary_op self op_name as_vector expected_result_type=Value_Type.Boolean skip_nulls=False new_name=result_name False -> set = Set.from_vector as_vector error_on_duplicates=False - set_has_nulls = set.contains Nothing - contains x = if set.contains x then True else (if set_has_nulls then Nothing else False) - run_unary_op self contains new_name=result_name skip_nulls=True expected_result_type=Value_Type.Boolean + run_unary_op self set.contains_relational new_name=result_name skip_nulls=True expected_result_type=Value_Type.Boolean ## GROUP Standard.Base.Conversions ICON convert diff --git a/test/Base_Tests/src/Data/Set_Spec.enso b/test/Base_Tests/src/Data/Set_Spec.enso index 13deef702dad..9f47fb97491b 100644 --- a/test/Base_Tests/src/Data/Set_Spec.enso +++ b/test/Base_Tests/src/Data/Set_Spec.enso @@ -30,6 +30,16 @@ spec = s1.contains 3 . should_be_true s1.contains 4 . should_be_false + Test.specify "should allow checking contains with relational NULL logic" <| + Set.from_vector [1, 2] . contains_relational 1 . should_be_true + Set.from_vector [1, 2] . contains_relational 3 . should_be_false + Set.from_vector [1, 2, Nothing] . contains_relational 1 . should_be_true + Set.from_vector [1, 2, Nothing] . contains_relational 3 . should_equal Nothing + Set.from_vector [1, 2, Nothing] . contains_relational Nothing . should_equal Nothing + Set.from_vector [1, 2] . contains_relational Nothing . should_equal Nothing + Set.from_vector [Nothing] . contains_relational Nothing . should_equal Nothing + Set.from_vector [] . contains_relational Nothing . should_be_false + Test.specify "should allow to compute a union, intersection and difference" <| s1 = Set.from_vector [1, 2] s2 = Set.from_vector [2, 3] diff --git a/test/Table_Tests/src/In_Memory/Table_Spec.enso b/test/Table_Tests/src/In_Memory/Table_Spec.enso index c9c4f7ba08d8..4ea6fa2498de 100644 --- a/test/Table_Tests/src/In_Memory/Table_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Table_Spec.enso @@ -809,29 +809,29 @@ spec = nulls = ["nulls", [Nothing, Nothing, Nothing, 0]] custom = ["custom", [2, My.Data 2 1, Nothing, Nothing]] [str, int, int2, dbl, dates, dts, tod, mix, nulls, custom] - varied_type_table.filter "strs" (Filter_Condition.Is_In (ins.at "str")) . at "strs" . to_vector . should_equal ["b", "c", Nothing] - varied_type_table.filter "strs" (Filter_Condition.Is_In (ins.at "str" . to_vector)) . at "strs" . to_vector . should_equal ["b", "c", Nothing] + varied_type_table.filter "strs" (Filter_Condition.Is_In (ins.at "str")) . at "strs" . to_vector . should_equal ["b", "c"] + varied_type_table.filter "strs" (Filter_Condition.Is_In (ins.at "str" . to_vector)) . at "strs" . to_vector . should_equal ["b", "c"] varied_type_table.filter "ints" (Filter_Condition.Is_In (ins.at "int")) . at "ints" . to_vector . should_equal [1, 2] varied_type_table.filter "ints" (Filter_Condition.Is_In (ins.at "int" . to_vector)) . at "ints" . to_vector . should_equal [1, 2] - varied_type_table.filter "ints" (Filter_Condition.Is_In (ins.at "int2")) . at "ints" . to_vector . should_equal [Nothing, 1] - varied_type_table.filter "ints" (Filter_Condition.Is_In (ins.at "int2" . to_vector)) . at "ints" . to_vector . should_equal [Nothing, 1] - varied_type_table.filter "doubles" (Filter_Condition.Is_In (ins.at "dbl")) . at "doubles" . to_vector . should_equal [0.0, Nothing] - varied_type_table.filter "doubles" (Filter_Condition.Is_In (ins.at "dbl" . to_vector)) . at "doubles" . to_vector . should_equal [0.0, Nothing] + varied_type_table.filter "ints" (Filter_Condition.Is_In (ins.at "int2")) . at "ints" . to_vector . should_equal [1] + varied_type_table.filter "ints" (Filter_Condition.Is_In (ins.at "int2" . to_vector)) . at "ints" . to_vector . should_equal [1] + varied_type_table.filter "doubles" (Filter_Condition.Is_In (ins.at "dbl")) . at "doubles" . to_vector . should_equal [0.0] + varied_type_table.filter "doubles" (Filter_Condition.Is_In (ins.at "dbl" . to_vector)) . at "doubles" . to_vector . should_equal [0.0] varied_type_table.filter "dates" (Filter_Condition.Is_In (ins.at "dates")) . at "dates" . to_vector . should_equal [Date.new 2000, Date.new 1999 1 1] varied_type_table.filter "dates" (Filter_Condition.Is_In (ins.at "dates" . to_vector)) . at "dates" . to_vector . should_equal [Date.new 2000, Date.new 1999 1 1] - varied_type_table.filter "datetimes" (Filter_Condition.Is_In (ins.at "dts")) . at "datetimes" . to_vector . should_equal [Nothing, Date_Time.new 2022 8 27 11 22 25] - varied_type_table.filter "datetimes" (Filter_Condition.Is_In (ins.at "dts" . to_vector)) . at "datetimes" . to_vector . should_equal [Nothing, Date_Time.new 2022 8 27 11 22 25] + varied_type_table.filter "datetimes" (Filter_Condition.Is_In (ins.at "dts")) . at "datetimes" . to_vector . should_equal [Date_Time.new 2022 8 27 11 22 25] + varied_type_table.filter "datetimes" (Filter_Condition.Is_In (ins.at "dts" . to_vector)) . at "datetimes" . to_vector . should_equal [Date_Time.new 2022 8 27 11 22 25] varied_type_table.filter "times" (Filter_Condition.Is_In (ins.at "tod")) . at "times" . to_vector . should_equal [Time_Of_Day.new 18 00] varied_type_table.filter "times" (Filter_Condition.Is_In (ins.at "tod" . to_vector)) . at "times" . to_vector . should_equal [Time_Of_Day.new 18 00] - varied_type_table.filter "mixed" (Filter_Condition.Is_In [42, "a", 1, Nothing, Date.new 2022 8 27, Date_Time.new 2022 8 27]) . at "mixed" . to_vector . should_equal [1, "a", Nothing, Date.new 2022 8 27] + varied_type_table.filter "mixed" (Filter_Condition.Is_In [42, "a", 1, Nothing, Date.new 2022 8 27, Date_Time.new 2022 8 27]) . at "mixed" . to_vector . should_equal [1, "a", Date.new 2022 8 27] varied_type_table.filter "mixed" (Filter_Condition.Is_In (ins.at "mix")) . at "mixed" . to_vector . should_equal [1] varied_type_table.filter "mixed" (Filter_Condition.Is_In (ins.at "mix" . to_vector)) . at "mixed" . to_vector . should_equal [1] varied_type_table.filter "just_nulls" (Filter_Condition.Is_In []) . at "just_nulls" . to_vector . should_equal [] - varied_type_table.filter "just_nulls" (Filter_Condition.Is_In (ins.at "nulls")) . at "just_nulls" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing] - varied_type_table.filter "just_nulls" (Filter_Condition.Is_In (ins.at "nulls" . to_vector)) . at "just_nulls" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing] + varied_type_table.filter "just_nulls" (Filter_Condition.Is_In (ins.at "nulls")) . at "just_nulls" . to_vector . should_equal [] + varied_type_table.filter "just_nulls" (Filter_Condition.Is_In (ins.at "nulls" . to_vector)) . at "just_nulls" . to_vector . should_equal [] varied_type_table.filter "just_nulls" (Filter_Condition.Is_In [0]) . at "just_nulls" . to_vector . should_equal [] - varied_type_table.filter "custom_objects" (Filter_Condition.Is_In (ins.at "custom")) . at "custom_objects" . to_vector . should_equal [My.Data 1 2, Nothing, Nothing] - varied_type_table.filter "custom_objects" (Filter_Condition.Is_In (ins.at "custom" . to_vector)) . at "custom_objects" . to_vector . should_equal [My.Data 1 2, Nothing, Nothing] + varied_type_table.filter "custom_objects" (Filter_Condition.Is_In (ins.at "custom")) . at "custom_objects" . to_vector . should_equal [My.Data 1 2] + varied_type_table.filter "custom_objects" (Filter_Condition.Is_In (ins.at "custom" . to_vector)) . at "custom_objects" . to_vector . should_equal [My.Data 1 2] t2 = Table.new [["ints", [1, 2, 3]], ["doubles", [1.2, 0.0, 1.0]]] t2.filter "ints" (Filter_Condition.Is_In [2.0, 1.5, 3, 4]) . at "ints" . to_vector . should_equal [2, 3] @@ -849,6 +849,7 @@ spec = if has_true then vec_builder.append True if has_false then vec_builder.append False in_vector = vec_builder.to_vector + in_vector_set = Set.from_vector in_vector vectors = [[True, False, Nothing], [Nothing, Nothing, Nothing], [False, False, True], [True, True, True], [False, False, False], [Nothing, True, True], [False, Nothing, False]] vectors.each column_vector-> @@ -863,10 +864,10 @@ spec = t = t0.drop 1 in_column = Column.from_vector "in" in_vector - expected_vector = column_vector.filter (Filter_Condition.Is_In in_vector) - expected_neg_vector = negated_column_vector.filter (Filter_Condition.Is_In in_vector) + expected_vector = column_vector.filter (x-> in_vector_set.contains_relational x == True) + expected_neg_vector = negated_column_vector.filter (x-> in_vector_set.contains_relational x == True) - Test.with_clue "(Is_In "+in_vector.to_text+"): " <| + Test.with_clue "("+column_vector.to_text+" Is_In "+in_vector.to_text+"): " <| t.filter "X" (Filter_Condition.Is_In in_vector) . at "X" . to_vector . should_equal expected_vector t.filter "X" (Filter_Condition.Is_In in_column) . at "X" . to_vector . should_equal expected_vector t2 = t.set (t.at "X" . not) new_name="Y" From cfce03142544731eb50961df046f9c25e654c02f Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 15:09:13 -0500 Subject: [PATCH 45/55] fix Filter_Spec --- .../Common_Table_Operations/Filter_Spec.enso | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Filter_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Filter_Spec.enso index b208edc20f3f..f121d942d7f1 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Filter_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Filter_Spec.enso @@ -268,11 +268,14 @@ spec setup = Test.specify "by an Is_In check" <| t = table_builder [["ix", [1, 2, 3, Nothing, 5, 6]], ["X", ["a", "b", "ccc", "X", "f", "2"]]] t1 = table_builder [["txt", ["X", "a", "c", Nothing]], ["int", [Nothing, 2, 5, 4]], ["bool", [True, Nothing, Nothing, True]]] + t2 = table_builder [["txt", ["X", "a", "c", "q"]], ["int", [123, 2, 5, 4]], ["bool", [True, True, True, True]]] t.filter "X" (Filter_Condition.Is_In (t1.at "txt")) . at "X" . to_vector . should_equal ["a", "X"] t.filter "X" (Filter_Condition.Is_In (t1.at "txt" . to_vector)) . at "X" . to_vector . should_equal ["a", "X"] - t.filter "X" (Filter_Condition.Not_In (t1.at "txt")) . at "X" . to_vector . should_equal ["b", "ccc", "f", "2"] - t.filter "X" (Filter_Condition.Not_In (t1.at "txt" . to_vector)) . at "X" . to_vector . should_equal ["b", "ccc", "f", "2"] + t.filter "X" (Filter_Condition.Not_In (t1.at "txt")) . at "X" . to_vector . should_equal [] + t.filter "X" (Filter_Condition.Not_In (t2.at "txt")) . at "X" . to_vector . should_equal ["b", "ccc", "f", "2"] + t.filter "X" (Filter_Condition.Not_In (t1.at "txt" . to_vector)) . at "X" . to_vector . should_equal [] + t.filter "X" (Filter_Condition.Not_In (t2.at "txt" . to_vector)) . at "X" . to_vector . should_equal ["b", "ccc", "f", "2"] t.filter "X" (Filter_Condition.Is_In ["ccc"]) . at "X" . to_vector . should_equal ["ccc"] t.filter "X" (Filter_Condition.Is_In []) . at "X" . to_vector . should_equal [] t.filter "X" (Filter_Condition.Not_In []) . at "X" . to_vector . should_equal ["a", "b", "ccc", "X", "f", "2"] @@ -282,12 +285,12 @@ spec setup = # Some backends (e.g. SQLite) allow to coerce integer and text types when doing mixed type comparisons. ((mixed == []) || (mixed == ["2"])).should_be_true - t.filter "ix" (Filter_Condition.Is_In (t1.at "int")) . at "ix" . to_vector . should_equal [2, Nothing, 5] - t.filter "ix" (Filter_Condition.Is_In (t1.at "int" . to_vector)) . at "ix" . to_vector . should_equal [2, Nothing, 5] + t.filter "ix" (Filter_Condition.Is_In (t1.at "int")) . at "ix" . to_vector . should_equal [2, 5] + t.filter "ix" (Filter_Condition.Is_In (t1.at "int" . to_vector)) . at "ix" . to_vector . should_equal [2, 5] t.filter "ix" (Filter_Condition.Is_In [2, 5, 4]) . at "ix" . to_vector . should_equal [2, 5] - t.filter "ix" (Filter_Condition.Is_In [Nothing]) . at "ix" . to_vector . should_equal [Nothing] - t.filter "ix" (Filter_Condition.Not_In [Nothing]) . at "ix" . to_vector . should_equal [1, 2, 3, 5, 6] - t.filter "ix" (Filter_Condition.Not_In [1, 3]) . at "ix" . to_vector . should_equal [2, Nothing, 5, 6] + t.filter "ix" (Filter_Condition.Is_In [Nothing]) . at "ix" . to_vector . should_equal [] + t.filter "ix" (Filter_Condition.Not_In [Nothing]) . at "ix" . to_vector . should_equal [] + t.filter "ix" (Filter_Condition.Not_In [1, 3]) . at "ix" . to_vector . should_equal [2, 5, 6] v1 = t.filter "X" (Filter_Condition.Is_In ["c", "f", "b", "b", "b", 15, Nothing]) . at "X" . to_vector case test_selection.allows_mixed_type_comparisons of @@ -298,14 +301,14 @@ spec setup = True -> v2.should_equal [2, 3] False -> v2.should_fail_with SQL_Error - t2 = table_builder [["A", [True, False, True]], ["B", [False, False, False]], ["C", [True, False, Nothing]]] - t2.filter "A" (Filter_Condition.Is_In (t1.at "bool")) . at "A" . to_vector . should_equal [True, True] - t2.filter "A" (Filter_Condition.Is_In (t1.at "bool" . to_vector)) . at "A" . to_vector . should_equal [True, True] - t2.filter "B" (Filter_Condition.Is_In [True, Nothing]) . at "B" . to_vector . should_equal [] - t2.filter "C" (Filter_Condition.Is_In [True, Nothing]) . at "C" . to_vector . should_equal [True, Nothing] - t2.filter "A" (Filter_Condition.Is_In [False]) . at "A" . to_vector . should_equal [False] - t2.filter "B" (Filter_Condition.Is_In [False]) . at "B" . to_vector . should_equal [False, False, False] - t2.filter "C" (Filter_Condition.Is_In [False, False]) . at "C" . to_vector . should_equal [False] + t3 = table_builder [["A", [True, False, True]], ["B", [False, False, False]], ["C", [True, False, Nothing]]] + t3.filter "A" (Filter_Condition.Is_In (t1.at "bool")) . at "A" . to_vector . should_equal [True, True] + t3.filter "A" (Filter_Condition.Is_In (t1.at "bool" . to_vector)) . at "A" . to_vector . should_equal [True, True] + t3.filter "B" (Filter_Condition.Is_In [True, Nothing]) . at "B" . to_vector . should_equal [] + t3.filter "C" (Filter_Condition.Is_In [True, Nothing]) . at "C" . to_vector . should_equal [True] + t3.filter "A" (Filter_Condition.Is_In [False]) . at "A" . to_vector . should_equal [False] + t3.filter "B" (Filter_Condition.Is_In [False]) . at "B" . to_vector . should_equal [False, False, False] + t3.filter "C" (Filter_Condition.Is_In [False, False]) . at "C" . to_vector . should_equal [False] Test.specify "does not allow Column_Ref in Is_In/Not_In because that would be confusing" <| ## Is In and Not In check if a value is contained anywhere in a provided collection (e.g. column), From 305c018ae9a23fbb3baa9aa96360fcf9a12229dd Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 15:14:19 -0500 Subject: [PATCH 46/55] fix integer_overflow_spec --- test/Table_Tests/src/In_Memory/Integer_Overflow_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Table_Tests/src/In_Memory/Integer_Overflow_Spec.enso b/test/Table_Tests/src/In_Memory/Integer_Overflow_Spec.enso index 66bc81de4802..b55491eae70b 100644 --- a/test/Table_Tests/src/In_Memory/Integer_Overflow_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Integer_Overflow_Spec.enso @@ -322,7 +322,7 @@ spec = x.is_nothing . to_vector . should_equal [False, False, True, False] x.is_nan . to_vector . should_equal [False, False, Nothing, False] x.is_infinite . to_vector . should_equal [False, False, Nothing, False] - x.is_in [3, 2^70] . to_vector . should_equal [False, True, False, True] + x.is_in [3, 2^70] . to_vector . should_equal [False, True, Nothing, True] Test.specify "allows arithmetic on Decimal columns and other numeric columns" <| t = Table.new [["X", [10^30, 2^70, Nothing, 3]], ["Y", [1, 2, 3, 4]], ["Z", [1.5, 2.5, 3.5, 4.5]]] From fd40de352fab6c09ab008f31b58caf104eda5fc0 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 15:16:52 -0500 Subject: [PATCH 47/55] fix Codegen_Spec --- test/Table_Tests/src/Database/Codegen_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Table_Tests/src/Database/Codegen_Spec.enso b/test/Table_Tests/src/Database/Codegen_Spec.enso index 4c9123d8fe87..05deb9263e0f 100644 --- a/test/Table_Tests/src/Database/Codegen_Spec.enso +++ b/test/Table_Tests/src/Database/Codegen_Spec.enso @@ -64,7 +64,7 @@ spec = Test.specify "should generate an IN expression" <| t2 = t1.filter "A" (Filter_Condition.Is_In [1, 2, 'foo']) - t2.to_sql.prepare . should_equal ['SELECT "T1"."A" AS "A", "T1"."B" AS "B", "T1"."C" AS "C" FROM "T1" AS "T1" WHERE COALESCE("T1"."A" IN (?, ?, ?), FALSE)', [1, 2, "foo"]] + t2.to_sql.prepare . should_equal ['SELECT "T1"."A" AS "A", "T1"."B" AS "B", "T1"."C" AS "C" FROM "T1" AS "T1" WHERE "T1"."A" IN (?, ?, ?)', [1, 2, "foo"]] t3 = t1.filter "A" (Filter_Condition.Is_In [1]) t3.to_sql.prepare . should_equal ['SELECT "T1"."A" AS "A", "T1"."B" AS "B", "T1"."C" AS "C" FROM "T1" AS "T1" WHERE COALESCE("T1"."A" IN (?), FALSE)', [1]] From 564587dcca9665baecbcb2a18c2dd63d32336534 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 15:39:31 -0500 Subject: [PATCH 48/55] fix Codegen_Spec --- test/Table_Tests/src/Database/Codegen_Spec.enso | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Table_Tests/src/Database/Codegen_Spec.enso b/test/Table_Tests/src/Database/Codegen_Spec.enso index 05deb9263e0f..15d77d450fee 100644 --- a/test/Table_Tests/src/Database/Codegen_Spec.enso +++ b/test/Table_Tests/src/Database/Codegen_Spec.enso @@ -67,13 +67,13 @@ spec = t2.to_sql.prepare . should_equal ['SELECT "T1"."A" AS "A", "T1"."B" AS "B", "T1"."C" AS "C" FROM "T1" AS "T1" WHERE "T1"."A" IN (?, ?, ?)', [1, 2, "foo"]] t3 = t1.filter "A" (Filter_Condition.Is_In [1]) - t3.to_sql.prepare . should_equal ['SELECT "T1"."A" AS "A", "T1"."B" AS "B", "T1"."C" AS "C" FROM "T1" AS "T1" WHERE COALESCE("T1"."A" IN (?), FALSE)', [1]] + t3.to_sql.prepare . should_equal ['SELECT "T1"."A" AS "A", "T1"."B" AS "B", "T1"."C" AS "C" FROM "T1" AS "T1" WHERE "T1"."A" IN (?)', [1]] t4 = t1.filter "A" (Filter_Condition.Is_In []) t4.to_sql.prepare . should_equal ['SELECT "T1"."A" AS "A", "T1"."B" AS "B", "T1"."C" AS "C" FROM "T1" AS "T1" WHERE (FALSE)', []] t5 = t1.filter "A" (Filter_Condition.Is_In [Nothing]) - t5.to_sql.prepare . should_equal ['SELECT "T1"."A" AS "A", "T1"."B" AS "B", "T1"."C" AS "C" FROM "T1" AS "T1" WHERE ((FALSE) OR ("T1"."A" IS NULL))', []] + t5.to_sql.prepare . should_equal ['SELECT "T1"."A" AS "A", "T1"."B" AS "B", "T1"."C" AS "C" FROM "T1" AS "T1" WHERE "T1"."A" IN (?)', [Nothing]] Test.group "[Codegen] Handling Missing Values" <| Test.specify "fill_nothing should allow to replace missing values in a column with a constant" <| From 391a421284d5a005030f18c56cbe1ec91295c312 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 15:49:58 -0500 Subject: [PATCH 49/55] java fmt --- .../operation/map/SpecializedIsInOp.java | 2 +- .../operation/map/bool/BooleanIsInOp.java | 7 +- .../org/enso/table/util/ImmutableBitSet.java | 130 +++++++++--------- 3 files changed, 70 insertions(+), 69 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java index 16f2e2674ab3..9c612c3e71f9 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java @@ -58,7 +58,7 @@ public Storage runMap(S storage, List arg) { BitSet missing = new BitSet(); for (int i = 0; i < storage.size(); i++) { if (storage.isNa(i)) { - missing.set(i); + missing.set(i); } else if (compactRepresentation.coercedValues.contains(storage.getItemBoxed(i))) { newVals.set(i); } else if (compactRepresentation.hasNulls) { diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java index a7ec1638e62a..b19f687e2c8a 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/bool/BooleanIsInOp.java @@ -1,6 +1,5 @@ package org.enso.table.data.column.operation.map.bool; -import java.util.BitSet; import java.util.List; import org.enso.table.data.column.operation.map.BinaryMapOperation; import org.enso.table.data.column.operation.map.MapOperationProblemAggregator; @@ -71,10 +70,12 @@ private BoolStorage run(BoolStorage storage, boolean hadNull, boolean hadTrue, b if (hadTrue && !hadFalse) { newValues = storage.isNegated() ? missing.notAndNot(values) : missing.notAnd(values); - newMissing = hadNull ? (storage.isNegated() ? missing.or(values) : missing.orNot(values)) : missing; + newMissing = + hadNull ? (storage.isNegated() ? missing.or(values) : missing.orNot(values)) : missing; } else if (!hadTrue && hadFalse) { newValues = storage.isNegated() ? missing.notAnd(values) : missing.notAndNot(values); - newMissing = hadNull ? (storage.isNegated() ? missing.orNot(values) : missing.or(values)) : missing; + newMissing = + hadNull ? (storage.isNegated() ? missing.orNot(values) : missing.or(values)) : missing; } else if (hadTrue && hadFalse) { newValues = missing.not(); newMissing = missing; diff --git a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java index 3a4e298183c5..ce46540f1d1c 100644 --- a/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java +++ b/std-bits/table/src/main/java/org/enso/table/util/ImmutableBitSet.java @@ -1,83 +1,83 @@ package org.enso.table.util; import java.util.BitSet; + /** - * A wrapper around BitSet that implements boolean operations conveniently. - * Unlike BitSet, ImmutableBitSet takes a size parameter, which allows .not to - * be implemented. + * A wrapper around BitSet that implements boolean operations conveniently. Unlike BitSet, + * ImmutableBitSet takes a size parameter, which allows .not to be implemented. */ public class ImmutableBitSet { - private BitSet bitSet; - private int size; + private BitSet bitSet; + private int size; - public ImmutableBitSet(BitSet bitSet, int size) { - this.bitSet = bitSet; - this.size = size; - } + public ImmutableBitSet(BitSet bitSet, int size) { + this.bitSet = bitSet; + this.size = size; + } - public BitSet toBitSet() { - return bitSet; - } + public BitSet toBitSet() { + return bitSet; + } - public ImmutableBitSet and(ImmutableBitSet other) { - assert size == other.size; - BitSet result = (BitSet) bitSet.clone(); - result.and(other.bitSet); - return new ImmutableBitSet(result, size); - } + public ImmutableBitSet and(ImmutableBitSet other) { + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + result.and(other.bitSet); + return new ImmutableBitSet(result, size); + } - public ImmutableBitSet or(ImmutableBitSet other) { - assert size == other.size; - BitSet result = (BitSet) bitSet.clone(); - result.or(other.bitSet); - return new ImmutableBitSet(result, size); - } + public ImmutableBitSet or(ImmutableBitSet other) { + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + result.or(other.bitSet); + return new ImmutableBitSet(result, size); + } - public ImmutableBitSet andNot(ImmutableBitSet other) { - assert size == other.size; - BitSet result = (BitSet) bitSet.clone(); - result.andNot(other.bitSet); - return new ImmutableBitSet(result, size); - } + public ImmutableBitSet andNot(ImmutableBitSet other) { + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + result.andNot(other.bitSet); + return new ImmutableBitSet(result, size); + } - public ImmutableBitSet not() { - BitSet result = (BitSet) bitSet.clone(); - result.flip(0, size); - return new ImmutableBitSet(result, size); - } + public ImmutableBitSet not() { + BitSet result = (BitSet) bitSet.clone(); + result.flip(0, size); + return new ImmutableBitSet(result, size); + } - public ImmutableBitSet notAnd(ImmutableBitSet other) { - assert size == other.size; - BitSet result = (BitSet) bitSet.clone(); - result.flip(0, size); - result.and(other.bitSet); - return new ImmutableBitSet(result, size); - } + public ImmutableBitSet notAnd(ImmutableBitSet other) { + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + result.flip(0, size); + result.and(other.bitSet); + return new ImmutableBitSet(result, size); + } - public ImmutableBitSet notAndNot(ImmutableBitSet other) { - assert size == other.size; - BitSet result = (BitSet) bitSet.clone(); - result.flip(0, size); - result.andNot(other.bitSet); - return new ImmutableBitSet(result, size); - } + public ImmutableBitSet notAndNot(ImmutableBitSet other) { + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + result.flip(0, size); + result.andNot(other.bitSet); + return new ImmutableBitSet(result, size); + } - public ImmutableBitSet orNot(ImmutableBitSet other) { - // Doing an extra operation to avoid doing an extra allocation. - // a || !b => !(!a && b) - assert size == other.size; - BitSet result = (BitSet) bitSet.clone(); - result.flip(0, size); - result.and(other.bitSet); - result.flip(0, size); - return new ImmutableBitSet(result, size); - } + public ImmutableBitSet orNot(ImmutableBitSet other) { + // Doing an extra operation to avoid doing an extra allocation. + // a || !b => !(!a && b) + assert size == other.size; + BitSet result = (BitSet) bitSet.clone(); + result.flip(0, size); + result.and(other.bitSet); + result.flip(0, size); + return new ImmutableBitSet(result, size); + } - public static ImmutableBitSet allFalse(int size) { - return new ImmutableBitSet(new BitSet(), size); - } + public static ImmutableBitSet allFalse(int size) { + return new ImmutableBitSet(new BitSet(), size); + } - public static ImmutableBitSet allTrue(int size) { - return new ImmutableBitSet(new BitSet(), size).not(); - } + public static ImmutableBitSet allTrue(int size) { + return new ImmutableBitSet(new BitSet(), size).not(); + } } From b4db21b274242eea979d9019d539e2c6c7734cb1 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 19 Jan 2024 16:00:02 -0500 Subject: [PATCH 50/55] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6709a1ddfe87..4447e9853e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -603,6 +603,7 @@ - [Added text_length to Column][8606] - [Added none delimiter option for Data.Read][8627] - [Added text_left and text_right to Column][8691] +- [Implement relational NULL semantics for Nothing for in-memory Column operations.][5156] [debug-shortcuts]: https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug @@ -770,6 +771,7 @@ [4120]: https://github.com/enso-org/enso/pull/4120 [4050]: https://github.com/enso-org/enso/pull/4050 [4072]: https://github.com/enso-org/enso/pull/4072 +[5156]: https://github.com/enso-org/enso/pull/5156 [5582]: https://github.com/enso-org/enso/pull/5582 [5645]: https://github.com/enso-org/enso/pull/5645 [5646]: https://github.com/enso-org/enso/pull/5646 From 83645ba0c1de17cda339d3a9f926133f870e3c71 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 22 Jan 2024 12:21:22 -0500 Subject: [PATCH 51/55] Handle null vs empty in general is_in --- .../lib/Standard/Table/0.0.0-dev/src/Data/Column.enso | 2 +- .../table/data/column/operation/map/SpecializedIsInOp.java | 4 +++- .../Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso index d2596f446a02..717e3a9414c5 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso @@ -1600,7 +1600,7 @@ type Column run_vectorized_binary_op self op_name as_vector expected_result_type=Value_Type.Boolean skip_nulls=False new_name=result_name False -> set = Set.from_vector as_vector error_on_duplicates=False - run_unary_op self set.contains_relational new_name=result_name skip_nulls=True expected_result_type=Value_Type.Boolean + run_unary_op self set.contains_relational new_name=result_name skip_nulls=False expected_result_type=Value_Type.Boolean ## GROUP Standard.Base.Conversions ICON convert diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java index 9c612c3e71f9..2d6cb611f145 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java @@ -57,7 +57,9 @@ public Storage runMap(S storage, List arg) { BitSet newVals = new BitSet(); BitSet missing = new BitSet(); for (int i = 0; i < storage.size(); i++) { - if (storage.isNa(i)) { + if (arg.size() == 0) { + // Leave as default=false + } else if (storage.isNa(i)) { missing.set(i); } else if (compactRepresentation.coercedValues.contains(storage.getItemBoxed(i))) { newVals.set(i); diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 99f33c26b539..a80a23f03986 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -142,6 +142,9 @@ spec setup = Test.specify "Correctly handle Nothing in: Nothing.is_in Vector with Nothings (returning Nothing)" <| table.at "n" . is_in (table.at "x" . to_vector) . to_vector . should_equal [Nothing, Nothing] + Test.specify "Correctly handle Nothing in: "+value_type.to_text+".is_in empty Vector (returning False)" <| + table.at "x" . is_in [] . to_vector . should_equal [False, False] + Test.group prefix+"Boolean is_in" <| make_containing_values had_null had_true had_false = null_maybe = if had_null then [Nothing] else [] From b692bea22f8a849c52d2484c9bb8f885ae4f554d Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 22 Jan 2024 12:24:00 -0500 Subject: [PATCH 52/55] Handle null vs empty in general is_in, only check once --- .../operation/map/SpecializedIsInOp.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java index 2d6cb611f145..9409ca1d7862 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/map/SpecializedIsInOp.java @@ -56,20 +56,20 @@ public Storage runMap(S storage, List arg) { CompactRepresentation compactRepresentation = prepareList(arg); BitSet newVals = new BitSet(); BitSet missing = new BitSet(); - for (int i = 0; i < storage.size(); i++) { - if (arg.size() == 0) { - // Leave as default=false - } else if (storage.isNa(i)) { - missing.set(i); - } else if (compactRepresentation.coercedValues.contains(storage.getItemBoxed(i))) { - newVals.set(i); - } else if (compactRepresentation.hasNulls) { - missing.set(i); - } else { - // Leave as default=false - } + if (arg.size() > 0) { + for (int i = 0; i < storage.size(); i++) { + if (storage.isNa(i)) { + missing.set(i); + } else if (compactRepresentation.coercedValues.contains(storage.getItemBoxed(i))) { + newVals.set(i); + } else if (compactRepresentation.hasNulls) { + missing.set(i); + } else { + // Leave as default=false + } - context.safepoint(); + context.safepoint(); + } } return new BoolStorage(newVals, missing, storage.size(), false); } From 0d328cf9cdfc2a0ac7c18a14c3515b10454bc10c Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 22 Jan 2024 12:49:20 -0500 Subject: [PATCH 53/55] generate cases --- .../Common_Table_Operations/Nothing_Spec.enso | 34 +++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index a80a23f03986..5f3aa8315e12 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -152,31 +152,15 @@ spec setup = false_maybe = if had_false then [False] else [] null_maybe + true_maybe + false_maybe - # [input, had_null, had_true, had_false, output] - cases = [[True, True, True, True, True]] - + [[True, True, True, False, True]] - + [[True, False, True, True, True]] - + [[True, False, True, False, True]] - + [[True, True, False, True, Nothing]] - + [[True, True, False, False, Nothing]] - + [[True, False, False, True, False]] - + [[True, False, False, False, False]] - + [[False, True, True, True, True]] - + [[False, True, False, True, True]] - + [[False, False, True, True, True]] - + [[False, False, False, True, True]] - + [[False, True, True, False, Nothing]] - + [[False, True, False, False, Nothing]] - + [[False, False, True, False, False]] - + [[False, False, False, False, False]] - + [[Nothing, True, True, True, Nothing]] - + [[Nothing, True, False, True, Nothing]] - + [[Nothing, False, True, True, Nothing]] - + [[Nothing, False, False, True, Nothing]] - + [[Nothing, True, True, False, Nothing]] - + [[Nothing, True, False, False, Nothing]] - + [[Nothing, False, True, False, Nothing]] - + [[Nothing, False, False, False, False]] + bools = [True, False] + bools_and_nothing = [True, False, Nothing] + # Format: [input, had_null, had_true, had_false, output] + cases = bools_and_nothing.flat_map input-> bools.flat_map had_null-> bools.flat_map had_true-> bools.map had_false-> + output = case input of + True -> if had_true then True else (if had_null then Nothing else False) + False -> if had_false then True else (if had_null then Nothing else False) + Nothing -> if had_null || had_true || had_false then Nothing else False + [input, had_null, had_true, had_false, output] # To test negated columns, we invert the values before putting them into the Storage, and set the negated bit. negate : Vector | Column -> Vector | Column From 7f610d07ea80a4153641adfebf792e4a63168ba8 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 22 Jan 2024 15:24:37 -0500 Subject: [PATCH 54/55] fix changelog format --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4447e9853e50..1fd70668e350 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -603,7 +603,8 @@ - [Added text_length to Column][8606] - [Added none delimiter option for Data.Read][8627] - [Added text_left and text_right to Column][8691] -- [Implement relational NULL semantics for Nothing for in-memory Column operations.][5156] +- [Implement relational `NULL` semantics for `Nothing` for in-memory Column + operations.][5156] [debug-shortcuts]: https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug From 2f7777574ef26b72c6a765af4bf4b788ba150402 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 24 Jan 2024 11:04:35 -0500 Subject: [PATCH 55/55] rename neg_ vars --- .../src/Common_Table_Operations/Nothing_Spec.enso | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 5f3aa8315e12..57ac35f56e19 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -175,8 +175,8 @@ spec setup = negation_cases.map negation_case-> negation_desc = negation_case.at 0 - neg_input = negation_case.at 1 - neg_argument = negation_case.at 2 + transform_input = negation_case.at 1 + transform_argument = negation_case.at 2 cases.map cs-> input = cs.at 0 @@ -184,19 +184,19 @@ spec setup = containing_values = make_containing_values (cs.at 1) (cs.at 2) (cs.at 3) Test.specify "Boolean is_in: (Vector), "+negation_desc+" "+cs.to_text <| - input_column = neg_input [input] + input_column = transform_input [input] t = table_builder_typed [["input", input_column]] Value_Type.Boolean - c = (neg_input (t.at "input")) . is_in containing_values + c = (transform_input (t.at "input")) . is_in containing_values c.to_vector . should_equal [output] Test.specify "Boolean is_in: (Column), "+negation_desc+" "+cs.to_text <| - input_column = neg_input (Vector.fill containing_values.length input) - t = table_builder_typed [["input", input_column], ["containing", neg_argument containing_values]] Value_Type.Boolean + input_column = transform_input (Vector.fill containing_values.length input) + t = table_builder_typed [["input", input_column], ["containing", transform_argument containing_values]] Value_Type.Boolean expected_output = if input_column.is_empty then [] else [output] - c = (neg_input (t.at "input")) . is_in (neg_argument (t.at "containing")) + c = (transform_input (t.at "input")) . is_in (transform_argument (t.at "containing")) c.to_vector . length . should_equal input_column.length c.to_vector.distinct . should_equal expected_output