From a4f4efb6a650bf6beb7b5c38e42dfbe08d467997 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 5 Apr 2023 14:06:01 -0400 Subject: [PATCH 01/64] tests --- .../src/In_Memory/Split_Tokenaize_Spec.enso | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso new file mode 100644 index 000000000000..b9308229220c --- /dev/null +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -0,0 +1,66 @@ +from Standard.Base import all + +from Standard.Table import Table, Column + +from Standard.Test import Test, Test_Suite, Problems +import Standard.Test.Extensions + +spec = + Test.group "split' <| + Test.specify 'can do split_to_columns' <| + cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + t = Table.new cols + expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] + expected = Table.from_rows ['foo', 'bar'], expected_rows + t2 = t.split_to_columns 'bar', 'b' + t2.should_equal expected + ## + t2.at 'bar_0' . to_vector . should_equal ['a', 'c', 'gh'] + t2.at 'bar_1' . to_vector . should_equal ['c', 'd', 'ij'] + t2.at 'bar_2' . to_vector . should_equal [Nothing, 'ef', 'u'] + t2.get 'bar_1' . should_equal Nothing + t2.at 'foo' . to_vector . should_equal cols . at 0 . at 1 + t2.at 'foo' . should_equal cols . at 0 + + Test.specify 'can do split_to_rows' <| + cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + t = Table.new cols + expected_rows = [[0, 'a'], [0, 'c'], [1, 'c'], [1, 'd'], [1, 'ef'], [2, 'gh'], [2, 'ij'], [2, 'u']] + expected = Table.from_rows ['foo', 'bar'], expected_rows + t2 = t.split_to_rows 'bar', 'b' + t2.should_equal expected + + Test.group 'tokenize' <| + Test.specify 'can do tokenize_to_columns' <| + cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] + t = Table.new cols + expected_rows = [[0, '12', '34', '5'], [1, '23'], Nothing, Nothing, [2, '2', '4', '55']] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] + t2 = t.tokenize_to_columns 'bar' "\d+" + t2.should_equal expected + + Test.specify 'can do tokenize_to_rows' <| + cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] + t = Table.new cols + expected_rows = [[0, '12'], [0, '34'], [0, '5'], [1, '23'], [2, '2'], [2, '4'], [2, '55']] + expected = Table.from_rows ['foo', 'bar'], expected_rows + t2 = t.tokenize_to_rows 'bar' "\d+" + t2.should_equal expected + + Test.specify "can do tokenize_to_columns with groups" <| + cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] + t = Table.new cols + expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] + expected = Table.from_rows ['foo', 'bar'], expected_rows + t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" + t2.should_equal expected + + Test.specify "can do tokenize_to_rows with groups" <| + cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] + t = Table.new cols + expected_rows = [[0, 'a1'], [0, 'b12'], [0, 'd50'], [1, 'b10'], [1, 'c20']] + expected = Table.from_rows ['foo', 'bar'], expected_rows + t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" + t2.should_equal expected + +main = Test_Suite.run_main spec From c63861bc19302f6d8f6f6a694861196482d3a5db Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 5 Apr 2023 15:15:38 -0400 Subject: [PATCH 02/64] stubs and split to col --- .../Table/0.0.0-dev/src/Data/Table.enso | 66 +++++++++++++++++++ .../src/Data/Type/Split_Tokenize.enso | 62 +++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Split_Tokenize.enso diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index 43684aebeeb5..9be47932bf55 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -914,6 +914,72 @@ type Table result = Table.new new_columns problem_builder.attach_problems_after on_problems result + ## Splits a column of text into a set of new columns. + The original column will be removed from the table. + The new columns will be named with the name of the input with a + incrementing number after. + + Arguments: + - column: The column to split the text of. + - delimiter: The term or terms used to split the text. + - column_count: The number of columns to split to. + If `Auto` then columns will be added to fit all data. + If the data exceeds the number of columns, a `Column_Count_Exceeded` error + will follow the `on_problems` behavior. + - on_problems: Specifies the behavior when a problem occurs. + split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table + split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = + table_split_to_columns self column delimiter column_count on_problems + + ## Splits a column of text into a set of new rows. + The values of other columns are repeated for the new rows. + + Arguments: + - column: The column to split the text of. + - delimiter: The term or terms used to split the text. + - on_problems: Specifies the behavior when a problem occurs. + split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table + split_to_rows self column delimiter="," on_problems=Report_Error = + table_split_to_rows self column delimiter on_problems + + ## Splits a column of text into a set of new columns using a regular + expression. + If the pattern contains marked groups, the values are concatenated + together otherwise the whole match is returned. + The original column will be removed from the table. + The new columns will be named with the name of the input with a + incrementing number after. + + Arguments: + - column: The column to tokenize the text of. + - pattern: The pattern used to find within the text. + - case_sensitivity: Specifies if the text values should be compared case + sensitively. + - column_count: The number of columns to split to. + If `Auto` then columns will be added to fit all data. + If the data exceeds the number of columns, a `Column_Count_Exceeded` error + will follow the `on_problems` behavior. + - on_problems: Specifies the behavior when a problem occurs. + tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table + tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Auto on_problems=Report_Error = + table_tokenize_to_columns self column pattern case_sensitivity column_count on_problems + + ## Takes a regular expression pattern and returns all the matches as new + rows. + If the pattern contains marked groups, the values are concatenated + together otherwise the whole match is returned. + The values of other columns are repeated for the new rows. + + Arguments: + - column: The column to split the text of. + - pattern: The pattern used to find within the text. + - case_sensitivity: Specifies if the text values should be compared case + sensitively. + - on_problems: Specifies the behavior when a problem occurs. + tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table + tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = + table_tokenize_to_rows self column pattern case_sensitivity on_problems + ## ALIAS Filter Rows Selects only the rows of this table that correspond to `True` values of diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Split_Tokenize.enso new file mode 100644 index 000000000000..0f93d57ad6bd --- /dev/null +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Split_Tokenize.enso @@ -0,0 +1,62 @@ +from Standard.Base import all + +import project.Data.Column.Column +import project.Data.Table.Table +import project.Internal.Problem_Builder.Problem_Builder +import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy + +## PRIVATE + Splits a column of text into a set of new columns. + See `Table.split_to_columns`. +table_split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table +table_split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = + input_column = self.at column + new_column = map_column_to_multiple input_column (ignore_nothing (x-> x.split delimiter)) + +map_column_to_multiple : Column -> (Any -> Vector Any) -> Vector Column +map_column_to_multiple input_column function = + result_row_vectors = input_column.to_vector.map value-> function value + result_column_vectors = transpose_with_pad result_row_vectors + result_column_vectors.map_with_index i-> cv-> + name = input_column.name + "_" + i + Column.from_vector name cv + +## PRIVATE + Swap rows and columns of a vector-of-vectors, padding each vector to be the + same length first. + Assumes both dimensions are non-zero. +transpose_with_pad : Vector (Vector Any) -> Vector (Vector Any) +transpose_with_pad vecs = transpose (pad_vectors default=Nothing) + +## PRIVATE + Swap rows and columns of a vector-of-vectors. + Assumes both dimensions are non-zero. +transpose : Vector (Vector Any) -> Vector (Vector Any) +transpose vecs = + num_output_rows = vecs[0].length + builders = vecs.map _-> Vector.new_builder + vecs.map vec-> + vec.map_with_index i-> v-> + builders.at i . append v + builders.map .to_vector + +## PRIVATE + Pad vectors so they have the same length. +pad_vector : Vector (Vector Any) -> Any -> Vector (Vector Any) +pad_vector vecs pad_value = + length = maximum <| vecs.map .length + vecs.map v-> v.pad length pad_value + +## PRIVATE + Return the maximum value of the vector. + Throws Empty_Error if the vector is empty. +maximum : Vector Any -> Vector Any +maximum vec = if vec.is_empty then Nothing else + vec.reduce (a-> b-> a.max b) + +## PRIVATE + Wrap a function so that it returns Nothing if passed Nothing +ignore_nothing : (Any -> Any) -> (Any -> Any) +ignore_nothing function = x-> case x of + _ : Nothing -> Nothing + _ -> function x From 6ef0e20c7b0139c319c4509184fa69e766321170 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 07:58:50 -0400 Subject: [PATCH 03/64] links --- .../Table/0.0.0-dev/src/Data/Table.enso | 9 +++--- .../Type => Internal}/Split_Tokenize.enso | 30 +++++++++++++++---- test/Table_Tests/src/In_Memory/Main.enso | 2 ++ .../src/In_Memory/Split_Tokenaize_Spec.enso | 16 +++++----- 4 files changed, 40 insertions(+), 17 deletions(-) rename distribution/lib/Standard/Table/0.0.0-dev/src/{Data/Type => Internal}/Split_Tokenize.enso (62%) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index 9be47932bf55..496c4c177a7f 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -29,6 +29,7 @@ import project.Internal.Join_Helpers import project.Internal.Naming_Helpers.Naming_Helpers import project.Internal.Parse_Values_Helper import project.Internal.Problem_Builder.Problem_Builder +import project.Internal.Split_Tokenize import project.Internal.Table_Helpers import project.Internal.Table_Helpers.Table_Column_Helper import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy @@ -929,7 +930,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = - table_split_to_columns self column delimiter column_count on_problems + Split_Tokenize.table_split_to_columns self column delimiter column_count on_problems ## Splits a column of text into a set of new rows. The values of other columns are repeated for the new rows. @@ -940,7 +941,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows self column delimiter="," on_problems=Report_Error = - table_split_to_rows self column delimiter on_problems + Split_Tokenize.table_split_to_rows self column delimiter on_problems ## Splits a column of text into a set of new columns using a regular expression. @@ -962,7 +963,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Auto on_problems=Report_Error = - table_tokenize_to_columns self column pattern case_sensitivity column_count on_problems + Split_Tokenize.table_tokenize_to_columns self column pattern case_sensitivity column_count on_problems ## Takes a regular expression pattern and returns all the matches as new rows. @@ -978,7 +979,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = - table_tokenize_to_rows self column pattern case_sensitivity on_problems + Split_Tokenize.table_tokenize_to_rows self column pattern case_sensitivity on_problems ## ALIAS Filter Rows diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso similarity index 62% rename from distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Split_Tokenize.enso rename to distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 0f93d57ad6bd..4eb92517d647 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -1,17 +1,37 @@ from Standard.Base import all import project.Data.Column.Column +import project.Data.Set_Mode.Set_Mode import project.Data.Table.Table import project.Internal.Problem_Builder.Problem_Builder import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy +from project.Data.Type.Value_Type import Auto + ## PRIVATE Splits a column of text into a set of new columns. See `Table.split_to_columns`. table_split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table table_split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = + _ = [column_count, on_problems] + IO.println <| self.display input_column = self.at column - new_column = map_column_to_multiple input_column (ignore_nothing (x-> x.split delimiter)) + IO.println <| input_column.display + new_columns = map_column_to_multiple input_column (ignore_nothing (x-> x.split delimiter)) + + with_old_removed = self.remove_column column.name + IO.println <| with_old_removed.display + with_new_added = new_columns.fold with_old_removed (t-> c-> t.set c set_mode=Set_Mode.Add) + IO.println <| with_new_added.display + #new_columns.fold (self.remove_column column.name) (t-> c-> t.set c set_mode=Set_Mode.Add) + with_new_added + +split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table +split_to_rows _ _ = 1 +tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table +tokenize_to_columns _ _ = 1 +tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table +tokenize_to_rows _ _ = 1 map_column_to_multiple : Column -> (Any -> Vector Any) -> Vector Column map_column_to_multiple input_column function = @@ -26,7 +46,7 @@ map_column_to_multiple input_column function = same length first. Assumes both dimensions are non-zero. transpose_with_pad : Vector (Vector Any) -> Vector (Vector Any) -transpose_with_pad vecs = transpose (pad_vectors default=Nothing) +transpose_with_pad vecs = transpose (pad_vectors vecs Nothing) ## PRIVATE Swap rows and columns of a vector-of-vectors. @@ -34,7 +54,7 @@ transpose_with_pad vecs = transpose (pad_vectors default=Nothing) transpose : Vector (Vector Any) -> Vector (Vector Any) transpose vecs = num_output_rows = vecs[0].length - builders = vecs.map _-> Vector.new_builder + builders = vecs.map _-> Vector.new_builder num_output_rows vecs.map vec-> vec.map_with_index i-> v-> builders.at i . append v @@ -42,8 +62,8 @@ transpose vecs = ## PRIVATE Pad vectors so they have the same length. -pad_vector : Vector (Vector Any) -> Any -> Vector (Vector Any) -pad_vector vecs pad_value = +pad_vectors : Vector (Vector Any) -> Any -> Vector (Vector Any) +pad_vectors vecs pad_value = length = maximum <| vecs.map .length vecs.map v-> v.pad length pad_value diff --git a/test/Table_Tests/src/In_Memory/Main.enso b/test/Table_Tests/src/In_Memory/Main.enso index af32f71ecfeb..c8e11a8829b5 100644 --- a/test/Table_Tests/src/In_Memory/Main.enso +++ b/test/Table_Tests/src/In_Memory/Main.enso @@ -7,6 +7,7 @@ import project.In_Memory.Builders_Spec import project.In_Memory.Column_Spec import project.In_Memory.Common_Spec import project.In_Memory.Join_Performance_Spec +import project.In_Memory.Split_Tokenize_Spec import project.In_Memory.Table_Spec import project.In_Memory.Table_Date_Spec import project.In_Memory.Table_Date_Time_Spec @@ -22,5 +23,6 @@ spec = Aggregate_Column_Spec.spec Builders_Spec.spec Join_Performance_Spec.spec + Split_Tokenize_Spec.spec main = Test_Suite.run_main spec diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index b9308229220c..7ec634b3e632 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -11,8 +11,8 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] - expected = Table.from_rows ['foo', 'bar'], expected_rows - t2 = t.split_to_columns 'bar', 'b' + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.split_to_columns 'bar' 'b' t2.should_equal expected ## t2.at 'bar_0' . to_vector . should_equal ['a', 'c', 'gh'] @@ -26,8 +26,8 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols expected_rows = [[0, 'a'], [0, 'c'], [1, 'c'], [1, 'd'], [1, 'ef'], [2, 'gh'], [2, 'ij'], [2, 'u']] - expected = Table.from_rows ['foo', 'bar'], expected_rows - t2 = t.split_to_rows 'bar', 'b' + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.split_to_rows 'bar' 'b' t2.should_equal expected Test.group 'tokenize' <| @@ -35,7 +35,7 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] t = Table.new cols expected_rows = [[0, '12', '34', '5'], [1, '23'], Nothing, Nothing, [2, '2', '4', '55']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.tokenize_to_columns 'bar' "\d+" t2.should_equal expected @@ -43,7 +43,7 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] t = Table.new cols expected_rows = [[0, '12'], [0, '34'], [0, '5'], [1, '23'], [2, '2'], [2, '4'], [2, '55']] - expected = Table.from_rows ['foo', 'bar'], expected_rows + expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.tokenize_to_rows 'bar' "\d+" t2.should_equal expected @@ -51,7 +51,7 @@ spec = cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] t = Table.new cols expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] - expected = Table.from_rows ['foo', 'bar'], expected_rows + expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" t2.should_equal expected @@ -59,7 +59,7 @@ spec = cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] t = Table.new cols expected_rows = [[0, 'a1'], [0, 'b12'], [0, 'd50'], [1, 'b10'], [1, 'c20']] - expected = Table.from_rows ['foo', 'bar'], expected_rows + expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" t2.should_equal expected From c1eb857bfff674dcfb5b0ffaa75ccb66bc28d2e6 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 08:39:46 -0400 Subject: [PATCH 04/64] 1 test --- .../src/Internal/Split_Tokenize.enso | 24 +++++++++---------- .../src/In_Memory/Split_Tokenaize_Spec.enso | 13 ++++++++-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 4eb92517d647..31be7e86c8f5 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -11,26 +11,26 @@ from project.Data.Type.Value_Type import Auto ## PRIVATE Splits a column of text into a set of new columns. See `Table.split_to_columns`. -table_split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table -table_split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = +table_split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table +table_split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = _ = [column_count, on_problems] - IO.println <| self.display - input_column = self.at column + IO.println <| table.display + input_column = table.at column IO.println <| input_column.display new_columns = map_column_to_multiple input_column (ignore_nothing (x-> x.split delimiter)) - with_old_removed = self.remove_column column.name + with_old_removed = table.remove_columns column IO.println <| with_old_removed.display with_new_added = new_columns.fold with_old_removed (t-> c-> t.set c set_mode=Set_Mode.Add) IO.println <| with_new_added.display - #new_columns.fold (self.remove_column column.name) (t-> c-> t.set c set_mode=Set_Mode.Add) with_new_added + #new_columns.fold (table.remove_columns column) (t-> c-> t.set c set_mode=Set_Mode.Add) -split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table -split_to_rows _ _ = 1 -tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table +split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table +split_to_rows = 1 +tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns _ _ = 1 -tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table +tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows _ _ = 1 map_column_to_multiple : Column -> (Any -> Vector Any) -> Vector Column @@ -38,7 +38,7 @@ map_column_to_multiple input_column function = result_row_vectors = input_column.to_vector.map value-> function value result_column_vectors = transpose_with_pad result_row_vectors result_column_vectors.map_with_index i-> cv-> - name = input_column.name + "_" + i + name = input_column.name + "_" + i.to_text Column.from_vector name cv ## PRIVATE @@ -53,7 +53,7 @@ transpose_with_pad vecs = transpose (pad_vectors vecs Nothing) Assumes both dimensions are non-zero. transpose : Vector (Vector Any) -> Vector (Vector Any) transpose vecs = - num_output_rows = vecs[0].length + num_output_rows = vecs.first.length builders = vecs.map _-> Vector.new_builder num_output_rows vecs.map vec-> vec.map_with_index i-> v-> diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index 7ec634b3e632..eb8a771705fa 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -6,14 +6,23 @@ from Standard.Test import Test, Test_Suite, Problems import Standard.Test.Extensions spec = + tables_equal t0 t1 = + same_headers = (t0.columns.map .name) == (t1.columns.map .name) + same_columns = (t0.columns.map .to_vector) == (t1.columns.map .to_vector) + same_headers && same_columns + Test.group "split' <| Test.specify 'can do split_to_columns' <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] - expected = Table.from_rows ['foo', 'bar'] expected_rows + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.split_to_columns 'bar' 'b' - t2.should_equal expected + IO.println '----' + IO.println t2.display + IO.println expected.display + (tables_equal t2 expected) . should_equal True + #t2.should_equal expected ## t2.at 'bar_0' . to_vector . should_equal ['a', 'c', 'gh'] t2.at 'bar_1' . to_vector . should_equal ['c', 'd', 'ij'] From 69d3fc2adce59e148d57e5d9825cf26e85b4bd46 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 08:44:39 -0400 Subject: [PATCH 05/64] tables_should_be_equal --- test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index eb8a771705fa..4a30454f424e 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -10,6 +10,7 @@ spec = same_headers = (t0.columns.map .name) == (t1.columns.map .name) same_columns = (t0.columns.map .to_vector) == (t1.columns.map .to_vector) same_headers && same_columns + tables_should_be_equal t0 t1 = tables_equal t0 t1 . should_equal True Test.group "split' <| Test.specify 'can do split_to_columns' <| @@ -21,7 +22,8 @@ spec = IO.println '----' IO.println t2.display IO.println expected.display - (tables_equal t2 expected) . should_equal True + tables_should_be_equal t2 expected + #(tables_equal t2 expected) . should_equal True #t2.should_equal expected ## t2.at 'bar_0' . to_vector . should_equal ['a', 'c', 'gh'] From ad92d85eb1b4ae5afe8e9e6f9d86df33c8e735a3 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 08:47:31 -0400 Subject: [PATCH 06/64] cleanup --- .../src/Internal/Split_Tokenize.enso | 10 +-- .../src/In_Memory/Split_Tokenaize_Spec.enso | 88 ++++++++----------- 2 files changed, 39 insertions(+), 59 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 31be7e86c8f5..16368fc8442e 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -14,17 +14,9 @@ from project.Data.Type.Value_Type import Auto table_split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table table_split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = _ = [column_count, on_problems] - IO.println <| table.display input_column = table.at column - IO.println <| input_column.display new_columns = map_column_to_multiple input_column (ignore_nothing (x-> x.split delimiter)) - - with_old_removed = table.remove_columns column - IO.println <| with_old_removed.display - with_new_added = new_columns.fold with_old_removed (t-> c-> t.set c set_mode=Set_Mode.Add) - IO.println <| with_new_added.display - with_new_added - #new_columns.fold (table.remove_columns column) (t-> c-> t.set c set_mode=Set_Mode.Add) + new_columns.fold (table.remove_columns column) (t-> c-> t.set c set_mode=Set_Mode.Add) split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows = 1 diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index 4a30454f424e..94e993991f1c 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -19,59 +19,47 @@ spec = expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.split_to_columns 'bar' 'b' - IO.println '----' - IO.println t2.display - IO.println expected.display tables_should_be_equal t2 expected - #(tables_equal t2 expected) . should_equal True - #t2.should_equal expected - ## - t2.at 'bar_0' . to_vector . should_equal ['a', 'c', 'gh'] - t2.at 'bar_1' . to_vector . should_equal ['c', 'd', 'ij'] - t2.at 'bar_2' . to_vector . should_equal [Nothing, 'ef', 'u'] - t2.get 'bar_1' . should_equal Nothing - t2.at 'foo' . to_vector . should_equal cols . at 0 . at 1 - t2.at 'foo' . should_equal cols . at 0 + ## + Test.specify 'can do split_to_rows' <| + cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + t = Table.new cols + expected_rows = [[0, 'a'], [0, 'c'], [1, 'c'], [1, 'd'], [1, 'ef'], [2, 'gh'], [2, 'ij'], [2, 'u']] + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.split_to_rows 'bar' 'b' + t2.should_equal expected + ## + Test.group 'tokenize' <| + Test.specify 'can do tokenize_to_columns' <| + cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] + t = Table.new cols + expected_rows = [[0, '12', '34', '5'], [1, '23'], Nothing, Nothing, [2, '2', '4', '55']] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows + t2 = t.tokenize_to_columns 'bar' "\d+" + t2.should_equal expected - Test.specify 'can do split_to_rows' <| - cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] - t = Table.new cols - expected_rows = [[0, 'a'], [0, 'c'], [1, 'c'], [1, 'd'], [1, 'ef'], [2, 'gh'], [2, 'ij'], [2, 'u']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.split_to_rows 'bar' 'b' - t2.should_equal expected + Test.specify 'can do tokenize_to_rows' <| + cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] + t = Table.new cols + expected_rows = [[0, '12'], [0, '34'], [0, '5'], [1, '23'], [2, '2'], [2, '4'], [2, '55']] + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.tokenize_to_rows 'bar' "\d+" + t2.should_equal expected - Test.group 'tokenize' <| - Test.specify 'can do tokenize_to_columns' <| - cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] - t = Table.new cols - expected_rows = [[0, '12', '34', '5'], [1, '23'], Nothing, Nothing, [2, '2', '4', '55']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows - t2 = t.tokenize_to_columns 'bar' "\d+" - t2.should_equal expected + Test.specify "can do tokenize_to_columns with groups" <| + cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] + t = Table.new cols + expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" + t2.should_equal expected - Test.specify 'can do tokenize_to_rows' <| - cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] - t = Table.new cols - expected_rows = [[0, '12'], [0, '34'], [0, '5'], [1, '23'], [2, '2'], [2, '4'], [2, '55']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.tokenize_to_rows 'bar' "\d+" - t2.should_equal expected - - Test.specify "can do tokenize_to_columns with groups" <| - cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] - t = Table.new cols - expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" - t2.should_equal expected - - Test.specify "can do tokenize_to_rows with groups" <| - cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] - t = Table.new cols - expected_rows = [[0, 'a1'], [0, 'b12'], [0, 'd50'], [1, 'b10'], [1, 'c20']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" - t2.should_equal expected + Test.specify "can do tokenize_to_rows with groups" <| + cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] + t = Table.new cols + expected_rows = [[0, 'a1'], [0, 'b12'], [0, 'd50'], [1, 'b10'], [1, 'c20']] + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" + t2.should_equal expected main = Test_Suite.run_main spec From f02648f77d78c92adb44dce202f2d7ea0448dd22 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 08:51:01 -0400 Subject: [PATCH 07/64] comment --- .../Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 16368fc8442e..c0e0fb81cabe 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -25,6 +25,15 @@ tokenize_to_columns _ _ = 1 tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows _ _ = 1 +## PRIVATE + Transform a column into a set of columns. Takes a function that maps a + single element of the input column to a vector of output values. The + vectors of output values are padded with Nothing to be the same length. + + Arguments: + - input_column: The column to transform + - function: A function that transforms a single element of `input_column` + to multiple values. map_column_to_multiple : Column -> (Any -> Vector Any) -> Vector Column map_column_to_multiple input_column function = result_row_vectors = input_column.to_vector.map value-> function value From e0a68218704ff58c2c88d139eaa4245fae1c521c Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 09:25:12 -0400 Subject: [PATCH 08/64] replace_column_with_columns --- .../Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index c0e0fb81cabe..b89355d15b70 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -16,7 +16,7 @@ table_split_to_columns table column delimiter="," column_count=Auto on_problems= _ = [column_count, on_problems] input_column = table.at column new_columns = map_column_to_multiple input_column (ignore_nothing (x-> x.split delimiter)) - new_columns.fold (table.remove_columns column) (t-> c-> t.set c set_mode=Set_Mode.Add) + replace_column_with_columns table column new_columns split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows = 1 @@ -42,6 +42,11 @@ map_column_to_multiple input_column function = name = input_column.name + "_" + i.to_text Column.from_vector name cv +replace_column_with_columns : Table -> Column -> Vector Column -> Table +replace_column_with_columns table old_column new_columns = + with_column_removed = table.remove_columns old_column error_on_missing_columns=True + new_columns.fold with_column_removed (t-> c-> t.set c set_mode=Set_Mode.Add) + ## PRIVATE Swap rows and columns of a vector-of-vectors, padding each vector to be the same length first. From f6371abbcdcc9c49ef8ed383122cf4bee6494dc0 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 09:37:20 -0400 Subject: [PATCH 09/64] test 2 --- .../Table/0.0.0-dev/src/Data/Table.enso | 8 +++--- .../src/Internal/Split_Tokenize.enso | 15 ++++++++--- .../src/In_Memory/Split_Tokenaize_Spec.enso | 27 ++++++++++--------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index 496c4c177a7f..51ccdc547765 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -930,7 +930,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = - Split_Tokenize.table_split_to_columns self column delimiter column_count on_problems + Split_Tokenize.split_to_columns self column delimiter column_count on_problems ## Splits a column of text into a set of new rows. The values of other columns are repeated for the new rows. @@ -941,7 +941,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows self column delimiter="," on_problems=Report_Error = - Split_Tokenize.table_split_to_rows self column delimiter on_problems + Split_Tokenize.split_to_rows self column delimiter on_problems ## Splits a column of text into a set of new columns using a regular expression. @@ -963,7 +963,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Auto on_problems=Report_Error = - Split_Tokenize.table_tokenize_to_columns self column pattern case_sensitivity column_count on_problems + Split_Tokenize.tokenize_to_columns self column pattern case_sensitivity column_count on_problems ## Takes a regular expression pattern and returns all the matches as new rows. @@ -979,7 +979,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = - Split_Tokenize.table_tokenize_to_rows self column pattern case_sensitivity on_problems + Split_Tokenize.tokenize_to_rows self column pattern case_sensitivity on_problems ## ALIAS Filter Rows diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index b89355d15b70..013d6c989f76 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -11,8 +11,8 @@ from project.Data.Type.Value_Type import Auto ## PRIVATE Splits a column of text into a set of new columns. See `Table.split_to_columns`. -table_split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table -table_split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = +split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table +split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = _ = [column_count, on_problems] input_column = table.at column new_columns = map_column_to_multiple input_column (ignore_nothing (x-> x.split delimiter)) @@ -20,8 +20,17 @@ table_split_to_columns table column delimiter="," column_count=Auto on_problems= split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows = 1 + +## PRIVATE + Tokenize a column of text into a set of new columns. + See `Table.tokenize_to_columns`. tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table -tokenize_to_columns _ _ = 1 +tokenize_to_columns table column pattern case_sensitivity column_count on_problems = + _ = [column_count, on_problems] + input_column = table.at column + new_columns = map_column_to_multiple input_column (ignore_nothing (x-> x.tokenize pattern case_sensitivity)) + replace_column_with_columns table column new_columns + tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows _ _ = 1 diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index 94e993991f1c..01a4471ba784 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -27,24 +27,25 @@ spec = expected_rows = [[0, 'a'], [0, 'c'], [1, 'c'], [1, 'd'], [1, 'ef'], [2, 'gh'], [2, 'ij'], [2, 'u']] expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.split_to_rows 'bar' 'b' - t2.should_equal expected - ## - Test.group 'tokenize' <| - Test.specify 'can do tokenize_to_columns' <| - cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] - t = Table.new cols - expected_rows = [[0, '12', '34', '5'], [1, '23'], Nothing, Nothing, [2, '2', '4', '55']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows - t2 = t.tokenize_to_columns 'bar' "\d+" - t2.should_equal expected + tables_should_be_equal t2 expected + Test.group 'tokenize' <| + Test.specify 'can do tokenize_to_columns' <| + cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] + t = Table.new cols + expected_rows = [[0, '12', '34', '5'], [1, '23', Nothing, Nothing], [2, '2', '4', '55']] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows + t2 = t.tokenize_to_columns 'bar' "\d+" + tables_should_be_equal t2 expected + + ## Test.specify 'can do tokenize_to_rows' <| cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] t = Table.new cols expected_rows = [[0, '12'], [0, '34'], [0, '5'], [1, '23'], [2, '2'], [2, '4'], [2, '55']] expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.tokenize_to_rows 'bar' "\d+" - t2.should_equal expected + tables_should_be_equal t2 expected Test.specify "can do tokenize_to_columns with groups" <| cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] @@ -52,7 +53,7 @@ spec = expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" - t2.should_equal expected + tables_should_be_equal t2 expected Test.specify "can do tokenize_to_rows with groups" <| cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] @@ -60,6 +61,6 @@ spec = expected_rows = [[0, 'a1'], [0, 'b12'], [0, 'd50'], [1, 'b10'], [1, 'c20']] expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" - t2.should_equal expected + tables_should_be_equal t2 expected main = Test_Suite.run_main spec From 351539f045e6108c87a00df81dc8b13e537037fe Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 10:23:21 -0400 Subject: [PATCH 10/64] to_cols --- .../src/Internal/Split_Tokenize.enso | 5 +++-- .../src/In_Memory/Split_Tokenaize_Spec.enso | 21 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 013d6c989f76..c0669b79cb8a 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -68,8 +68,9 @@ transpose_with_pad vecs = transpose (pad_vectors vecs Nothing) Assumes both dimensions are non-zero. transpose : Vector (Vector Any) -> Vector (Vector Any) transpose vecs = - num_output_rows = vecs.first.length - builders = vecs.map _-> Vector.new_builder num_output_rows + num_output_rows = vecs.length + num_output_cols = vecs.first.length + builders = (0.up_to num_output_cols).map _-> Vector.new_builder num_output_rows vecs.map vec-> vec.map_with_index i-> v-> builders.at i . append v diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index 01a4471ba784..e2658d252592 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -10,7 +10,11 @@ spec = same_headers = (t0.columns.map .name) == (t1.columns.map .name) same_columns = (t0.columns.map .to_vector) == (t1.columns.map .to_vector) same_headers && same_columns - tables_should_be_equal t0 t1 = tables_equal t0 t1 . should_equal True + tables_should_be_equal t0 t1 = + equal = tables_equal t0 t1 + if equal.not then + msg = 'Tables differ:\n' + t0.display + '\n' + t1.display + Test.fail msg Test.group "split' <| Test.specify 'can do split_to_columns' <| @@ -47,14 +51,15 @@ spec = t2 = t.tokenize_to_rows 'bar' "\d+" tables_should_be_equal t2 expected - Test.specify "can do tokenize_to_columns with groups" <| - cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] - t = Table.new cols - expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" - tables_should_be_equal t2 expected + Test.specify "can do tokenize_to_columns with groups" <| + cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + t = Table.new cols + expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows + t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" + tables_should_be_equal t2 expected + ## Test.specify "can do tokenize_to_rows with groups" <| cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] t = Table.new cols From f6bc31cc76a1c92229a13eae5b283665e53efb25 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 10:32:16 -0400 Subject: [PATCH 11/64] transform_table_column_to_columns --- .../src/Internal/Split_Tokenize.enso | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index c0669b79cb8a..27fb1cb020cb 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -14,9 +14,7 @@ from project.Data.Type.Value_Type import Auto split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = _ = [column_count, on_problems] - input_column = table.at column - new_columns = map_column_to_multiple input_column (ignore_nothing (x-> x.split delimiter)) - replace_column_with_columns table column new_columns + transform_table_column_to_columns table column (ignore_nothing (x-> x.split delimiter)) split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows = 1 @@ -51,6 +49,22 @@ map_column_to_multiple input_column function = name = input_column.name + "_" + i.to_text Column.from_vector name cv +## PRIVATE + Transform a table by transforming a column into a set of columns. Takes a + function that maps a single element of the input column to a vector of output + values. The original column is replaced by the new columns. + + Arguments: + - table: The table to transform. + - input_column: The column to transform. + - function: A function that transforms a single element of `input_column` + to multiple values. +transform_table_column_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Table +transform_table_column_to_columns table column function = + input_column = table.at column + new_columns = map_column_to_multiple input_column function + replace_column_with_columns table column new_columns + replace_column_with_columns : Table -> Column -> Vector Column -> Table replace_column_with_columns table old_column new_columns = with_column_removed = table.remove_columns old_column error_on_missing_columns=True From d9b79dfd731f94b566eca617107a8e361b28f27c Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 10:33:20 -0400 Subject: [PATCH 12/64] fin --- .../Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 27fb1cb020cb..7a1ec0dd3d52 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -25,9 +25,7 @@ split_to_rows = 1 tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns table column pattern case_sensitivity column_count on_problems = _ = [column_count, on_problems] - input_column = table.at column - new_columns = map_column_to_multiple input_column (ignore_nothing (x-> x.tokenize pattern case_sensitivity)) - replace_column_with_columns table column new_columns + transform_table_column_to_columns table column (ignore_nothing (x-> x.tokenize pattern case_sensitivity)) tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows _ _ = 1 From 7663d2754eaf25adff7f377b64192433d7aa8d86 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 14:16:55 -0400 Subject: [PATCH 13/64] split_to_rows --- .../src/Internal/Split_Tokenize.enso | 50 ++++++++++++++++++- .../src/In_Memory/Split_Tokenaize_Spec.enso | 16 +++--- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 7a1ec0dd3d52..1138a1f63940 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -17,7 +17,53 @@ split_to_columns table column delimiter="," column_count=Auto on_problems=Report transform_table_column_to_columns table column (ignore_nothing (x-> x.split delimiter)) split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table -split_to_rows = 1 +split_to_rows table column delimiter="," on_problems=Report_Error = + _ = on_problems + multiply_rows table column (x-> x.split delimiter) + +## Private + Transform a column by applying the given function to the values in the + column. The function produces multiple outputs, so each row is duplicated, + with each row getting a distinct output value in place of the original + input value. The other column values are just duplicated. + + Arguments: + - table: The table to transform. + - input_column: The column to transform. + - function: A function that transforms a single element of `input_column` + to multiple values. +multiply_rows : Table -> Text | Integer -> (Any -> Vector Any) -> Table +multiply_rows table column function = + input_column = table.at column + + # Transform each input value to an output set. + output_value_sets = input_column.to_vector.map function + output_value_set_sizes = output_value_sets.map .length + num_input_rows = table.row_count + num_output_rows = output_value_set_sizes.fold 0 (+) + + ## Generate new columns for the output table. For the input_column, the + new column consists of the concatenation of the output value sets. For + each of the other columns, it consists of the elements of the input + column repeated a number times, with that number equal to the size of + the corresponding output value set. + new_columns = table.columns.map column-> + builder = Vector.new_builder num_output_rows + case column.name == input_column.name of + True -> + # The transformed column: concatenate output value sets. + output_value_sets.map values-> + builder.append_vector_range values + False -> + # The other columns: repeat the input values. + column_vector = column.to_vector + 0.up_to num_input_rows . map i-> + repetition_count = output_value_set_sizes.at i + input_value = column_vector.at i + 0.up_to repetition_count . map _-> + builder.append input_value + Column.from_vector column.name (builder.to_vector) + Table.new new_columns ## PRIVATE Tokenize a column of text into a set of new columns. @@ -36,7 +82,7 @@ tokenize_to_rows _ _ = 1 vectors of output values are padded with Nothing to be the same length. Arguments: - - input_column: The column to transform + - input_column: The column to transform. - function: A function that transforms a single element of `input_column` to multiple values. map_column_to_multiple : Column -> (Any -> Vector Any) -> Vector Column diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index e2658d252592..f467445ffa0a 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -24,14 +24,14 @@ spec = expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.split_to_columns 'bar' 'b' tables_should_be_equal t2 expected - ## - Test.specify 'can do split_to_rows' <| - cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] - t = Table.new cols - expected_rows = [[0, 'a'], [0, 'c'], [1, 'c'], [1, 'd'], [1, 'ef'], [2, 'gh'], [2, 'ij'], [2, 'u']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.split_to_rows 'bar' 'b' - tables_should_be_equal t2 expected + + Test.specify 'can do split_to_rows' <| + cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + t = Table.new cols + expected_rows = [[0, 'a'], [0, 'c'], [1, 'c'], [1, 'd'], [1, 'ef'], [2, 'gh'], [2, 'ij'], [2, 'u']] + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.split_to_rows 'bar' 'b' + tables_should_be_equal t2 expected Test.group 'tokenize' <| Test.specify 'can do tokenize_to_columns' <| From 345ac273f818b5b857649e60a74a14e6fcda5bd6 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 14:27:33 -0400 Subject: [PATCH 14/64] tok rows --- .../src/Internal/Split_Tokenize.enso | 6 ++-- .../src/In_Memory/Split_Tokenaize_Spec.enso | 30 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 1138a1f63940..3ee35a03d572 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -19,7 +19,7 @@ split_to_columns table column delimiter="," column_count=Auto on_problems=Report split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows table column delimiter="," on_problems=Report_Error = _ = on_problems - multiply_rows table column (x-> x.split delimiter) + multiply_rows table column (ignore_nothing (x-> x.split delimiter)) ## Private Transform a column by applying the given function to the values in the @@ -74,7 +74,9 @@ tokenize_to_columns table column pattern case_sensitivity column_count on_proble transform_table_column_to_columns table column (ignore_nothing (x-> x.tokenize pattern case_sensitivity)) tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table -tokenize_to_rows _ _ = 1 +tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = + _ = on_problems + multiply_rows table column (ignore_nothing (x-> x.tokenize pattern case_sensitivity)) ## PRIVATE Transform a column into a set of columns. Takes a function that maps a diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index f467445ffa0a..f455d6d0e983 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -42,14 +42,13 @@ spec = t2 = t.tokenize_to_columns 'bar' "\d+" tables_should_be_equal t2 expected - ## - Test.specify 'can do tokenize_to_rows' <| - cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] - t = Table.new cols - expected_rows = [[0, '12'], [0, '34'], [0, '5'], [1, '23'], [2, '2'], [2, '4'], [2, '55']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.tokenize_to_rows 'bar' "\d+" - tables_should_be_equal t2 expected + Test.specify 'can do tokenize_to_rows' <| + cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] + t = Table.new cols + expected_rows = [[0, '12'], [0, '34'], [0, '5'], [1, '23'], [2, '2'], [2, '4'], [2, '55']] + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.tokenize_to_rows 'bar' "\d+" + tables_should_be_equal t2 expected Test.specify "can do tokenize_to_columns with groups" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] @@ -59,13 +58,12 @@ spec = t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" tables_should_be_equal t2 expected - ## - Test.specify "can do tokenize_to_rows with groups" <| - cols = [['foo', 0, 1], ['bar', 'r a-1, b-12,qd-50', 'ab-10:bc-20c']] - t = Table.new cols - expected_rows = [[0, 'a1'], [0, 'b12'], [0, 'd50'], [1, 'b10'], [1, 'c20']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" - tables_should_be_equal t2 expected + Test.specify "can do tokenize_to_rows with groups" <| + cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + t = Table.new cols + expected_rows = [[0, 'a1'], [0, 'b12'], [0, 'd50'], [1, 'b10'], [1, 'c20']] + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" + tables_should_be_equal t2 expected main = Test_Suite.run_main spec From a97f2c0911ca9b980f74eb51948977407b000f75 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 14:29:26 -0400 Subject: [PATCH 15/64] _. form --- .../Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 3ee35a03d572..306ebf40faf7 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -14,12 +14,12 @@ from project.Data.Type.Value_Type import Auto split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = _ = [column_count, on_problems] - transform_table_column_to_columns table column (ignore_nothing (x-> x.split delimiter)) + transform_table_column_to_columns table column (ignore_nothing (_.split delimiter)) split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows table column delimiter="," on_problems=Report_Error = _ = on_problems - multiply_rows table column (ignore_nothing (x-> x.split delimiter)) + multiply_rows table column (ignore_nothing (_.split delimiter)) ## Private Transform a column by applying the given function to the values in the @@ -71,12 +71,12 @@ multiply_rows table column function = tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns table column pattern case_sensitivity column_count on_problems = _ = [column_count, on_problems] - transform_table_column_to_columns table column (ignore_nothing (x-> x.tokenize pattern case_sensitivity)) + transform_table_column_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = _ = on_problems - multiply_rows table column (ignore_nothing (x-> x.tokenize pattern case_sensitivity)) + multiply_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) ## PRIVATE Transform a column into a set of columns. Takes a function that maps a From 96ddaaad3f46f7cf19b474f7fbbd3c96d847ca93 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 14:36:36 -0400 Subject: [PATCH 16/64] rearrange --- .../Table/0.0.0-dev/src/Data/Table.enso | 6 ++-- .../src/Internal/Split_Tokenize.enso | 36 +++++++++++-------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index 51ccdc547765..1e9838f28ab6 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -943,7 +943,7 @@ type Table split_to_rows self column delimiter="," on_problems=Report_Error = Split_Tokenize.split_to_rows self column delimiter on_problems - ## Splits a column of text into a set of new columns using a regular + ## Tokenizes a column of text into a set of new columns using a regular expression. If the pattern contains marked groups, the values are concatenated together otherwise the whole match is returned. @@ -965,8 +965,8 @@ type Table tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Auto on_problems=Report_Error = Split_Tokenize.tokenize_to_columns self column pattern case_sensitivity column_count on_problems - ## Takes a regular expression pattern and returns all the matches as new - rows. + ## Tokenizes a column of text into a set of new rows using a regular + expression. If the pattern contains marked groups, the values are concatenated together otherwise the whole match is returned. The values of other columns are repeated for the new rows. diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 306ebf40faf7..dd34d2cb29aa 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -16,12 +16,33 @@ split_to_columns table column delimiter="," column_count=Auto on_problems=Report _ = [column_count, on_problems] transform_table_column_to_columns table column (ignore_nothing (_.split delimiter)) +## PRIVATE + Splits a column of text into a set of new rows. + See `Table.split_to_rows`. split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows table column delimiter="," on_problems=Report_Error = _ = on_problems multiply_rows table column (ignore_nothing (_.split delimiter)) -## Private +## PRIVATE + Tokenizes a column of text into a set of new columns using a regular + expression. + See `Table.tokenize_to_columns`. +tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table +tokenize_to_columns table column pattern case_sensitivity column_count on_problems = + _ = [column_count, on_problems] + transform_table_column_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) + +## PRIVATE + Tokenizes a column of text into a set of new rows using a regular + expression. + See `Table.tokenize_to_rows`. +tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table +tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = + _ = on_problems + multiply_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) + +## PRIVATE Transform a column by applying the given function to the values in the column. The function produces multiple outputs, so each row is duplicated, with each row getting a distinct output value in place of the original @@ -65,19 +86,6 @@ multiply_rows table column function = Column.from_vector column.name (builder.to_vector) Table.new new_columns -## PRIVATE - Tokenize a column of text into a set of new columns. - See `Table.tokenize_to_columns`. -tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table -tokenize_to_columns table column pattern case_sensitivity column_count on_problems = - _ = [column_count, on_problems] - transform_table_column_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) - -tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table -tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = - _ = on_problems - multiply_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) - ## PRIVATE Transform a column into a set of columns. Takes a function that maps a single element of the input column to a vector of output values. The From a8522a1de6069638ded0727f9ea8edb442e9e047 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 14:38:10 -0400 Subject: [PATCH 17/64] comments --- .../src/Internal/Split_Tokenize.enso | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index dd34d2cb29aa..4e7a5ddc98de 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -86,6 +86,22 @@ multiply_rows table column function = Column.from_vector column.name (builder.to_vector) Table.new new_columns +## PRIVATE + Transform a table by transforming a column into a set of columns. Takes a + function that maps a single element of the input column to a vector of output + values. The original column is replaced by the new columns. + + Arguments: + - table: The table to transform. + - input_column: The column to transform. + - function: A function that transforms a single element of `input_column` + to multiple values. +transform_table_column_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Table +transform_table_column_to_columns table column function = + input_column = table.at column + new_columns = map_column_to_multiple input_column function + replace_column_with_columns table column new_columns + ## PRIVATE Transform a column into a set of columns. Takes a function that maps a single element of the input column to a vector of output values. The @@ -104,21 +120,7 @@ map_column_to_multiple input_column function = Column.from_vector name cv ## PRIVATE - Transform a table by transforming a column into a set of columns. Takes a - function that maps a single element of the input column to a vector of output - values. The original column is replaced by the new columns. - - Arguments: - - table: The table to transform. - - input_column: The column to transform. - - function: A function that transforms a single element of `input_column` - to multiple values. -transform_table_column_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Table -transform_table_column_to_columns table column function = - input_column = table.at column - new_columns = map_column_to_multiple input_column function - replace_column_with_columns table column new_columns - + Remove a column and add new columns. replace_column_with_columns : Table -> Column -> Vector Column -> Table replace_column_with_columns table old_column new_columns = with_column_removed = table.remove_columns old_column error_on_missing_columns=True From 95f3adc75006c5999dfa0c1a6c2103de25648d2b Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 6 Apr 2023 14:40:28 -0400 Subject: [PATCH 18/64] rename --- .../src/Internal/Split_Tokenize.enso | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 4e7a5ddc98de..a052588c11dd 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -14,7 +14,7 @@ from project.Data.Type.Value_Type import Auto split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = _ = [column_count, on_problems] - transform_table_column_to_columns table column (ignore_nothing (_.split delimiter)) + fan_out_to_columns table column (ignore_nothing (_.split delimiter)) ## PRIVATE Splits a column of text into a set of new rows. @@ -22,7 +22,7 @@ split_to_columns table column delimiter="," column_count=Auto on_problems=Report split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows table column delimiter="," on_problems=Report_Error = _ = on_problems - multiply_rows table column (ignore_nothing (_.split delimiter)) + fan_out_to_rows table column (ignore_nothing (_.split delimiter)) ## PRIVATE Tokenizes a column of text into a set of new columns using a regular @@ -31,7 +31,7 @@ split_to_rows table column delimiter="," on_problems=Report_Error = tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns table column pattern case_sensitivity column_count on_problems = _ = [column_count, on_problems] - transform_table_column_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) + fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) ## PRIVATE Tokenizes a column of text into a set of new rows using a regular @@ -40,7 +40,23 @@ tokenize_to_columns table column pattern case_sensitivity column_count on_proble tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = _ = on_problems - multiply_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) + fan_out_to_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) + +## PRIVATE + Transform a table by transforming a column into a set of columns. Takes a + function that maps a single element of the input column to a vector of output + values. The original column is replaced by the new columns. + + Arguments: + - table: The table to transform. + - input_column: The column to transform. + - function: A function that transforms a single element of `input_column` + to multiple values. +fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Table +fan_out_to_columns table column function = + input_column = table.at column + new_columns = map_column_to_multiple input_column function + replace_column_with_columns table column new_columns ## PRIVATE Transform a column by applying the given function to the values in the @@ -53,8 +69,8 @@ tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sens - input_column: The column to transform. - function: A function that transforms a single element of `input_column` to multiple values. -multiply_rows : Table -> Text | Integer -> (Any -> Vector Any) -> Table -multiply_rows table column function = +fan_out_to_rows : Table -> Text | Integer -> (Any -> Vector Any) -> Table +fan_out_to_rows table column function = input_column = table.at column # Transform each input value to an output set. @@ -86,22 +102,6 @@ multiply_rows table column function = Column.from_vector column.name (builder.to_vector) Table.new new_columns -## PRIVATE - Transform a table by transforming a column into a set of columns. Takes a - function that maps a single element of the input column to a vector of output - values. The original column is replaced by the new columns. - - Arguments: - - table: The table to transform. - - input_column: The column to transform. - - function: A function that transforms a single element of `input_column` - to multiple values. -transform_table_column_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Table -transform_table_column_to_columns table column function = - input_column = table.at column - new_columns = map_column_to_multiple input_column function - replace_column_with_columns table column new_columns - ## PRIVATE Transform a column into a set of columns. Takes a function that maps a single element of the input column to a vector of output values. The From 7f6af1d1b2d0890574d0c554ecbe344fff3a4535 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 7 Apr 2023 11:23:57 -0400 Subject: [PATCH 19/64] must be text --- .../0.0.0-dev/src/Internal/Split_Tokenize.enso | 14 ++++++++++---- .../src/In_Memory/Split_Tokenaize_Spec.enso | 15 +++++++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index a052588c11dd..462e570c33e5 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -3,10 +3,12 @@ from Standard.Base import all import project.Data.Column.Column import project.Data.Set_Mode.Set_Mode import project.Data.Table.Table +from Standard.Table import Value_Type import project.Internal.Problem_Builder.Problem_Builder import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy from project.Data.Type.Value_Type import Auto +from Standard.Table import Value_Type ## PRIVATE Splits a column of text into a set of new columns. @@ -14,7 +16,8 @@ from project.Data.Type.Value_Type import Auto split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = _ = [column_count, on_problems] - fan_out_to_columns table column (ignore_nothing (_.split delimiter)) + Value_Type.expect_text (table.at column).value_type <| + fan_out_to_columns table column (ignore_nothing (_.split delimiter)) ## PRIVATE Splits a column of text into a set of new rows. @@ -22,7 +25,8 @@ split_to_columns table column delimiter="," column_count=Auto on_problems=Report split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows table column delimiter="," on_problems=Report_Error = _ = on_problems - fan_out_to_rows table column (ignore_nothing (_.split delimiter)) + Value_Type.expect_text (table.at column).value_type <| + fan_out_to_rows table column (ignore_nothing (_.split delimiter)) ## PRIVATE Tokenizes a column of text into a set of new columns using a regular @@ -31,7 +35,8 @@ split_to_rows table column delimiter="," on_problems=Report_Error = tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns table column pattern case_sensitivity column_count on_problems = _ = [column_count, on_problems] - fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) + Value_Type.expect_text (table.at column).value_type <| + fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) ## PRIVATE Tokenizes a column of text into a set of new rows using a regular @@ -40,7 +45,8 @@ tokenize_to_columns table column pattern case_sensitivity column_count on_proble tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = _ = on_problems - fan_out_to_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) + Value_Type.expect_text (table.at column).value_type <| + fan_out_to_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) ## PRIVATE Transform a table by transforming a column into a set of columns. Takes a diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index f455d6d0e983..01e1fbc0b206 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -1,9 +1,11 @@ from Standard.Base import all +import Standard.Test.Extensions + from Standard.Table import Table, Column from Standard.Test import Test, Test_Suite, Problems -import Standard.Test.Extensions +from Standard.Table.Errors import Invalid_Value_Type spec = tables_equal t0 t1 = @@ -16,7 +18,7 @@ spec = msg = 'Tables differ:\n' + t0.display + '\n' + t1.display Test.fail msg - Test.group "split' <| + Test.group 'split' <| Test.specify 'can do split_to_columns' <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols @@ -66,4 +68,13 @@ spec = t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" tables_should_be_equal t2 expected + Test.group 'errors' <| + Test.specify "won't work on a non-text column" <| + cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + t = Table.new cols + t.split_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type + t.split_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type + main = Test_Suite.run_main spec From 3c00b5aa81c29b9ccb530ba61b893e527c7d9b83 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 7 Apr 2023 17:51:05 -0400 Subject: [PATCH 20/64] problems --- .../Table/0.0.0-dev/src/Data/Table.enso | 4 +-- .../Standard/Table/0.0.0-dev/src/Errors.enso | 9 +++++++ .../src/Internal/Split_Tokenize.enso | 25 +++++++++++++------ .../src/In_Memory/Split_Tokenaize_Spec.enso | 20 +++++++++++++-- 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index 1e9838f28ab6..c9d534a53e17 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -925,7 +925,7 @@ type Table - delimiter: The term or terms used to split the text. - column_count: The number of columns to split to. If `Auto` then columns will be added to fit all data. - If the data exceeds the number of columns, a `Column_Count_Exceeded` error + If the data exceeds the `column_count`, a `Column_Count_Exceeded` error will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table @@ -958,7 +958,7 @@ type Table sensitively. - column_count: The number of columns to split to. If `Auto` then columns will be added to fit all data. - If the data exceeds the number of columns, a `Column_Count_Exceeded` error + If the data exceeds the `column_count`, a `Column_Count_Exceeded` error will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso index 0485528556d2..b5263dfc71e1 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso @@ -457,3 +457,12 @@ type Invalid_Value_For_Type to_display_text : Text to_display_text self = "The value ["+self.value.to_text+"] is not valid for the column type ["+self.value_type.to_text+"]." + +type Column_Count_Exceeded + ## Indicates that an operation generating new columns produced more columns + than allowed by the limit. + Error (limit : Integer) (column_count : Integer) + + to_display_text : Text + to_display_text self = + "The operation produced more columns than the specified limit. The limit is "+self.limit.to_text+" and the number of new columns was "+self.column_count.to_text+". The limit may be turned off by setting the `limit` option to `Auto`." diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 462e570c33e5..0604b59eebb3 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -3,21 +3,20 @@ from Standard.Base import all import project.Data.Column.Column import project.Data.Set_Mode.Set_Mode import project.Data.Table.Table -from Standard.Table import Value_Type import project.Internal.Problem_Builder.Problem_Builder import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy from project.Data.Type.Value_Type import Auto from Standard.Table import Value_Type +from Standard.Table.Errors import Column_Count_Exceeded ## PRIVATE Splits a column of text into a set of new columns. See `Table.split_to_columns`. split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = - _ = [column_count, on_problems] Value_Type.expect_text (table.at column).value_type <| - fan_out_to_columns table column (ignore_nothing (_.split delimiter)) + fan_out_to_columns table column (ignore_nothing (_.split delimiter)) column_count on_problems ## PRIVATE Splits a column of text into a set of new rows. @@ -34,9 +33,8 @@ split_to_rows table column delimiter="," on_problems=Report_Error = See `Table.tokenize_to_columns`. tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns table column pattern case_sensitivity column_count on_problems = - _ = [column_count, on_problems] Value_Type.expect_text (table.at column).value_type <| - fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) + fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems ## PRIVATE Tokenizes a column of text into a set of new rows using a regular @@ -58,11 +56,22 @@ tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sens - input_column: The column to transform. - function: A function that transforms a single element of `input_column` to multiple values. -fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Table -fan_out_to_columns table column function = +fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Auto | Integer -> Problem_Behavior -> Table +fan_out_to_columns table column function column_count=Auto on_problems=Report_Error = input_column = table.at column new_columns = map_column_to_multiple input_column function - replace_column_with_columns table column new_columns + num_columns = new_columns.length + new_table = replace_column_with_columns table column new_columns + + problem_builder = Problem_Builder.new + + case column_count != Auto && num_columns > column_count of + True-> + problem = Column_Count_Exceeded.Error column_count num_columns + problem_builder.report_other_warning problem + False -> + Nothing + problem_builder.attach_problems_after on_problems new_table ## PRIVATE Transform a column by applying the given function to the values in the diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index 01e1fbc0b206..2b9e671b9dfe 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -1,11 +1,11 @@ from Standard.Base import all +import Standard.Base.Errors.Problem_Behavior.Problem_Behavior import Standard.Test.Extensions from Standard.Table import Table, Column - +from Standard.Table.Errors import Invalid_Value_Type, Column_Count_Exceeded from Standard.Test import Test, Test_Suite, Problems -from Standard.Table.Errors import Invalid_Value_Type spec = tables_equal t0 t1 = @@ -77,4 +77,20 @@ spec = t.tokenize_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type t.tokenize_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type + Test.specify "split should return warnings and errors when selected non-text column" <| + cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + t = Table.new cols + action = t.split_to_columns 'bar' 'b' column_count=1 on_problems=_ + tester = _.should_be_a Table + problems = [Column_Count_Exceeded.Error 1 3] + Problems.test_problem_handling action problems tester + + Test.specify "tokenize should return warnings and errors when selected non-text column" <| + cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + t = Table.new cols + action = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" column_count=1 on_problems=_ + tester = _.should_be_a Table + problems = [Column_Count_Exceeded.Error 1 3] + Problems.test_problem_handling action problems tester + main = Test_Suite.run_main spec From 90b27e5a0ddc8cb2c77abb7bdc440c29b617fb76 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 10 Apr 2023 11:31:34 -0400 Subject: [PATCH 21/64] unique names --- .../src/Internal/Split_Tokenize.enso | 28 ++++++++++----- .../src/In_Memory/Split_Tokenaize_Spec.enso | 35 ++++++++++++++----- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 0604b59eebb3..44c3448996d8 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -16,7 +16,7 @@ from Standard.Table.Errors import Column_Count_Exceeded split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = Value_Type.expect_text (table.at column).value_type <| - fan_out_to_columns table column (ignore_nothing (_.split delimiter)) column_count on_problems + fan_out_to_columns table column (ignore_nothing (_.split delimiter)) "split" column_count on_problems ## PRIVATE Splits a column of text into a set of new rows. @@ -34,7 +34,7 @@ split_to_rows table column delimiter="," on_problems=Report_Error = tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns table column pattern case_sensitivity column_count on_problems = Value_Type.expect_text (table.at column).value_type <| - fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems + fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) "tokenize" column_count on_problems ## PRIVATE Tokenizes a column of text into a set of new rows using a regular @@ -56,10 +56,11 @@ tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sens - input_column: The column to transform. - function: A function that transforms a single element of `input_column` to multiple values. -fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Auto | Integer -> Problem_Behavior -> Table -fan_out_to_columns table column function column_count=Auto on_problems=Report_Error = +fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Text -> Auto | Integer -> Problem_Behavior -> Table +fan_out_to_columns table column function function_name column_count=Auto on_problems=Report_Error = input_column = table.at column - new_columns = map_column_to_multiple input_column function + new_columns_unrenamed = map_column_to_multiple input_column function + new_columns = rename_columns table function_name new_columns_unrenamed num_columns = new_columns.length new_table = replace_column_with_columns table column new_columns @@ -122,6 +123,9 @@ fan_out_to_rows table column function = single element of the input column to a vector of output values. The vectors of output values are padded with Nothing to be the same length. + The names of the resulting columns will be set to the same name as the input + column, and must be renamed before being added to a table. + Arguments: - input_column: The column to transform. - function: A function that transforms a single element of `input_column` @@ -130,9 +134,7 @@ map_column_to_multiple : Column -> (Any -> Vector Any) -> Vector Column map_column_to_multiple input_column function = result_row_vectors = input_column.to_vector.map value-> function value result_column_vectors = transpose_with_pad result_row_vectors - result_column_vectors.map_with_index i-> cv-> - name = input_column.name + "_" + i.to_text - Column.from_vector name cv + result_column_vectors.map cv-> Column.from_vector input_column.name cv ## PRIVATE Remove a column and add new columns. @@ -168,6 +170,16 @@ pad_vectors vecs pad_value = length = maximum <| vecs.map .length vecs.map v-> v.pad length pad_value +## PRIVATE + Rename a set of columns to be unique if they were added to a table. +rename_columns : Table -> Text -> Vector Column -> Vector Column +rename_columns table function_name columns = + old_names = columns.map .name + table_column_names = table.columns . map .name + unique = Unique_Name_Strategy.new + new_names = unique.combine_with_prefix table_column_names old_names (function_name + "_") + columns.map_with_index i-> column-> column.rename (new_names.at i) + ## PRIVATE Return the maximum value of the vector. Throws Empty_Error if the vector is empty. diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index 2b9e671b9dfe..beb26e54307c 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -1,5 +1,6 @@ from Standard.Base import all +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Errors.Problem_Behavior.Problem_Behavior import Standard.Test.Extensions @@ -12,10 +13,10 @@ spec = same_headers = (t0.columns.map .name) == (t1.columns.map .name) same_columns = (t0.columns.map .to_vector) == (t1.columns.map .to_vector) same_headers && same_columns - tables_should_be_equal t0 t1 = - equal = tables_equal t0 t1 + tables_should_be_equal actual expected = + equal = tables_equal actual expected if equal.not then - msg = 'Tables differ:\n' + t0.display + '\n' + t1.display + msg = 'Tables differ. Actual:\n' + actual.display + '\nExpected:\n' + expected.display Test.fail msg Test.group 'split' <| @@ -23,7 +24,7 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows + expected = Table.from_rows ['foo', 'split_bar', 'split_bar_1', 'split_bar_2'] expected_rows t2 = t.split_to_columns 'bar' 'b' tables_should_be_equal t2 expected @@ -40,7 +41,7 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] t = Table.new cols expected_rows = [[0, '12', '34', '5'], [1, '23', Nothing, Nothing], [2, '2', '4', '55']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows + expected = Table.from_rows ['foo', 'tokenize_bar', 'tokenize_bar_1', 'tokenize_bar_2'] expected_rows t2 = t.tokenize_to_columns 'bar' "\d+" tables_should_be_equal t2 expected @@ -56,7 +57,7 @@ spec = cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows + expected = Table.from_rows ['foo', 'tokenize_bar', 'tokenize_bar_1', 'tokenize_bar_2'] expected_rows t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" tables_should_be_equal t2 expected @@ -77,7 +78,7 @@ spec = t.tokenize_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type t.tokenize_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type - Test.specify "split should return warnings and errors when selected non-text column" <| + Test.specify "split should return problems when exceeding the column limit" <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols action = t.split_to_columns 'bar' 'b' column_count=1 on_problems=_ @@ -85,7 +86,7 @@ spec = problems = [Column_Count_Exceeded.Error 1 3] Problems.test_problem_handling action problems tester - Test.specify "tokenize should return warnings and errors when selected non-text column" <| + Test.specify "tokenize should return problems when exceeding the column limit" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols action = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" column_count=1 on_problems=_ @@ -93,4 +94,22 @@ spec = problems = [Column_Count_Exceeded.Error 1 3] Problems.test_problem_handling action problems tester + ## + Test.specify "???" <| + cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + t = Table.new cols + action = t.split_to_columns 'bar' '' on_problems=_ + tester = _.True # _.should_be_a Table + problems = [Illegal_Argument.Error "The delimiter cannot be empty." Nothing] + Problems.test_problem_handling action problems tester + + Test.group "name conflicts" <| + Test.specify 'will make column names unique' <| + cols = [['split_bar', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + t = Table.new cols + expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] + expected = Table.from_rows ['split_bar', 'split_bar_1', 'split_bar_2', 'split_bar_3'] expected_rows + t2 = t.split_to_columns 'bar' 'b' + tables_should_be_equal t2 expected + main = Test_Suite.run_main spec From 3d2a0af481ffdb6a0328883aa8d1a8c9a4014ab2 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 10 Apr 2023 11:52:32 -0400 Subject: [PATCH 22/64] unimplementeds --- .../Database/0.0.0-dev/src/Data/Table.enso | 72 ++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index 7941c2756dc3..a5cb21fb71fc 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1316,7 +1316,7 @@ type Table ## Parsing values is not supported in database tables, the table has to be loaded into memory first with `read`. - parse_values : Text | Integer | Column_Selector | Vector (Text | Integer | Column_Selector) -> (Auto|Integer|Decimal|Date|Date_Time|Time_Of_Day|Boolean) -> Text | Data_Formatter -> Boolean -> Problem_Behavior -> Table + parse_values : Text | Integer | Column_Selector | Vector (Text | Integer | Column_Selector) -> (Auto|Integer|Decimal|Date|Date_Time|Time_Of_Day|Boolean) -> Text | Data_Formatter -> Boolean -> Problem_Behavior -> Table parse_values columns=(self.columns . filter (c-> c.value_type.is_text) . map .name) type=Auto format=Data_Formatter.Value error_on_missing_columns=True on_problems=Report_Warning = ## Avoid unused arguments warning. We cannot rename arguments to `_`, because we need to keep the API consistent with the in-memory table. @@ -1324,6 +1324,76 @@ type Table msg = "Parsing values is not supported in database tables, the table has to be materialized first with `read`." Error.throw (Unsupported_Database_Operation.Error msg) + ## Splits a column of text into a set of new columns. + The original column will be removed from the table. + The new columns will be named with the name of the input with a + incrementing number after. + + Arguments: + - column: The column to split the text of. + - delimiter: The term or terms used to split the text. + - column_count: The number of columns to split to. + If `Auto` then columns will be added to fit all data. + If the data exceeds the `column_count`, a `Column_Count_Exceeded` error + will follow the `on_problems` behavior. + - on_problems: Specifies the behavior when a problem occurs. + split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table + split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = + _ = [column delimiter column_count on_problems] + Unimplemented.throw "This is an interface only." + + ## Splits a column of text into a set of new rows. + The values of other columns are repeated for the new rows. + + Arguments: + - column: The column to split the text of. + - delimiter: The term or terms used to split the text. + - on_problems: Specifies the behavior when a problem occurs. + split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table + split_to_rows self column delimiter="," on_problems=Report_Error = + _ = [column delimiter on_problems] + Unimplemented.throw "This is an interface only." + + ## Tokenizes a column of text into a set of new columns using a regular + expression. + If the pattern contains marked groups, the values are concatenated + together otherwise the whole match is returned. + The original column will be removed from the table. + The new columns will be named with the name of the input with a + incrementing number after. + + Arguments: + - column: The column to tokenize the text of. + - pattern: The pattern used to find within the text. + - case_sensitivity: Specifies if the text values should be compared case + sensitively. + - column_count: The number of columns to split to. + If `Auto` then columns will be added to fit all data. + If the data exceeds the `column_count`, a `Column_Count_Exceeded` error + will follow the `on_problems` behavior. + - on_problems: Specifies the behavior when a problem occurs. + tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table + tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Auto on_problems=Report_Error = + _ = [column pattern case_sensitivity column_count on_problems] + Unimplemented.throw "This is an interface only." + + ## Tokenizes a column of text into a set of new rows using a regular + expression. + If the pattern contains marked groups, the values are concatenated + together otherwise the whole match is returned. + The values of other columns are repeated for the new rows. + + Arguments: + - column: The column to split the text of. + - pattern: The pattern used to find within the text. + - case_sensitivity: Specifies if the text values should be compared case + sensitively. + - on_problems: Specifies the behavior when a problem occurs. + tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table + tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = + _ = [column pattern case_sensitivity on_problems] + Unimplemented.throw "This is an interface only." + ## ALIAS dropna ALIAS drop_missing_rows Remove rows which are all blank or containing blank values. From 321d245984819d5cc796acaa16851ff302693497 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 10 Apr 2023 12:03:30 -0400 Subject: [PATCH 23/64] map_column_vector_to_multiple --- .../src/Internal/Split_Tokenize.enso | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 44c3448996d8..b8d5dbea35fe 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -59,16 +59,16 @@ tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sens fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Text -> Auto | Integer -> Problem_Behavior -> Table fan_out_to_columns table column function function_name column_count=Auto on_problems=Report_Error = input_column = table.at column - new_columns_unrenamed = map_column_to_multiple input_column function - new_columns = rename_columns table function_name new_columns_unrenamed - num_columns = new_columns.length + new_column_vectors = map_column_vector_to_multiple input_column.to_vector function + new_columns = build_and_name_columns table column function_name new_column_vectors + num_new_columns = new_columns.length new_table = replace_column_with_columns table column new_columns problem_builder = Problem_Builder.new - case column_count != Auto && num_columns > column_count of + case column_count != Auto && num_new_columns > column_count of True-> - problem = Column_Count_Exceeded.Error column_count num_columns + problem = Column_Count_Exceeded.Error column_count num_new_columns problem_builder.report_other_warning problem False -> Nothing @@ -119,22 +119,19 @@ fan_out_to_rows table column function = Table.new new_columns ## PRIVATE - Transform a column into a set of columns. Takes a function that maps a - single element of the input column to a vector of output values. The - vectors of output values are padded with Nothing to be the same length. - - The names of the resulting columns will be set to the same name as the input - column, and must be renamed before being added to a table. + Transform a column vector into a set of column vectors. Takes a function + that maps a single element of the input column vector to a vector of output + values. The vectors of output values are padded with Nothing to be the same + length. Arguments: - input_column: The column to transform. - function: A function that transforms a single element of `input_column` to multiple values. -map_column_to_multiple : Column -> (Any -> Vector Any) -> Vector Column -map_column_to_multiple input_column function = - result_row_vectors = input_column.to_vector.map value-> function value - result_column_vectors = transpose_with_pad result_row_vectors - result_column_vectors.map cv-> Column.from_vector input_column.name cv +map_column_vector_to_multiple : Vector Any -> (Any -> Vector Any) -> Vector (Vector Any) +map_column_vector_to_multiple input_vector function = + result_row_vectors = input_vector.map value-> function value + transpose_with_pad result_row_vectors ## PRIVATE Remove a column and add new columns. @@ -171,14 +168,16 @@ pad_vectors vecs pad_value = vecs.map v-> v.pad length pad_value ## PRIVATE - Rename a set of columns to be unique if they were added to a table. -rename_columns : Table -> Text -> Vector Column -> Vector Column -rename_columns table function_name columns = - old_names = columns.map .name + Name a set of column vectors to be unique when added to a table. Base the + new names on the name of the original column from which they were derived. +build_and_name_columns : Table -> Text -> Text -> Vector (Vector Any) -> Vector Column +build_and_name_columns table original_column_name function_name vectors = + ## Start with copies of the original column name. + old_names = 0.up_to vectors.length . map _-> original_column_name table_column_names = table.columns . map .name unique = Unique_Name_Strategy.new new_names = unique.combine_with_prefix table_column_names old_names (function_name + "_") - columns.map_with_index i-> column-> column.rename (new_names.at i) + vectors.map_with_index i-> vector-> Column.from_vector (new_names.at i) vector ## PRIVATE Return the maximum value of the vector. From 3c47777222a4880e34f06081b5a78d4124f11b95 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 10 Apr 2023 14:16:49 -0400 Subject: [PATCH 24/64] problems, better --- .../src/Internal/Split_Tokenize.enso | 48 +++++++++++-------- .../src/In_Memory/Split_Tokenaize_Spec.enso | 22 +++++++-- .../Table_Tests/src/In_Memory/Table_Spec.enso | 1 - 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index b8d5dbea35fe..331e7ba385c1 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -8,14 +8,14 @@ import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy from project.Data.Type.Value_Type import Auto from Standard.Table import Value_Type -from Standard.Table.Errors import Column_Count_Exceeded +from Standard.Table.Errors import Column_Count_Exceeded, Invalid_Value_Type, Missing_Input_Columns ## PRIVATE Splits a column of text into a set of new columns. See `Table.split_to_columns`. split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = - Value_Type.expect_text (table.at column).value_type <| + expect_text_column table column on_problems <| fan_out_to_columns table column (ignore_nothing (_.split delimiter)) "split" column_count on_problems ## PRIVATE @@ -23,8 +23,8 @@ split_to_columns table column delimiter="," column_count=Auto on_problems=Report See `Table.split_to_rows`. split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows table column delimiter="," on_problems=Report_Error = - _ = on_problems - Value_Type.expect_text (table.at column).value_type <| + _ = [on_problems] + expect_text_column table column on_problems <| fan_out_to_rows table column (ignore_nothing (_.split delimiter)) ## PRIVATE @@ -33,7 +33,7 @@ split_to_rows table column delimiter="," on_problems=Report_Error = See `Table.tokenize_to_columns`. tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns table column pattern case_sensitivity column_count on_problems = - Value_Type.expect_text (table.at column).value_type <| + expect_text_column table column on_problems <| fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) "tokenize" column_count on_problems ## PRIVATE @@ -42,10 +42,25 @@ tokenize_to_columns table column pattern case_sensitivity column_count on_proble See `Table.tokenize_to_rows`. tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = - _ = on_problems - Value_Type.expect_text (table.at column).value_type <| + _ = [on_problems] + expect_text_column table column on_problems <| fan_out_to_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) +expect_text_column : Table -> Text | Integer -> Problem_Behavior -> Any -> Any +expect_text_column table column_id on_problems ~action = + column = table.get column_id + case column of + Nothing -> + problem = Missing_Input_Columns.Error [column_id] + on_problems.attach_problem_after Nothing problem + _ : Column -> + case Value_Type.is_text column.value_type of + False -> + problem = Invalid_Value_Type.Error Value_Type.Char column.value_type column_id + on_problems.attach_problem_after Nothing problem + True -> + action + ## PRIVATE Transform a table by transforming a column into a set of columns. Takes a function that maps a single element of the input column to a vector of output @@ -56,23 +71,18 @@ tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sens - input_column: The column to transform. - function: A function that transforms a single element of `input_column` to multiple values. -fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Text -> Auto | Integer -> Problem_Behavior -> Table +fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Text -> Auto | Integer -> Problem_Behavior -> Table | Nothing fan_out_to_columns table column function function_name column_count=Auto on_problems=Report_Error = - input_column = table.at column + input_column = table.get column new_column_vectors = map_column_vector_to_multiple input_column.to_vector function new_columns = build_and_name_columns table column function_name new_column_vectors num_new_columns = new_columns.length + too_many_columns = column_count != Auto && num_new_columns > column_count new_table = replace_column_with_columns table column new_columns - problem_builder = Problem_Builder.new - - case column_count != Auto && num_new_columns > column_count of - True-> - problem = Column_Count_Exceeded.Error column_count num_new_columns - problem_builder.report_other_warning problem - False -> - Nothing - problem_builder.attach_problems_after on_problems new_table + if too_many_columns.not then new_table else + problem = Column_Count_Exceeded.Error column_count num_new_columns + on_problems.attach_problem_after new_table problem ## PRIVATE Transform a column by applying the given function to the values in the @@ -87,7 +97,7 @@ fan_out_to_columns table column function function_name column_count=Auto on_prob to multiple values. fan_out_to_rows : Table -> Text | Integer -> (Any -> Vector Any) -> Table fan_out_to_rows table column function = - input_column = table.at column + input_column = table.get column # Transform each input value to an output set. output_value_sets = input_column.to_vector.map function diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index beb26e54307c..d19fb1b1c82f 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -5,7 +5,7 @@ import Standard.Base.Errors.Problem_Behavior.Problem_Behavior import Standard.Test.Extensions from Standard.Table import Table, Column -from Standard.Table.Errors import Invalid_Value_Type, Column_Count_Exceeded +from Standard.Table.Errors import Invalid_Value_Type, Column_Count_Exceeded, Missing_Input_Columns from Standard.Test import Test, Test_Suite, Problems spec = @@ -82,7 +82,7 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols action = t.split_to_columns 'bar' 'b' column_count=1 on_problems=_ - tester = _.should_be_a Table + tester = _-> True # _.should_be_a Table problems = [Column_Count_Exceeded.Error 1 3] Problems.test_problem_handling action problems tester @@ -90,10 +90,26 @@ spec = cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols action = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" column_count=1 on_problems=_ - tester = _.should_be_a Table + tester = _-> True # _.should_be_a Table problems = [Column_Count_Exceeded.Error 1 3] Problems.test_problem_handling action problems tester + Test.specify "*_to_columns handles missing input column" <| + cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + t = Table.new cols + action = t.tokenize_to_columns 'invalid_name' "([a-z]).(\d+)" on_problems=_ + tester = _.should_equal Nothing + problems = [Missing_Input_Columns.Error ['invalid_name']] + Problems.test_problem_handling action problems tester + + Test.specify "*_to_rows handles missing input column" <| + cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + t = Table.new cols + action = t.tokenize_to_rows 'invalid_name' "([a-z]).(\d+)" on_problems=_ + tester = _.should_equal Nothing + problems = [Missing_Input_Columns.Error ['invalid_name']] + Problems.test_problem_handling action problems tester + ## Test.specify "???" <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] diff --git a/test/Table_Tests/src/In_Memory/Table_Spec.enso b/test/Table_Tests/src/In_Memory/Table_Spec.enso index 3361aeaaf6dd..47e51e0e328e 100644 --- a/test/Table_Tests/src/In_Memory/Table_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Table_Spec.enso @@ -8,7 +8,6 @@ from Standard.Table import Table, Column, Sort_Column, Column_Selector, Aggregat import Standard.Table.Main as Table_Module from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all hiding First, Last import Standard.Table.Data.Type.Value_Type.Value_Type -from Standard.Table.Errors import Invalid_Output_Column_Names, Duplicate_Output_Column_Names, No_Input_Columns_Selected, Missing_Input_Columns, No_Such_Column, Floating_Point_Equality, Invalid_Value_Type import Standard.Visualization From b439e88bcaa62f9f0d1d1a6cfb11d9afcb269663 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 10 Apr 2023 14:33:39 -0400 Subject: [PATCH 25/64] cleanup --- .../src/Internal/Split_Tokenize.enso | 32 ++++++++++--------- .../src/In_Memory/Split_Tokenaize_Spec.enso | 11 +------ 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 331e7ba385c1..752dcfcc758f 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -46,21 +46,6 @@ tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sens expect_text_column table column on_problems <| fan_out_to_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) -expect_text_column : Table -> Text | Integer -> Problem_Behavior -> Any -> Any -expect_text_column table column_id on_problems ~action = - column = table.get column_id - case column of - Nothing -> - problem = Missing_Input_Columns.Error [column_id] - on_problems.attach_problem_after Nothing problem - _ : Column -> - case Value_Type.is_text column.value_type of - False -> - problem = Invalid_Value_Type.Error Value_Type.Char column.value_type column_id - on_problems.attach_problem_after Nothing problem - True -> - action - ## PRIVATE Transform a table by transforming a column into a set of columns. Takes a function that maps a single element of the input column to a vector of output @@ -202,3 +187,20 @@ ignore_nothing : (Any -> Any) -> (Any -> Any) ignore_nothing function = x-> case x of _ : Nothing -> Nothing _ -> function x + +## PRIVATE + Asserts that a column exists in the table and is a text column. +expect_text_column : Table -> Text | Integer -> Problem_Behavior -> Any -> Any +expect_text_column table column_id on_problems ~action = + column = table.get column_id + case column of + Nothing -> + problem = Missing_Input_Columns.Error [column_id] + on_problems.attach_problem_after Nothing problem + _ : Column -> + case Value_Type.is_text column.value_type of + False -> + problem = Invalid_Value_Type.Error Value_Type.Char column.value_type column_id + on_problems.attach_problem_after Nothing problem + True -> + action diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index d19fb1b1c82f..a75a7c03b0cd 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -90,7 +90,7 @@ spec = cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols action = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" column_count=1 on_problems=_ - tester = _-> True # _.should_be_a Table + tester = _.should_be_a Table problems = [Column_Count_Exceeded.Error 1 3] Problems.test_problem_handling action problems tester @@ -110,15 +110,6 @@ spec = problems = [Missing_Input_Columns.Error ['invalid_name']] Problems.test_problem_handling action problems tester - ## - Test.specify "???" <| - cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] - t = Table.new cols - action = t.split_to_columns 'bar' '' on_problems=_ - tester = _.True # _.should_be_a Table - problems = [Illegal_Argument.Error "The delimiter cannot be empty." Nothing] - Problems.test_problem_handling action problems tester - Test.group "name conflicts" <| Test.specify 'will make column names unique' <| cols = [['split_bar', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] From 896d47a1d2be86a0c6d0777c9896f865dbdb901d Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 10 Apr 2023 16:19:40 -0400 Subject: [PATCH 26/64] tmp --- CHANGELOG.md | 9 + Cargo.lock | 5 +- Cargo.toml | 1 + app/gui/src/config.rs | 74 +- app/gui/src/controller/ide/desktop.rs | 12 +- app/gui/src/ide/initializer.rs | 41 +- app/gui/src/presenter/graph.rs | 4 +- app/gui/src/tests.rs | 5 +- app/ide-desktop/eslint.config.js | 10 +- .../lib/client/electron-builder-config.ts | 10 +- .../lib/client/file-associations.ts | 13 + app/ide-desktop/lib/client/package.json | 2 + .../lib/client/src/config/parser.ts | 9 +- .../lib/client/src/file-associations.ts | 151 ++++ app/ide-desktop/lib/client/src/index.ts | 54 +- app/ide-desktop/lib/client/src/paths.ts | 3 + .../lib/client/src/project-management.ts | 294 ++++++ .../src/authentication/config.ts | 25 +- .../src/authentication/providers/auth.tsx | 15 +- .../src/authentication/service.tsx | 24 +- .../src/authentication/src/config.ts | 12 +- .../authentication/src/dashboard/service.ts | 852 ++++++++++++++++++ .../authentication/src/dashboard/service.tsx | 110 --- .../dashboard/src/authentication/src/http.tsx | 12 - .../src/authentication/src/newtype.ts | 39 + .../dashboard/src/authentication/src/utils.ts | 28 - app/ide-desktop/lib/types/globals.d.ts | 2 +- app/ide-desktop/package-lock.json | 181 +++- app/ide-desktop/utils.ts | 24 + build.sbt | 6 +- build/base/src/fs/wrappers.rs | 19 +- build/build/src/ide/web.rs | 2 +- build/build/src/project/backend.rs | 4 +- build/ci_utils/Cargo.toml | 1 + build/ci_utils/src/fs/wrappers/tokio.rs | 27 +- build/enso-formatter/src/lib.rs | 15 +- distribution/engine/THIRD-PARTY/NOTICE | 11 +- .../LICENSE.txt | 2 +- .../engine/THIRD-PARTY/licenses/MIT-0 | 16 + .../NOTICES | 10 + .../NOTICES | 0 .../org.xerial.sqlite-jdbc-3.36.0.3/NOTICES | 56 -- .../LICENSE | 0 .../LICENSE.zentus | 0 .../org.xerial.sqlite-jdbc-3.41.2.1/NOTICES | 11 + .../lib/Standard/Base/0.0.0-dev/src/Any.enso | 3 +- .../Base/0.0.0-dev/src/Data/Array.enso | 8 +- .../Base/0.0.0-dev/src/Data/Array_Proxy.enso | 11 +- .../0.0.0-dev/src/Data/Filter_Condition.enso | 2 + .../Base/0.0.0-dev/src/Data/Json.enso | 27 +- .../0.0.0-dev/src/Data/Json/Extensions.enso | 30 +- .../Base/0.0.0-dev/src/Data/List.enso | 3 +- .../Base/0.0.0-dev/src/Data/Locale.enso | 14 +- .../Standard/Base/0.0.0-dev/src/Data/Map.enso | 3 +- .../Base/0.0.0-dev/src/Data/Numbers.enso | 4 +- .../Base/0.0.0-dev/src/Data/Ordering.enso | 12 +- .../src/Data/Ordering/Sort_Direction.enso | 3 +- .../Base/0.0.0-dev/src/Data/Regression.enso | 3 +- .../Base/0.0.0-dev/src/Data/Statistics.enso | 3 +- .../Base/0.0.0-dev/src/Data/Text.enso | 2 - .../0.0.0-dev/src/Data/Text/Encoding.enso | 9 +- .../src/Data/Text/Line_Ending_Style.enso | 3 +- .../Base/0.0.0-dev/src/Data/Text/Regex.enso | 6 +- .../0.0.0-dev/src/Data/Text/Regex/Match.enso | 6 +- .../src/Data/Text/Regex/Pattern.enso | 44 +- .../src/Data/Text/Regex/Replacer.enso | 13 +- .../Base/0.0.0-dev/src/Data/Text/Span.enso | 3 +- .../Base/0.0.0-dev/src/Data/Time/Date.enso | 3 +- .../0.0.0-dev/src/Data/Time/Date_Time.enso | 14 +- .../0.0.0-dev/src/Data/Time/Day_Of_Week.enso | 4 + .../0.0.0-dev/src/Data/Time/Duration.enso | 35 +- .../0.0.0-dev/src/Data/Time/Time_Of_Day.enso | 3 +- .../0.0.0-dev/src/Data/Time/Time_Zone.enso | 3 +- .../Base/0.0.0-dev/src/Data/Vector.enso | 33 +- .../Standard/Base/0.0.0-dev/src/Error.enso | 11 +- .../Base/0.0.0-dev/src/Errors/Common.enso | 26 +- .../Base/0.0.0-dev/src/Errors/File_Error.enso | 3 +- .../src/Errors/Problem_Behavior.enso | 23 +- .../0.0.0-dev/src/Errors/Unimplemented.enso | 5 +- .../lib/Standard/Base/0.0.0-dev/src/IO.enso | 12 +- .../lib/Standard/Base/0.0.0-dev/src/Main.enso | 2 + .../lib/Standard/Base/0.0.0-dev/src/Meta.enso | 73 +- .../Base/0.0.0-dev/src/Meta/Enso_Project.enso | 3 +- .../Base/0.0.0-dev/src/Network/HTTP.enso | 10 +- .../src/Network/HTTP/HTTP_Method.enso | 4 +- .../0.0.0-dev/src/Network/HTTP/Header.enso | 15 +- .../0.0.0-dev/src/Network/HTTP/Response.enso | 3 +- .../src/Network/HTTP/Response_Body.enso | 15 +- .../Base/0.0.0-dev/src/Network/URI.enso | 33 +- .../Standard/Base/0.0.0-dev/src/Panic.enso | 24 +- .../Standard/Base/0.0.0-dev/src/Polyglot.enso | 74 +- .../Standard/Base/0.0.0-dev/src/Runtime.enso | 142 ++- .../Base/0.0.0-dev/src/Runtime/Debug.enso | 4 +- .../Base/0.0.0-dev/src/Runtime/Lazy.enso | 75 -- .../src/Runtime/Managed_Resource.enso | 15 +- .../Base/0.0.0-dev/src/Runtime/Ref.enso | 8 +- .../src/Runtime/Source_Location.enso | 30 +- .../Base/0.0.0-dev/src/Runtime/State.enso | 12 +- .../Base/0.0.0-dev/src/Runtime/Thread.enso | 4 +- .../Standard/Base/0.0.0-dev/src/System.enso | 8 +- .../Base/0.0.0-dev/src/System/File.enso | 69 +- .../src/System/File/File_Permissions.enso | 11 +- .../0.0.0-dev/src/System/File_Format.enso | 20 +- .../Base/0.0.0-dev/src/System/Platform.enso | 2 - .../Base/0.0.0-dev/src/System/Process.enso | 4 +- .../src/System/Process/Exit_Code.enso | 4 +- .../src/System/Process/Process_Builder.enso | 16 +- .../Standard/Base/0.0.0-dev/src/Warning.enso | 38 +- .../Database/0.0.0-dev/THIRD-PARTY/NOTICE | 2 +- .../org.xerial.sqlite-jdbc-3.36.0.3/NOTICES | 60 -- .../LICENSE | 0 .../LICENSE.zentus | 0 .../org.xerial.sqlite-jdbc-3.41.2.1/NOTICES | 11 + .../0.0.0-dev/src/Connection/Connection.enso | 50 +- .../src/Connection/Connection_Options.enso | 5 +- .../0.0.0-dev/src/Connection/Credentials.enso | 3 +- .../src/Connection/Postgres_Options.enso | 9 +- .../src/Connection/Redshift_Options.enso | 9 +- .../src/Connection/SQLite_Format.enso | 9 +- .../src/Connection/SQLite_Options.enso | 9 +- .../Database/0.0.0-dev/src/Data/Column.enso | 117 ++- .../Database/0.0.0-dev/src/Data/Dialect.enso | 71 +- .../Database/0.0.0-dev/src/Data/SQL.enso | 43 +- .../0.0.0-dev/src/Data/SQL_Statement.enso | 20 +- .../Database/0.0.0-dev/src/Data/SQL_Type.enso | 18 +- .../Database/0.0.0-dev/src/Data/Table.enso | 171 +++- .../Database/0.0.0-dev/src/Errors.enso | 45 +- .../src/Internal/Aggregate_Helper.enso | 4 +- .../src/Internal/Base_Generator.enso | 18 +- .../Internal/Common/Database_Join_Helper.enso | 10 +- .../0.0.0-dev/src/Internal/IR/From_Spec.enso | 17 + .../src/Internal/IR/SQL_Expression.enso | 6 + .../src/Internal/JDBC_Connection.enso | 7 +- .../Internal/Postgres/Postgres_Dialect.enso | 42 +- .../Postgres/Postgres_Type_Mapping.enso | 24 +- .../Internal/Redshift/Redshift_Dialect.enso | 33 +- .../src/Internal/SQL_Type_Mapping.enso | 30 +- .../src/Internal/SQL_Type_Reference.enso | 14 +- .../src/Internal/SQLite/SQLite_Dialect.enso | 45 +- .../Internal/SQLite/SQLite_Type_Mapping.enso | 16 +- .../Image/0.0.0-dev/src/Data/Histogram.enso | 6 +- .../Image/0.0.0-dev/src/Data/Image.enso | 53 +- .../Image/0.0.0-dev/src/Data/Matrix.enso | 54 +- .../0.0.0-dev/src/Data/Matrix_Error.enso | 12 +- .../0.0.0-dev/src/Image_File_Format.enso | 9 +- .../lib/Standard/Table/0.0.0-dev/package.yaml | 1 + .../Table/0.0.0-dev/src/Data/Column.enso | 57 +- .../0.0.0-dev/src/Data/Data_Formatter.enso | 60 +- .../Table/0.0.0-dev/src/Data/Expression.enso | 16 +- .../Table/0.0.0-dev/src/Data/Row.enso | 3 +- .../Table/0.0.0-dev/src/Data/Table.enso | 46 +- .../0.0.0-dev/src/Data/Type/Enso_Types.enso | 2 +- .../0.0.0-dev/src/Data/Type/Value_Type.enso | 26 +- .../src/Data/Type/Value_Type_Helpers.enso | 26 +- .../src/Delimited/Delimited_Format.enso | 16 +- .../Standard/Table/0.0.0-dev/src/Errors.enso | 149 ++- .../0.0.0-dev/src/Excel/Excel_Format.enso | 16 +- .../0.0.0-dev/src/Excel/Excel_Range.enso | 18 +- .../src/Internal/Parse_Values_Helper.enso | 6 +- .../src/Internal/Problem_Builder.enso | 15 +- .../0.0.0-dev/src/Internal/Rows_View.enso | 1 + .../src/Internal/Unique_Name_Strategy.enso | 38 +- .../src/Internal/Widget_Helpers.enso | 6 +- .../Standard/Test/0.0.0-dev/src/Problems.enso | 6 +- .../Visualization/0.0.0-dev/src/Helpers.enso | 55 +- .../0.0.0-dev/src/Histogram.enso | 1 - .../Visualization/0.0.0-dev/src/Id.enso | 49 +- .../protocol-language-server.md | 8 + .../src/main/resources/application.conf | 3 +- .../search/SuggestionsHandler.scala | 45 +- .../search/SuggestionsHandlerSpec.scala | 52 +- .../json/SuggestionsHandlerEventsTest.scala | 146 +-- .../scala/org/enso/polyglot/Suggestion.scala | 47 +- .../org/enso/runner/ContextFactory.scala | 5 + .../src/main/scala/org/enso/runner/Main.scala | 20 +- .../test/instrument/BuiltinTypesTest.scala | 4 +- .../test/instrument/RuntimeServerTest.scala | 2 +- .../benchmarks/semantic/ListBenchmarks.java | 39 +- .../org/enso/interpreter/EnsoLanguage.java | 15 +- .../permission/PermissionGuardNode.java | 53 -- .../builtin/error/ForbiddenOperation.java | 6 + .../builtin/error/Unimplemented.java | 20 + .../expression/builtin/runtime/Context.java | 27 + .../builtin/runtime/ContextIsEnabledNode.java | 32 + ...untimeCurrentExecutionEnvironmentNode.java | 17 + ...va => RuntimeWithDisabledContextNode.java} | 11 +- ...ava => RuntimeWithEnabledContextNode.java} | 11 +- .../builtin/text/AnyToDisplayTextNode.java | 19 + .../enso/interpreter/runtime/EnsoContext.java | 10 +- .../interpreter/runtime/builtin/Builtins.java | 10 + .../interpreter/runtime/builtin/Error.java | 7 + .../callable/atom/AtomConstructor.java | 2 +- .../callable/atom/unboxing/Layout.java | 38 +- .../unboxing/SuspendedFieldGetterNode.java | 65 ++ .../runtime/state/ExecutionEnvironment.java | 110 +++ .../runtime/state/IOPermissions.java | 49 - .../enso/interpreter/runtime/state/State.java | 31 +- .../enso/compiler/SerializationManager.scala | 1 + .../enso/compiler/codegen/IrToTruffle.scala | 8 +- .../job/AnalyzeModuleInScopeJob.scala | 13 +- .../interpreter/test/LazyAtomFieldTest.java | 135 +++ .../interpreter/test/ValuesGenerator.java | 1 + .../DesignExecutionEnvironmentTest.scala | 117 +++ .../test/semantic/IOContextTest.scala | 92 -- .../LiveExecutionEnvironmentTest.scala | 102 +++ lib/rust/bitmap/src/lib.rs | 4 +- .../text/src/font/msdf/src/texture.rs | 8 +- .../src/display/shape/compound/rectangle.rs | 7 +- lib/rust/ensogl/core/src/gui/component.rs | 1 - .../examples/instance-ordering/src/lib.rs | 2 +- .../common_utils/Core_Text_Utils.java | 14 + .../sql/SuggestionsRepoBenchmark.java | 9 +- .../enso/searcher/sql/SuggestionRandom.scala | 6 - .../org/enso/searcher/SuggestionsRepo.scala | 29 +- .../searcher/sql/SqlSuggestionsRepo.scala | 426 +++------ .../scala/org/enso/searcher/sql/Tables.scala | 143 +-- .../src/test/resources/logback-test.xml | 7 +- .../searcher/sql/SuggestionsRepoTest.scala | 401 ++------- ...estionEntryEqualityIgnoringArguments.scala | 18 + .../SuggestionEqualityIgnoringArguments.scala | 20 + ...stionOptionEqualityIgnoringArguments.scala | 19 + .../sql/equality/SuggestionsEquality.scala | 19 + .../main/java/org/enso/base/Text_Utils.java | 8 +- .../table/parsing/TypeInferringParser.java | 9 +- .../Common_Table_Operations/Cast_Spec.enso | 93 ++ .../Column_Operations_Spec.enso | 6 +- .../Join/Join_Spec.enso | 62 +- .../Join/Union_Spec.enso | 187 ++-- .../src/Common_Table_Operations/Main.enso | 9 +- .../src/Database/Postgres_Spec.enso | 2 +- .../Table_Tests/src/Database/SQLite_Spec.enso | 2 +- .../Types/Postgres_Type_Mapping_Spec.enso | 4 +- .../src/Formatting/Data_Formatter_Spec.enso | 21 + .../src/Formatting/Parse_Values_Spec.enso | 241 ++--- .../src/Helpers/Value_Type_Spec.enso | 37 +- test/Tests/src/Data/Text/Utils_Spec.enso | 23 + test/Tests/src/Data/Time/Duration_Spec.enso | 4 - test/Tests/src/Runtime/Lazy_Spec.enso | 6 +- test/Tests/src/Semantic/Runtime_Spec.enso | 32 +- test/Visualization_Tests/src/Table_Spec.enso | 2 +- .../src/Widgets/Database_Widgets_Spec.enso | 2 +- .../Base/0.0.0-dev/src/Errors/Common.enso | 4 + .../Standard/Base/0.0.0-dev/src/Function.enso | 2 + .../Standard/Base/0.0.0-dev/src/Runtime.enso | 42 +- .../copyright-add | 31 - .../copyright-ignore | 4 - .../copyright-keep | 6 - .../copyright-keep-context | 1 - .../copyright-ignore} | 1 - .../copyright-keep} | 3 + .../custom-license | 0 .../files-keep | 2 +- tools/legal-review/Database/report-state | 4 +- .../custom-license | 0 .../files-add/LICENSE.txt | 2 +- .../copyright-keep-context | 1 + .../copyright-add | 0 .../copyright-keep-context | 0 .../copyright-add | 15 - .../copyright-keep-context | 2 - .../copyright-ignore | 3 + .../copyright-keep | 6 + .../custom-license | 0 .../files-keep | 2 +- tools/legal-review/engine/report-state | 4 +- .../engine/reviewed-licenses/MIT-0 | 1 + tools/legal-review/license-texts/MIT-0 | 16 + .../engine-benchmarks/bench_download.py | 127 ++- 268 files changed, 5439 insertions(+), 3067 deletions(-) create mode 100644 app/ide-desktop/lib/client/file-associations.ts create mode 100644 app/ide-desktop/lib/client/src/file-associations.ts create mode 100644 app/ide-desktop/lib/client/src/project-management.ts create mode 100644 app/ide-desktop/lib/dashboard/src/authentication/src/dashboard/service.ts delete mode 100644 app/ide-desktop/lib/dashboard/src/authentication/src/dashboard/service.tsx create mode 100644 app/ide-desktop/lib/dashboard/src/authentication/src/newtype.ts rename distribution/engine/THIRD-PARTY/{com.typesafe.slick.slick_2.13-3.3.3 => com.typesafe.slick.slick_2.13-3.4.1}/LICENSE.txt (97%) create mode 100644 distribution/engine/THIRD-PARTY/licenses/MIT-0 create mode 100644 distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.4/NOTICES rename distribution/engine/THIRD-PARTY/{org.scala-lang.modules.scala-collection-compat_2.13-2.0.0 => org.scala-lang.modules.scala-collection-compat_2.13-2.8.1}/NOTICES (100%) delete mode 100644 distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/NOTICES rename distribution/engine/THIRD-PARTY/{org.xerial.sqlite-jdbc-3.36.0.3 => org.xerial.sqlite-jdbc-3.41.2.1}/LICENSE (100%) rename distribution/engine/THIRD-PARTY/{org.xerial.sqlite-jdbc-3.36.0.3 => org.xerial.sqlite-jdbc-3.41.2.1}/LICENSE.zentus (100%) create mode 100644 distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/NOTICES delete mode 100644 distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Lazy.enso delete mode 100644 distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/NOTICES rename distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/{org.xerial.sqlite-jdbc-3.36.0.3 => org.xerial.sqlite-jdbc-3.41.2.1}/LICENSE (100%) rename distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/{org.xerial.sqlite-jdbc-3.36.0.3 => org.xerial.sqlite-jdbc-3.41.2.1}/LICENSE.zentus (100%) create mode 100644 distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/NOTICES delete mode 100644 engine/runtime/src/main/java/org/enso/interpreter/node/controlflow/permission/PermissionGuardNode.java create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/error/Unimplemented.java create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/Context.java create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/ContextIsEnabledNode.java create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeCurrentExecutionEnvironmentNode.java rename engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/{AllowOutputInNode.java => RuntimeWithDisabledContextNode.java} (65%) rename engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/{AllowInputInNode.java => RuntimeWithEnabledContextNode.java} (65%) create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/unboxing/SuspendedFieldGetterNode.java create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/runtime/state/ExecutionEnvironment.java delete mode 100644 engine/runtime/src/main/java/org/enso/interpreter/runtime/state/IOPermissions.java create mode 100644 engine/runtime/src/test/java/org/enso/interpreter/test/LazyAtomFieldTest.java create mode 100644 engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/DesignExecutionEnvironmentTest.scala delete mode 100644 engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/IOContextTest.scala create mode 100644 engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/LiveExecutionEnvironmentTest.scala create mode 100644 lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionEntryEqualityIgnoringArguments.scala create mode 100644 lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionEqualityIgnoringArguments.scala create mode 100644 lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionOptionEqualityIgnoringArguments.scala create mode 100644 lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionsEquality.scala create mode 100644 test/Table_Tests/src/Common_Table_Operations/Cast_Spec.enso create mode 100644 test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Function.enso delete mode 100644 tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-add delete mode 100644 tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-ignore delete mode 100644 tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep delete mode 100644 tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep-context rename tools/legal-review/{engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep => Database/org.xerial.sqlite-jdbc-3.41.2.1/copyright-ignore} (55%) rename tools/legal-review/{engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-ignore => Database/org.xerial.sqlite-jdbc-3.41.2.1/copyright-keep} (59%) rename tools/legal-review/Database/{org.xerial.sqlite-jdbc-3.36.0.3 => org.xerial.sqlite-jdbc-3.41.2.1}/custom-license (100%) rename tools/legal-review/Database/{org.xerial.sqlite-jdbc-3.36.0.3 => org.xerial.sqlite-jdbc-3.41.2.1}/files-keep (100%) rename tools/legal-review/engine/{com.typesafe.slick.slick_2.13-3.3.3 => com.typesafe.slick.slick_2.13-3.4.1}/custom-license (100%) rename tools/legal-review/engine/{com.typesafe.slick.slick_2.13-3.3.3 => com.typesafe.slick.slick_2.13-3.4.1}/files-add/LICENSE.txt (97%) create mode 100644 tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.4/copyright-keep-context rename tools/legal-review/engine/{org.scala-lang.modules.scala-collection-compat_2.13-2.0.0 => org.scala-lang.modules.scala-collection-compat_2.13-2.8.1}/copyright-add (100%) rename tools/legal-review/engine/{org.scala-lang.modules.scala-collection-compat_2.13-2.0.0 => org.scala-lang.modules.scala-collection-compat_2.13-2.8.1}/copyright-keep-context (100%) delete mode 100644 tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-add delete mode 100644 tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep-context create mode 100644 tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/copyright-ignore create mode 100644 tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/copyright-keep rename tools/legal-review/engine/{org.xerial.sqlite-jdbc-3.36.0.3 => org.xerial.sqlite-jdbc-3.41.2.1}/custom-license (100%) rename tools/legal-review/engine/{org.xerial.sqlite-jdbc-3.36.0.3 => org.xerial.sqlite-jdbc-3.41.2.1}/files-keep (100%) create mode 100644 tools/legal-review/engine/reviewed-licenses/MIT-0 create mode 100644 tools/legal-review/license-texts/MIT-0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 715210590afa..cbbe085dbe63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,6 +128,8 @@ eliminating the need for fully qualified names. - [Added tooltips to icon buttons][6035] for improved usability. Users can now quickly understand each button's function. +- [File associations are created on Windows and macOS][6077]. This allows + opening Enso files by double-clicking them in the file explorer. #### EnsoGL (rendering engine) @@ -375,6 +377,7 @@ - [Added support for Date/Time columns in the Postgres backend and added `year`/`month`/`day` operations to Table columns.][6153] - [`Text.split` can now take a vector of delimiters.][6156] +- [Implemented `Table.union` for the Database backend.][6204] [debug-shortcuts]: https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug @@ -568,6 +571,8 @@ [6150]: https://github.com/enso-org/enso/pull/6150 [6153]: https://github.com/enso-org/enso/pull/6153 [6156]: https://github.com/enso-org/enso/pull/6156 +[6204]: https://github.com/enso-org/enso/pull/6204 +[6077]: https://github.com/enso-org/enso/pull/6077 #### Enso Compiler @@ -670,6 +675,8 @@ - [Don't install Python component on Windows][5900] - [Detect potential name conflicts between exported types and FQNs][5966] - [Ensure calls involving warnings remain instrumented][6067] +- [One can define lazy atom fields][6151] +- [Replace IOContexts with Execution Environment and generic Context][6171] [3227]: https://github.com/enso-org/enso/pull/3227 [3248]: https://github.com/enso-org/enso/pull/3248 @@ -773,6 +780,8 @@ [5900]: https://github.com/enso-org/enso/pull/5900 [5966]: https://github.com/enso-org/enso/pull/5966 [6067]: https://github.com/enso-org/enso/pull/6067 +[6151]: https://github.com/enso-org/enso/pull/6151 +[6171]: https://github.com/enso-org/enso/pull/6171 # Enso 2.0.0-alpha.18 (2021-10-12) diff --git a/Cargo.lock b/Cargo.lock index e00746f594a2..dc76105ab303 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4237,6 +4237,7 @@ dependencies = [ "tar", "tempfile", "tokio", + "tokio-stream", "tokio-util", "tracing", "tracing-subscriber", @@ -7105,9 +7106,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite", diff --git a/Cargo.toml b/Cargo.toml index 95d7563700a2..80b1d0a9c9a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,7 @@ regex = { version = "1.6.0" } serde_yaml = { version = "0.9.16" } serde-wasm-bindgen = { version = "0.4.5" } tokio = { version = "1.23.0", features = ["full", "tracing"] } +tokio-stream = { version = "0.1.12", features = ["fs"] } tokio-util = { version = "0.7.4", features = ["full"] } wasm-bindgen = { version = "0.2.83", features = ["serde-serialize"] } wasm-bindgen-test = { version = "0.3.33" } diff --git a/app/gui/src/config.rs b/app/gui/src/config.rs index 5bd5cd4c5426..b3bfaccda853 100644 --- a/app/gui/src/config.rs +++ b/app/gui/src/config.rs @@ -4,9 +4,11 @@ use crate::prelude::*; use crate::constants; +use engine_protocol::project_manager::ProjectMetadata; use engine_protocol::project_manager::ProjectName; use enso_config::Args; use enso_config::ARGS; +use failure::ResultExt; @@ -110,26 +112,27 @@ impl BackendService { #[derive(Clone, Debug, Default)] pub struct Startup { /// The configuration of connection to the backend service. - pub backend: BackendService, + pub backend: BackendService, /// The project name we want to open on startup. - pub project_name: Option, + pub project_to_open: Option, /// Whether to open directly to the project view, skipping the welcome screen. - pub initial_view: InitialView, + pub initial_view: InitialView, /// Identifies the element to create the IDE's DOM nodes as children of. - pub dom_parent_id: Option, + pub dom_parent_id: Option, } impl Startup { /// Read configuration from the web arguments. See also [`web::Arguments`] documentation. pub fn from_web_arguments() -> FallibleResult { let backend = BackendService::from_web_arguments(&ARGS)?; - let project_name = ARGS.groups.startup.options.project.value.as_str(); - let no_project_name = project_name.is_empty(); + let project = ARGS.groups.startup.options.project.value.as_str(); + let no_project: bool = project.is_empty(); let initial_view = - if no_project_name { InitialView::WelcomeScreen } else { InitialView::Project }; - let project_name = (!no_project_name).as_some_from(|| project_name.to_owned().into()); + if no_project { InitialView::WelcomeScreen } else { InitialView::Project }; + let project_to_open = + (!no_project).as_some_from(|| ProjectToOpen::from_str(project)).transpose()?; let dom_parent_id = None; - Ok(Startup { backend, project_name, initial_view, dom_parent_id }) + Ok(Startup { backend, project_to_open, initial_view, dom_parent_id }) } /// Identifies the element to create the IDE's DOM nodes as children of. @@ -155,3 +158,56 @@ pub enum InitialView { /// Start to the Project View. Project, } + + +// === ProjectToOpen === + +/// The project to open on startup. +/// +/// We both support opening a project by name and by ID. This is because: +/// * names are more human-readable, but they are not guaranteed to be unique; +/// * IDs are guaranteed to be unique, but they are not human-readable. +#[derive(Clone, Debug)] +pub enum ProjectToOpen { + /// Open the project with the given name. + Name(ProjectName), + /// Open the project with the given ID. + Id(Uuid), +} + +impl ProjectToOpen { + /// Check if provided project metadata matches the requested project. + pub fn matches(&self, project_metadata: &ProjectMetadata) -> bool { + match self { + ProjectToOpen::Name(name) => name == &project_metadata.name, + ProjectToOpen::Id(id) => id == &project_metadata.id, + } + } +} + +impl FromStr for ProjectToOpen { + type Err = failure::Error; + + fn from_str(s: &str) -> Result { + // While in theory it is possible that Uuid string representation is a valid project name, + // in practice it is very unlikely. Additionally, Uuid representation used by us is + // hyphenated, which will never be the case for project name. This, we can use this as a + // heuristic to determine whether the provided string is a project name or a project ID. + if s.contains('-') { + let id = Uuid::from_str(s) + .context(format!("Failed to parse project ID from string: '{s}'."))?; + Ok(ProjectToOpen::Id(id)) + } else { + Ok(ProjectToOpen::Name(ProjectName::new_unchecked(s))) + } + } +} + +impl Display for ProjectToOpen { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ProjectToOpen::Name(name) => write!(f, "{name}"), + ProjectToOpen::Id(id) => write!(f, "{id}"), + } + } +} diff --git a/app/gui/src/controller/ide/desktop.rs b/app/gui/src/controller/ide/desktop.rs index 6f9cb48ec6a3..c84598e88a7f 100644 --- a/app/gui/src/controller/ide/desktop.rs +++ b/app/gui/src/controller/ide/desktop.rs @@ -4,6 +4,7 @@ use crate::prelude::*; +use crate::config::ProjectToOpen; use crate::controller::ide::ManagingProjectAPI; use crate::controller::ide::Notification; use crate::controller::ide::StatusNotificationPublisher; @@ -53,10 +54,11 @@ impl Handle { /// Screen. pub async fn new( project_manager: Rc, - project_name: Option, + project_to_open: Option, ) -> FallibleResult { - let project = match project_name { - Some(name) => Some(Self::init_project_model(project_manager.clone_ref(), name).await?), + let project = match project_to_open { + Some(project_to_open) => + Some(Self::init_project_model(project_manager.clone_ref(), project_to_open).await?), None => None, }; Ok(Self::new_with_project_model(project_manager, project)) @@ -86,12 +88,12 @@ impl Handle { /// Open project with provided name. async fn init_project_model( project_manager: Rc, - project_name: ProjectName, + project_to_open: ProjectToOpen, ) -> FallibleResult { // TODO[ao]: Reuse of initializer used in previous code design. It should be soon replaced // anyway, because we will soon resign from the "open or create" approach when opening // IDE. See https://github.com/enso-org/ide/issues/1492 for details. - let initializer = initializer::WithProjectManager::new(project_manager, project_name); + let initializer = initializer::WithProjectManager::new(project_manager, project_to_open); let model = initializer.initialize_project_model().await?; Ok(model) } diff --git a/app/gui/src/ide/initializer.rs b/app/gui/src/ide/initializer.rs index 9ed4c45e8d49..9251c4d3d8aa 100644 --- a/app/gui/src/ide/initializer.rs +++ b/app/gui/src/ide/initializer.rs @@ -3,6 +3,7 @@ use crate::prelude::*; use crate::config; +use crate::config::ProjectToOpen; use crate::ide::Ide; use crate::transport::web::WebSocket; use crate::FailedIde; @@ -40,9 +41,9 @@ const INITIALIZATION_RETRY_TIMES: &[Duration] = /// Error raised when project with given name was not found. #[derive(Clone, Debug, Fail)] -#[fail(display = "Project with the name {} was not found.", name)] +#[fail(display = "Project '{}' was not found.", name)] pub struct ProjectNotFound { - name: ProjectName, + name: ProjectToOpen, } @@ -129,8 +130,8 @@ impl Initializer { match &self.config.backend { ProjectManager { endpoint } => { let project_manager = self.setup_project_manager(endpoint).await?; - let project_name = self.config.project_name.clone(); - let controller = controller::ide::Desktop::new(project_manager, project_name); + let project_to_open = self.config.project_to_open.clone(); + let controller = controller::ide::Desktop::new(project_manager, project_to_open); Ok(Rc::new(controller.await?)) } LanguageServer { json_endpoint, binary_endpoint, namespace, project_name } => { @@ -186,13 +187,16 @@ impl Initializer { pub struct WithProjectManager { #[derivative(Debug = "ignore")] pub project_manager: Rc, - pub project_name: ProjectName, + pub project_to_open: ProjectToOpen, } impl WithProjectManager { /// Constructor. - pub fn new(project_manager: Rc, project_name: ProjectName) -> Self { - Self { project_manager, project_name } + pub fn new( + project_manager: Rc, + project_to_open: ProjectToOpen, + ) -> Self { + Self { project_manager, project_to_open } } /// Create and initialize a new Project Model, for a project with name passed in constructor. @@ -205,13 +209,12 @@ impl WithProjectManager { } /// Creates a new project and returns its id, so the newly connected project can be opened. - pub async fn create_project(&self) -> FallibleResult { + pub async fn create_project(&self, project_name: &ProjectName) -> FallibleResult { use project_manager::MissingComponentAction::Install; - info!("Creating a new project named '{}'.", self.project_name); + info!("Creating a new project named '{}'.", project_name); let version = &enso_config::ARGS.groups.engine.options.preferred_version.value; let version = (!version.is_empty()).as_some_from(|| version.clone()); - let name = &self.project_name; - let response = self.project_manager.create_project(name, &None, &version, &Install); + let response = self.project_manager.create_project(project_name, &None, &version, &Install); Ok(response.await?.project_id) } @@ -219,9 +222,9 @@ impl WithProjectManager { let response = self.project_manager.list_projects(&None).await?; let mut projects = response.projects.iter(); projects - .find(|project_metadata| project_metadata.name == self.project_name) + .find(|project_metadata| self.project_to_open.matches(project_metadata)) .map(|md| md.id) - .ok_or_else(|| ProjectNotFound { name: self.project_name.clone() }.into()) + .ok_or_else(|| ProjectNotFound { name: self.project_to_open.clone() }.into()) } /// Look for the project with the name specified when constructing this initializer, @@ -230,9 +233,14 @@ impl WithProjectManager { let project = self.lookup_project().await; if let Ok(project_id) = project { Ok(project_id) + } else if let ProjectToOpen::Name(name) = &self.project_to_open { + info!("Attempting to create {}", name); + self.create_project(name).await } else { - info!("Attempting to create {}", self.project_name); - self.create_project().await + // This can happen only if we are told to open project by id but it cannot be found. + // We cannot fallback to creating a new project in this case, as we cannot create a + // project with a given id. Thus, we simply propagate the lookup result. + project } } } @@ -305,7 +313,8 @@ mod test { expect_call!(mock_client.list_projects(count) => Ok(project_lists)); let project_manager = Rc::new(mock_client); - let initializer = WithProjectManager { project_manager, project_name }; + let project_to_open = ProjectToOpen::Name(project_name); + let initializer = WithProjectManager { project_manager, project_to_open }; let project = initializer.get_project_or_create_new().await; assert_eq!(expected_id, project.expect("Couldn't get project.")) } diff --git a/app/gui/src/presenter/graph.rs b/app/gui/src/presenter/graph.rs index a897fffbce86..ef483952fb26 100644 --- a/app/gui/src/presenter/graph.rs +++ b/app/gui/src/presenter/graph.rs @@ -2,8 +2,6 @@ //! about presenters in general. use crate::prelude::*; -use double_representation::context_switch::Context; -use double_representation::context_switch::ContextSwitch; use enso_web::traits::*; use crate::controller::graph::widget::Request as WidgetRequest; @@ -11,6 +9,8 @@ use crate::controller::upload::NodeFromDroppedFileHandler; use crate::executor::global::spawn_stream_handler; use crate::presenter::graph::state::State; +use double_representation::context_switch::Context; +use double_representation::context_switch::ContextSwitch; use double_representation::context_switch::ContextSwitchExpression; use engine_protocol::language_server::SuggestionId; use enso_frp as frp; diff --git a/app/gui/src/tests.rs b/app/gui/src/tests.rs index 8dcf7ab9193d..31f4de972528 100644 --- a/app/gui/src/tests.rs +++ b/app/gui/src/tests.rs @@ -1,5 +1,6 @@ use super::prelude::*; +use crate::config::ProjectToOpen; use crate::ide; use crate::transport::test_utils::TestWithMockedTransport; @@ -28,7 +29,9 @@ fn failure_to_open_project_is_reported() { let project_manager = Rc::new(project_manager::Client::new(transport)); executor::global::spawn(project_manager.runner()); let name = ProjectName::new_unchecked(crate::constants::DEFAULT_PROJECT_NAME.to_owned()); - let initializer = ide::initializer::WithProjectManager::new(project_manager, name); + let project_to_open = ProjectToOpen::Name(name); + let initializer = + ide::initializer::WithProjectManager::new(project_manager, project_to_open); let result = initializer.initialize_project_model().await; result.expect_err("Error should have been reported."); }); diff --git a/app/ide-desktop/eslint.config.js b/app/ide-desktop/eslint.config.js index 14971bbde4d4..1972a77805d3 100644 --- a/app/ide-desktop/eslint.config.js +++ b/app/ide-desktop/eslint.config.js @@ -28,7 +28,7 @@ const DEFAULT_IMPORT_ONLY_MODULES = const ALLOWED_DEFAULT_IMPORT_MODULES = `${DEFAULT_IMPORT_ONLY_MODULES}|postcss|react-hot-toast` const OUR_MODULES = 'enso-content-config|enso-common' const RELATIVE_MODULES = - 'bin\\u002Fproject-manager|bin\\u002Fserver|config\\u002Fparser|authentication|config|debug|index|ipc|naming|paths|preload|security' + 'bin\\u002Fproject-manager|bin\\u002Fserver|config\\u002Fparser|authentication|config|debug|file-associations|index|ipc|naming|paths|preload|security' const STRING_LITERAL = ':matches(Literal[raw=/^["\']/], TemplateLiteral)' const JSX = ':matches(JSXElement, JSXFragment)' const NOT_PASCAL_CASE = '/^(?!_?([A-Z][a-z0-9]*)+$)/' @@ -102,7 +102,7 @@ const RESTRICTED_SYNTAXES = [ }, { selector: - ':not(:matches(FunctionDeclaration, FunctionExpression, ArrowFunctionExpression, SwitchStatement, SwitchCase, IfStatement:has(.consequent > :matches(ReturnStatement, ThrowStatement)):has(.alternate :matches(ReturnStatement, ThrowStatement)), TryStatement:has(.block > :matches(ReturnStatement, ThrowStatement)):has(:matches([handler=null], .handler :matches(ReturnStatement, ThrowStatement))):has(:matches([finalizer=null], .finalizer :matches(ReturnStatement, ThrowStatement))))) > * > ReturnStatement', + ':not(:matches(FunctionDeclaration, FunctionExpression, ArrowFunctionExpression, SwitchStatement, SwitchCase, IfStatement:has(.consequent > :matches(ReturnStatement, ThrowStatement)):has(.alternate :matches(ReturnStatement, ThrowStatement)), TryStatement:has(.block > :matches(ReturnStatement, ThrowStatement)):has(:matches([handler=null], .handler :matches(ReturnStatement, ThrowStatement))):has(:matches([finalizer=null], .finalizer :matches(ReturnStatement, ThrowStatement))))) > * > :matches(ReturnStatement, ThrowStatement)', message: 'No early returns', }, { @@ -166,6 +166,10 @@ const RESTRICTED_SYNTAXES = [ 'ImportDeclaration[source.value=/^(?:assert|async_hooks|buffer|child_process|cluster|console|constants|crypto|dgram|diagnostics_channel|dns|domain|events|fs|fs\\u002Fpromises|http|http2|https|inspector|module|net|os|path|perf_hooks|process|punycode|querystring|readline|repl|stream|string_decoder|timers|tls|trace_events|tty|url|util|v8|vm|wasi|worker_threads|zlib)$/]', message: 'Use `node:` prefix to import builtin node modules', }, + { + selector: 'TSEnumDeclaration:not(:has(TSEnumMember))', + message: 'Enums must not be empty', + }, { selector: 'ImportDeclaration[source.value=/^(?!node:)/] ~ ImportDeclaration[source.value=/^node:/]', @@ -310,7 +314,7 @@ export default [ ], 'no-redeclare': 'off', // Important to warn on accidental duplicated `interface`s e.g. when writing API wrappers. - '@typescript-eslint/no-redeclare': 'error', + '@typescript-eslint/no-redeclare': ['error', { ignoreDeclarationMerge: false }], 'no-shadow': 'off', '@typescript-eslint/no-shadow': 'warn', 'no-unused-expressions': 'off', diff --git a/app/ide-desktop/lib/client/electron-builder-config.ts b/app/ide-desktop/lib/client/electron-builder-config.ts index f68262c27332..ddbee125f117 100644 --- a/app/ide-desktop/lib/client/electron-builder-config.ts +++ b/app/ide-desktop/lib/client/electron-builder-config.ts @@ -19,6 +19,7 @@ import * as common from 'enso-common' import * as paths from './paths.js' import signArchivesMacOs from './tasks/signArchivesMacOs.js' +import { BUNDLED_PROJECT_EXTENSION, SOURCE_FILE_EXTENSION } from './file-associations.js' import BUILD_INFO from '../../build.json' assert { type: 'json' } @@ -156,8 +157,13 @@ export function createElectronBuilderConfig(passedArgs: Arguments): electronBuil ], fileAssociations: [ { - ext: 'enso', - name: 'Enso Source File', + ext: SOURCE_FILE_EXTENSION, + name: `${common.PRODUCT_NAME} Source File`, + role: 'Editor', + }, + { + ext: BUNDLED_PROJECT_EXTENSION, + name: `${common.PRODUCT_NAME} Project Bundle`, role: 'Editor', }, ], diff --git a/app/ide-desktop/lib/client/file-associations.ts b/app/ide-desktop/lib/client/file-associations.ts new file mode 100644 index 000000000000..5c3989862239 --- /dev/null +++ b/app/ide-desktop/lib/client/file-associations.ts @@ -0,0 +1,13 @@ +/** @file File associations for client application. */ + +/** The extension for the source file, without the leading period character. */ +export const SOURCE_FILE_EXTENSION = 'enso' + +/** The extension for the project bundle, without the leading period character. */ +export const BUNDLED_PROJECT_EXTENSION = 'enso-project' + +/** The filename suffix for the source file, including the leading period character. */ +export const SOURCE_FILE_SUFFIX = `.${SOURCE_FILE_EXTENSION}` + +/** The filename suffix for the project bundle, including the leading period character. */ +export const BUNDLED_PROJECT_SUFFIX = `.${BUNDLED_PROJECT_EXTENSION}` diff --git a/app/ide-desktop/lib/client/package.json b/app/ide-desktop/lib/client/package.json index afd084173af7..89201c8b875c 100644 --- a/app/ide-desktop/lib/client/package.json +++ b/app/ide-desktop/lib/client/package.json @@ -25,6 +25,8 @@ "mime-types": "^2.1.35", "opener": "^1.5.2", "string-length": "^5.0.1", + "@types/tar": "^6.1.4", + "tar": "^6.1.13", "yargs": "17.6.2" }, "comments": { diff --git a/app/ide-desktop/lib/client/src/config/parser.ts b/app/ide-desktop/lib/client/src/config/parser.ts index 1aa357c09e57..f782784cf56b 100644 --- a/app/ide-desktop/lib/client/src/config/parser.ts +++ b/app/ide-desktop/lib/client/src/config/parser.ts @@ -3,15 +3,14 @@ import chalk from 'chalk' import stringLength from 'string-length' -import * as yargsHelpers from 'yargs/helpers' import yargs from 'yargs/yargs' import yargsModule from 'yargs' import * as contentConfig from 'enso-content-config' import * as config from 'config' +import * as fileAssociations from 'file-associations' import * as naming from 'naming' - import BUILD_INFO from '../../../../build.json' assert { type: 'json' } const logger = contentConfig.logger @@ -267,11 +266,9 @@ function argvAndChromeOptions(processArgs: string[]): ArgvAndChromeOptions { // ===================== /** Parses command line arguments. */ -export function parseArgs() { +export function parseArgs(clientArgs: string[] = fileAssociations.CLIENT_ARGUMENTS) { const args = config.CONFIG - const { argv, chromeOptions } = argvAndChromeOptions( - fixArgvNoPrefix(yargsHelpers.hideBin(process.argv)) - ) + const { argv, chromeOptions } = argvAndChromeOptions(fixArgvNoPrefix(clientArgs)) const yargsOptions = args .optionsRecursive() .reduce((opts: Record, option) => { diff --git a/app/ide-desktop/lib/client/src/file-associations.ts b/app/ide-desktop/lib/client/src/file-associations.ts new file mode 100644 index 000000000000..ae759ab51dd2 --- /dev/null +++ b/app/ide-desktop/lib/client/src/file-associations.ts @@ -0,0 +1,151 @@ +/** @file + * This module provides functionality for handling file opening events in the Enso IDE. + * + * It includes utilities for determining if a file can be opened, managing the file opening + * process, and launching new instances of the IDE when necessary. The module also exports + * constants related to file associations and project handling. */ + +import * as childProcess from 'node:child_process' +import * as fsSync from 'node:fs' +import * as pathModule from 'node:path' +import process from 'node:process' + +import * as electron from 'electron' +import electronIsDev from 'electron-is-dev' + +import * as common from 'enso-common' +import * as config from 'enso-content-config' +import * as fileAssociations from '../file-associations' +import * as project from './project-management' + +const logger = config.logger + +// ================= +// === Reexports === +// ================= + +export const BUNDLED_PROJECT_EXTENSION = fileAssociations.BUNDLED_PROJECT_EXTENSION +export const SOURCE_FILE_EXTENSION = fileAssociations.SOURCE_FILE_EXTENSION +export const BUNDLED_PROJECT_SUFFIX = fileAssociations.BUNDLED_PROJECT_SUFFIX +export const SOURCE_FILE_SUFFIX = fileAssociations.SOURCE_FILE_SUFFIX + +// ========================== +// === Arguments Handling === +// ========================== + +/** + * Check if the given list of application startup arguments denotes an attempt to open a file. + * + * For example, this happens when the user double-clicks on a file in the file explorer and the + * application is launched with the file path as an argument. + * + * @param clientArgs - A list of arguments passed to the application, stripped from the initial + * executable name and any electron dev mode arguments. + * @returns The path to the file to open, or `null` if no file was specified. + */ +export function argsDenoteFileOpenAttempt(clientArgs: string[]): string | null { + const arg = clientArgs[0] + let result: string | null = null + // If the application is invoked with exactly one argument and this argument is a file, we + // assume that we have been launched with a file to open. In this case, we must translate this + // path to the actual argument that'd open the project containing this file. + if (clientArgs.length === 1 && typeof arg !== 'undefined') { + try { + fsSync.accessSync(arg, fsSync.constants.R_OK) + result = arg + } catch (e) { + logger.log(`The single argument '${arg}' does not denote a readable file: ${String(e)}`) + } + } + return result +} + +/** Get the arguments, excluding the initial program name and any electron dev mode arguments. */ +export const CLIENT_ARGUMENTS = getClientArguments() + +/** Decide what are client arguments, @see {@link CLIENT_ARGUMENTS}. */ +function getClientArguments(): string[] { + if (electronIsDev) { + // Client arguments are separated from the electron dev mode arguments by a '--' argument. + const separator = '--' + const separatorIndex = process.argv.indexOf(separator) + const notFoundIndexPlaceholder = -1 + if (separatorIndex === notFoundIndexPlaceholder) { + // If there is no separator, client gets no arguments. + return [] + } else { + // Drop everything before the separator. + return process.argv.slice(separatorIndex + 1) + } + } else { + // Drop the leading executable name. + return process.argv.slice(1) + } +} + +// ========================= +// === File Associations === +// ========================= + +/* Check if the given path looks like a file that we can open. */ +export function isFileOpenable(path: string): boolean { + const extension = pathModule.extname(path).toLowerCase() + return ( + extension === fileAssociations.BUNDLED_PROJECT_EXTENSION || + extension === fileAssociations.SOURCE_FILE_EXTENSION + ) +} + +/* On macOS when Enso-associated file is opened, the application is first started and then it + * receives the `open-file` event. However, if there is already an instance of Enso running, + * it receives the `open-file` event (and no new instance is created for us). In this case, + * we manually start a new instance of the application and pass the file path to it (using the + * Windows-style command). + */ +export function onFileOpened(event: Event, path: string) { + if (isFileOpenable(path)) { + // If we are not ready, we can still decide to open a project rather than enter the welcome + // screen. However, we still check for the presence of arguments, to prevent hijacking the + // user-spawned IDE instance (OS-spawned will not have arguments set). + if (!electron.app.isReady() && CLIENT_ARGUMENTS.length === 0) { + event.preventDefault() + logger.log(`Opening file '${path}'.`) + // eslint-disable-next-line no-restricted-syntax + return handleOpenFile(path) + } else { + // We need to start another copy of the application, as the first one is already running. + logger.log( + `The application is already initialized. Starting a new instance to open file '${path}'.` + ) + const args = [path] + const child = childProcess.spawn(process.execPath, args, { + detached: true, + stdio: 'ignore', + }) + // Prevent parent (this) process from waiting for the child to exit. + child.unref() + } + } +} + +/** Handle the case where IDE is invoked with a file to open. + * + * Imports project if necessary. Returns the ID of the project to open. In case of an error, displays an error message and rethrows the error. + * + * @throws An `Error`, if the project from the file cannot be opened or imported. */ +export function handleOpenFile(openedFile: string): string { + try { + return project.importProjectFromPath(openedFile) + } catch (e: unknown) { + // Since the user has explicitly asked us to open a file, in case of an error, we should + // display a message box with the error details. + let message = `Cannot open file '${openedFile}'.` + message += `\n\nReason:\n${e?.toString() ?? 'Unknown error'}` + if (e instanceof Error && typeof e.stack !== 'undefined') { + message += `\n\nDetails:\n${e.stack}` + } + logger.error(e) + electron.dialog.showErrorBox(common.PRODUCT_NAME, message) + throw e + } +} diff --git a/app/ide-desktop/lib/client/src/index.ts b/app/ide-desktop/lib/client/src/index.ts index ac8583e68242..e9e74d3bf093 100644 --- a/app/ide-desktop/lib/client/src/index.ts +++ b/app/ide-desktop/lib/client/src/index.ts @@ -12,28 +12,25 @@ import process from 'node:process' import * as electron from 'electron' +import * as common from 'enso-common' import * as contentConfig from 'enso-content-config' import * as authentication from 'authentication' import * as config from 'config' import * as configParser from 'config/parser' import * as debug from 'debug' +// eslint-disable-next-line no-restricted-syntax +import * as fileAssociations from 'file-associations' import * as ipc from 'ipc' import * as naming from 'naming' import * as paths from 'paths' import * as projectManager from 'bin/project-manager' import * as security from 'security' import * as server from 'bin/server' +import * as utils from '../../../utils' const logger = contentConfig.logger -// ================= -// === Constants === -// ================= - -/** Indent size for outputting JSON. */ -const INDENT_SIZE = 4 - // =========== // === App === // =========== @@ -47,16 +44,30 @@ class App { isQuitting = false async run() { - const { args, windowSize, chromeOptions } = configParser.parseArgs() - this.args = args + // Register file associations for macOS. + electron.app.on('open-file', fileAssociations.onFileOpened) + + const { windowSize, chromeOptions, fileToOpen } = this.processArguments() + if (fileToOpen != null) { + try { + // This makes the IDE open the relevant project. Also, this prevents us from using this + // method after IDE has been fully set up, as the initializing code would have already + // read the value of this argument. + this.args.groups.startup.options.project.value = + fileAssociations.handleOpenFile(fileToOpen) + } catch (e) { + // If we failed to open the file, we should enter the usual welcome screen. + // The `handleOpenFile` function will have already displayed an error message. + } + } if (this.args.options.version.value) { await this.printVersion() - process.exit() + electron.app.quit() } else if (this.args.groups.debug.options.info.value) { await electron.app.whenReady().then(async () => { await debug.printInfo() + electron.app.quit() }) - process.exit() } else { this.setChromeOptions(chromeOptions) security.enableAll() @@ -74,6 +85,19 @@ class App { } } + processArguments() { + // We parse only "client arguments", so we don't have to worry about the Electron-Dev vs + // Electron-Proper distinction. + const fileToOpen = fileAssociations.argsDenoteFileOpenAttempt( + fileAssociations.CLIENT_ARGUMENTS + ) + // If we are opening a file (i.e. we were spawned with just a path of the file to open as + // the argument), it means that effectively we don't have any non-standard arguments. + // We just need to let caller know that we are opening a file. + const argsToParse = fileToOpen ? [] : fileAssociations.CLIENT_ARGUMENTS + return { ...configParser.parseArgs(argsToParse), fileToOpen } + } + /** Set Chrome options based on the app configuration. For comprehensive list of available * Chrome options refer to: https://peter.sh/experiments/chromium-command-line-switches. */ setChromeOptions(chromeOptions: configParser.ChromeOption[]) { @@ -292,7 +316,7 @@ class App { } printVersion(): Promise { - const indent = ' '.repeat(INDENT_SIZE) + const indent = ' '.repeat(utils.INDENT_SIZE) let maxNameLen = 0 for (const name in debug.VERSION_INFO) { maxNameLen = Math.max(maxNameLen, name.length) @@ -355,5 +379,11 @@ class App { // === App startup === // =================== +process.on('uncaughtException', (err, origin) => { + console.error(`Uncaught exception: ${String(err)}\nException origin: ${origin}`) + electron.dialog.showErrorBox(common.PRODUCT_NAME, err.stack ?? err.toString()) + electron.app.exit(1) +}) + const APP = new App() void APP.run() diff --git a/app/ide-desktop/lib/client/src/paths.ts b/app/ide-desktop/lib/client/src/paths.ts index 0f23f5f9f5c6..fc12a701d74e 100644 --- a/app/ide-desktop/lib/client/src/paths.ts +++ b/app/ide-desktop/lib/client/src/paths.ts @@ -35,3 +35,6 @@ export const PROJECT_MANAGER_PATH = path.join( // Placeholder for a bundler-provided define. PROJECT_MANAGER_IN_BUNDLE_PATH ) + +/** Relative path of Enso Project PM metadata relative to project's root. */ +export const PROJECT_METADATA_RELATIVE = path.join('.enso', 'project.json') diff --git a/app/ide-desktop/lib/client/src/project-management.ts b/app/ide-desktop/lib/client/src/project-management.ts new file mode 100644 index 000000000000..79ba0c51e808 --- /dev/null +++ b/app/ide-desktop/lib/client/src/project-management.ts @@ -0,0 +1,294 @@ +/** @file This module contains functions for importing projects into the Project Manager. + * + * Eventually this module should be replaced with a new Project Manager API that supports importing projects. + * For now, we basically do the following: + * - if the project is already in the Project Manager's location, we just open it; + * - if the project is in a different location, we copy it to the Project Manager's location and open it. + * - if the project is a bundle, we extract it to the Project Manager's location and open it. + */ + +import * as crypto from 'node:crypto' +import * as fsSync from 'node:fs' +import * as fss from 'node:fs' +import * as pathModule from 'node:path' + +import * as electron from 'electron' +import * as tar from 'tar' + +import * as common from 'enso-common' +import * as config from 'enso-content-config' +import * as fileAssociations from '../file-associations' +import * as paths from './paths' +import * as utils from '../../../utils' + +const logger = config.logger + +// ====================== +// === Project Import === +// ====================== + +/** Open a project from the given path. Path can be either a source file under the project root, or the project + * bundle. If needed, the project will be imported into the Project Manager-enabled location. + * + * @returns Project ID (from Project Manager's metadata) identifying the imported project. + * @throws `Error` if the path does not belong to a valid project. + */ +export function importProjectFromPath(openedPath: string): string { + if (pathModule.extname(openedPath).endsWith(fileAssociations.BUNDLED_PROJECT_EXTENSION)) { + // The second part of condition is for the case when someone names a directory like `my-project.enso-project` + // and stores the project there. Not the most fortunate move, but... + if (isProjectRoot(openedPath)) { + return importDirectory(openedPath) + } else { + // Project bundle was provided, so we need to extract it first. + return importBundle(openedPath) + } + } else { + logger.log(`Opening file: '${openedPath}'.`) + const rootPath = getProjectRoot(openedPath) + // Check if the project root is under the projects directory. If it is, we can open it. + // Otherwise, we need to install it first. + if (rootPath == null) { + const message = `File '${openedPath}' does not belong to the ${common.PRODUCT_NAME} project.` + throw new Error(message) + } + return importDirectory(rootPath) + } +} + +/** Import the project from a bundle. + * + * @returns Project ID (from Project Manager's metadata) identifying the imported project. + */ +export function importBundle(bundlePath: string): string { + // The bundle is a tarball, so we just need to extract it to the right location. + const bundleRoot = directoryWithinBundle(bundlePath) + const targetDirectory = generateDirectoryName(bundleRoot ?? bundlePath) + fss.mkdirSync(targetDirectory, { recursive: true }) + // To be more resilient against different ways that user might attempt to create a bundle, we try to support + // both archives that: + // * contain a single directory with the project files - that directory name will be used to generate a new target + // directory name; + // * contain the project files directly - in this case, the archive filename will be used to generate a new target + // directory name. + // We try to tell apart these two cases by looking at the common prefix of the paths of the files in the archive. + // If there is any, everything is under a single directory, and we need to strip it. + tar.x({ + file: bundlePath, + cwd: targetDirectory, + sync: true, + strip: bundleRoot != null ? 1 : 0, + }) + return updateId(targetDirectory) +} + +/** Import the project, so it becomes visible to Project Manager. + * + * @param rootPath - The path to the project root. + * @returns Project ID (from Project Manager's metadata) identifying the imported project. + * @throws `Error` if there occurs race-condition when generating a unique project directory name. + */ +export function importDirectory(rootPath: string): string { + if (isProjectInstalled(rootPath)) { + // Project is already visible to Project Manager, so we can just return its ID. + logger.log(`Project already installed: '${rootPath}'.`) + return getProjectId(rootPath) + } else { + logger.log(`Importing a project copy from: '${rootPath}'.`) + const targetDirectory = generateDirectoryName(rootPath) + if (fsSync.existsSync(targetDirectory)) { + const message = `Project directory already exists: ${targetDirectory}.` + throw new Error(message) + } + + logger.log(`Copying: '${rootPath}' -> '${targetDirectory}'.`) + fsSync.cpSync(rootPath, targetDirectory, { recursive: true }) + // Update the project ID, so we are certain that it is unique. This would be violated, if we imported the same + // project multiple times. + return updateId(targetDirectory) + } +} + +// ================ +// === Metadata === +// ================ + +/** The Project Manager's metadata associated with a project. + * + * The property list is not exhaustive, it only contains the properties that we need. + */ +interface ProjectMetadata { + /** The ID of the project. It is only used in communication with project manager, it has no semantic meaning. */ + id: string +} + +/** + * Type guard function to check if an object conforms to the ProjectMetadata interface. + * + * This function checks if the input object has the required properties and correct types + * to match the ProjectMetadata interface. It can be used at runtime to validate that + * a given object has the expected shape. + * + * @param value - The object to check against the ProjectMetadata interface. + * @returns A boolean value indicating whether the object matches the ProjectMetadata interface. + */ +function isProjectMetadata(value: unknown): value is ProjectMetadata { + return ( + typeof value === 'object' && value != null && 'id' in value && typeof value.id === 'string' + ) +} + +/** Get the ID from the project metadata. */ +export function getProjectId(projectRoot: string): string { + return getMetadata(projectRoot).id +} + +/** Retrieve the project's metadata. + * + * @throws `Error` if the metadata file is missing or ill-formed. */ +export function getMetadata(projectRoot: string): ProjectMetadata { + const metadataPath = pathModule.join(projectRoot, paths.PROJECT_METADATA_RELATIVE) + const jsonText = fss.readFileSync(metadataPath, 'utf8') + const metadata: unknown = JSON.parse(jsonText) + if (isProjectMetadata(metadata)) { + return metadata + } else { + throw new Error('Invalid project metadata') + } +} + +/** Write the project's metadata. */ +export function writeMetadata(projectRoot: string, metadata: ProjectMetadata): void { + const metadataPath = pathModule.join(projectRoot, paths.PROJECT_METADATA_RELATIVE) + fss.writeFileSync(metadataPath, JSON.stringify(metadata, null, utils.INDENT_SIZE)) +} + +/** Update project's metadata. If the provided updater does not return anything, the metadata file is left intact. + * + * The updater function-returned metadata is passed over. + */ +export function updateMetadata( + projectRoot: string, + updater: (initialMetadata: ProjectMetadata) => ProjectMetadata +): ProjectMetadata { + const metadata = getMetadata(projectRoot) + const updatedMetadata = updater(metadata) + writeMetadata(projectRoot, updatedMetadata) + return updatedMetadata +} + +// ========================= +// === Project Directory === +// ========================= + +/* Check if the given path represents the root of an Enso project. This is decided by the presence + * of Project Manager's metadata. */ +export function isProjectRoot(candidatePath: string): boolean { + const projectJsonPath = pathModule.join(candidatePath, paths.PROJECT_METADATA_RELATIVE) + let isRoot = false + try { + fss.accessSync(projectJsonPath, fss.constants.R_OK) + isRoot = true + } catch (e) { + // No need to do anything, isRoot is already set to false + } + return isRoot +} + +/** Check if this bundle is a compressed directory (rather than directly containing the project files). If it is, we + * return the name of the directory. Otherwise, we return `null`. */ +export function directoryWithinBundle(bundlePath: string): string | null { + // We need to look up the root directory among the tarball entries. + let commonPrefix: string | null = null + tar.list({ + file: bundlePath, + sync: true, + onentry: entry => { + // We normalize to get rid of leading `.` (if any). + let path = entry.path.normalize() + commonPrefix = commonPrefix == null ? path : utils.getCommonPrefix(commonPrefix, path) + }, + }) + // ESLint doesn't understand that `commonPrefix` can be not `null` here due to the `onentry` callback. + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + return commonPrefix ? pathModule.basename(commonPrefix) : null +} + +/** Generate a name for project using given base string. Suffixes are added if there's a collision. + * + * For example 'Name' will become 'Name_1' if there's already a directory named 'Name'. + * If given a name like 'Name_1' it will become 'Name_2' if there's already a directory named 'Name_1'. + * If a path containing multiple components is given, only the last component is used for the name. */ +export function generateDirectoryName(name: string): string { + // Use only the last path component. + name = pathModule.parse(name).name + + // If the name already consists a suffix, reuse it. + const matches = name.match(/^(.*)_(\d+)$/) + let suffix = 0 + // Matches start with the whole match, so we need to skip it. Then come our two capture groups. + const [matchedName, matchedSuffix] = matches?.slice(1) ?? [] + if (typeof matchedName !== 'undefined' && typeof matchedSuffix !== 'undefined') { + name = matchedName + suffix = parseInt(matchedSuffix) + } + + const projectsDirectory = getProjectsDirectory() + for (; ; suffix++) { + let candidatePath = pathModule.join( + projectsDirectory, + `${name}${suffix === 0 ? '' : `_${suffix}`}` + ) + if (!fss.existsSync(candidatePath)) { + // eslint-disable-next-line no-restricted-syntax + return candidatePath + } + } + // Unreachable. +} + +/** Takes a path to a file, presumably located in a project's subtree. Returns the path to the project's root directory + * or `null` if the file is not located in a project. */ +export function getProjectRoot(subtreePath: string): string | null { + let currentPath = subtreePath + while (!isProjectRoot(currentPath)) { + const parent = pathModule.dirname(currentPath) + if (parent === currentPath) { + // eslint-disable-next-line no-restricted-syntax + return null + } + currentPath = parent + } + return currentPath +} + +/** Get the directory that stores Enso projects. */ +export function getProjectsDirectory(): string { + return pathModule.join(electron.app.getPath('home'), 'enso', 'projects') +} + +/** Check if the given project is installed, i.e. can be opened with the Project Manager. */ +export function isProjectInstalled(projectRoot: string): boolean { + // Project can be opened by project manager only if its root directory is directly under the projects directory. + const projectsDirectory = getProjectsDirectory() + const projectRootParent = pathModule.dirname(projectRoot) + // Should resolve symlinks and relative paths. Normalize before comparison. + return pathModule.resolve(projectRootParent) === pathModule.resolve(projectsDirectory) +} + +// ================== +// === Project ID === +// ================== + +/** Generates a unique UUID for a project. */ +export function generateId(): string { + return crypto.randomUUID() +} + +/** Update the project's ID to a new, unique value. */ +export function updateId(projectRoot: string): string { + return updateMetadata(projectRoot, metadata => ({ + ...metadata, + id: generateId(), + })).id +} diff --git a/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/config.ts b/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/config.ts index 67c52ecb08b0..4307e3ed6793 100644 --- a/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/config.ts +++ b/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/config.ts @@ -10,18 +10,21 @@ * pools, Amplify must be configured prior to use. This file defines all the information needed to * connect to and use these pools. */ -import * as utils from '../utils' +import * as newtype from '../newtype' // ================= // === Constants === // ================= /** AWS region in which our Cognito pool is located. */ -export const AWS_REGION = utils.brand('eu-west-1') +export const AWS_REGION = newtype.asNewtype('eu-west-1') /** Complete list of OAuth scopes used by the app. */ -export const OAUTH_SCOPES = [utils.brand('email'), utils.brand('openid')] +export const OAUTH_SCOPES = [ + newtype.asNewtype('email'), + newtype.asNewtype('openid'), +] /** OAuth response type used in the OAuth flows. */ -export const OAUTH_RESPONSE_TYPE = utils.brand('code') +export const OAUTH_RESPONSE_TYPE = newtype.asNewtype('code') // ============= // === Types === @@ -34,34 +37,34 @@ export const OAUTH_RESPONSE_TYPE = utils.brand('code') /** The AWS region in which our Cognito pool is located. This is always set to `eu-west-1` because * that is the only region in which our Cognito pools are currently available in. */ -type AwsRegion = utils.Brand<'AwsRegion'> & string +type AwsRegion = newtype.Newtype /** ID of the "Cognito user pool" that contains authentication & identity data of our users. * * This is created automatically by our Terraform scripts when the backend infrastructure is * created. Look in the `enso-org/cloud-v2` repo for details. */ -export type UserPoolId = utils.Brand<'UserPoolId'> & string +export type UserPoolId = newtype.Newtype /** ID of an OAuth client authorized to interact with the Cognito user pool specified by the * {@link UserPoolId}. * * This is created automatically by our Terraform scripts when the backend infrastructure is * created. Look in the `enso-org/cloud-v2` repo for details. */ -export type UserPoolWebClientId = utils.Brand<'UserPoolWebClientId'> & string +export type UserPoolWebClientId = newtype.Newtype /** Domain of the Cognito user pool used for authenticating/identifying the user. * * This must correspond to the public-facing domain name of the Cognito pool identified by the * {@link UserPoolId}, and must not contain an HTTP scheme, or a pathname. */ -export type OAuthDomain = utils.Brand<'OAuthDomain'> & string +export type OAuthDomain = newtype.Newtype /** Possible OAuth scopes to request from the federated identity provider during OAuth sign-in. */ -type OAuthScope = utils.Brand<'OAuthScope'> & string +type OAuthScope = newtype.Newtype /** The response type used to complete the OAuth flow. "code" means that the federated identity * provider will return an authorization code that can be exchanged for an access token. The * authorization code will be provided as a query parameter of the redirect URL. */ -type OAuthResponseType = utils.Brand<'OAuthResponseType'> & string +type OAuthResponseType = newtype.Newtype /** The URL used as a redirect (minus query parameters like `code` which get appended later), once * an OAuth flow (e.g., sign-in or sign-out) has completed. These must match the values set in the * Cognito pool and during the creation of the OAuth client. See the `enso-org/cloud-v2` repo for * details. */ -export type OAuthRedirect = utils.Brand<'OAuthRedirect'> & string +export type OAuthRedirect = newtype.Newtype /** Callback used to open URLs for the OAuth flow. This is only used in the desktop app (i.e., not in * the cloud). This is because in the cloud we just keep the user in their browser, but in the app * we want to open OAuth URLs in the system browser. This is because the user can't be expected to diff --git a/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/providers/auth.tsx b/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/providers/auth.tsx index 4abed93fd4e6..bbf3d70dac2a 100644 --- a/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/providers/auth.tsx +++ b/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/providers/auth.tsx @@ -12,6 +12,7 @@ import * as authServiceModule from '../service' import * as backendService from '../../dashboard/service' import * as errorModule from '../../error' import * as loggerProvider from '../../providers/logger' +import * as newtype from '../../newtype' import * as sessionProvider from './session' // ================= @@ -47,7 +48,7 @@ export interface FullUserSession { /** User's email address. */ email: string /** User's organization information. */ - organization: backendService.Organization + organization: backendService.UserOrOrganization } /** Object containing the currently signed-in user's session data, if the user has not yet set their @@ -155,7 +156,7 @@ export function AuthProvider(props: AuthProviderProps) { const { accessToken, email } = session.val const backend = backendService.createBackend(accessToken, logger) - const organization = await backend.getUser() + const organization = await backend.usersMe() let newUserSession: UserSession if (!organization) { newUserSession = { @@ -239,17 +240,15 @@ export function AuthProvider(props: AuthProviderProps) { }) const setUsername = async (accessToken: string, username: string, email: string) => { - const body: backendService.SetUsernameRequestBody = { - userName: username, - userEmail: email, - } - /** TODO [NP]: https://github.com/enso-org/cloud-v2/issues/343 * The API client is reinitialised on every request. That is an inefficient way of usage. * Fix it by using React context and implementing it as a singleton. */ const backend = backendService.createBackend(accessToken, logger) - await backend.setUsername(body) + await backend.createUser({ + userName: username, + userEmail: newtype.asNewtype(email), + }) navigate(app.DASHBOARD_PATH) toast.success(MESSAGES.setUsernameSuccess) } diff --git a/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/service.tsx b/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/service.tsx index 1c0eb81bbee1..5c2ac40a2985 100644 --- a/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/service.tsx +++ b/app/ide-desktop/lib/dashboard/src/authentication/src/authentication/service.tsx @@ -11,8 +11,8 @@ import * as cognito from './cognito' import * as config from '../config' import * as listen from './listen' import * as loggerProvider from '../providers/logger' +import * as newtype from '../newtype' import * as platformModule from '../platform' -import * as utils from '../utils' // ================= // === Constants === @@ -32,7 +32,7 @@ const CONFIRM_REGISTRATION_PATHNAME = '//auth/confirmation' const LOGIN_PATHNAME = '//auth/login' /** URL used as the OAuth redirect when running in the desktop app. */ -const DESKTOP_REDIRECT = utils.brand(`${common.DEEP_LINK_SCHEME}://auth`) +const DESKTOP_REDIRECT = newtype.asNewtype(`${common.DEEP_LINK_SCHEME}://auth`) /** Map from platform to the OAuth redirect URL that should be used for that platform. */ const PLATFORM_TO_CONFIG: Record< platformModule.Platform, @@ -58,16 +58,22 @@ const BASE_AMPLIFY_CONFIG = { const AMPLIFY_CONFIGS = { /** Configuration for @pbuchu's Cognito user pool. */ pbuchu: { - userPoolId: utils.brand('eu-west-1_jSF1RbgPK'), - userPoolWebClientId: utils.brand('1bnib0jfon3aqc5g3lkia2infr'), - domain: utils.brand('pb-enso-domain.auth.eu-west-1.amazoncognito.com'), + userPoolId: newtype.asNewtype('eu-west-1_jSF1RbgPK'), + userPoolWebClientId: newtype.asNewtype( + '1bnib0jfon3aqc5g3lkia2infr' + ), + domain: newtype.asNewtype( + 'pb-enso-domain.auth.eu-west-1.amazoncognito.com' + ), ...BASE_AMPLIFY_CONFIG, } satisfies Partial, /** Configuration for the production Cognito user pool. */ production: { - userPoolId: utils.brand('eu-west-1_9Kycu2SbD'), - userPoolWebClientId: utils.brand('4j9bfs8e7415erf82l129v0qhe'), - domain: utils.brand( + userPoolId: newtype.asNewtype('eu-west-1_9Kycu2SbD'), + userPoolWebClientId: newtype.asNewtype( + '4j9bfs8e7415erf82l129v0qhe' + ), + domain: newtype.asNewtype( 'production-enso-domain.auth.eu-west-1.amazoncognito.com' ), ...BASE_AMPLIFY_CONFIG, @@ -202,7 +208,7 @@ function setDeepLinkHandler(logger: loggerProvider.Logger, navigate: (url: strin navigate(app.LOGIN_PATH) break /** If the user is being redirected from a password reset email, then we need to navigate to - * the password reset page, with the verification code and email passed in the URL so they can + * the password reset page, with the verification code and email passed in the URL s-o they can * be filled in automatically. */ case app.RESET_PASSWORD_PATH: { const resetPasswordRedirectUrl = `${app.RESET_PASSWORD_PATH}${parsedUrl.search}` diff --git a/app/ide-desktop/lib/dashboard/src/authentication/src/config.ts b/app/ide-desktop/lib/dashboard/src/authentication/src/config.ts index 487c03dbf98b..3088d550c654 100644 --- a/app/ide-desktop/lib/dashboard/src/authentication/src/config.ts +++ b/app/ide-desktop/lib/dashboard/src/authentication/src/config.ts @@ -1,7 +1,7 @@ /** @file Configuration definition for the Dashboard. */ import * as auth from './authentication/config' -import * as utils from './utils' +import * as newtype from './newtype' // ================= // === Constants === @@ -16,14 +16,14 @@ const CLOUD_REDIRECTS = { * The redirect URL must be known ahead of time because it is registered with the OAuth provider * when it is created. In the native app, the port is unpredictable, but this is not a problem * because the native app does not use port-based redirects, but deep links. */ - development: utils.brand('http://localhost:8081'), - production: utils.brand('https://cloud.enso.org'), + development: newtype.asNewtype('http://localhost:8081'), + production: newtype.asNewtype('https://cloud.enso.org'), } /** All possible API URLs, sorted by environment. */ const API_URLS = { - pbuchu: utils.brand('https://xw0g8j3tsb.execute-api.eu-west-1.amazonaws.com'), - production: utils.brand('https://7aqkn3tnbc.execute-api.eu-west-1.amazonaws.com'), + pbuchu: newtype.asNewtype('https://xw0g8j3tsb.execute-api.eu-west-1.amazonaws.com'), + production: newtype.asNewtype('https://7aqkn3tnbc.execute-api.eu-west-1.amazonaws.com'), } /** All possible configuration options, sorted by environment. */ @@ -65,4 +65,4 @@ export type Environment = 'pbuchu' | 'production' // =========== /** Base URL for requests to our Cloud API backend. */ -type ApiUrl = utils.Brand<'ApiUrl'> & string +type ApiUrl = newtype.Newtype diff --git a/app/ide-desktop/lib/dashboard/src/authentication/src/dashboard/service.ts b/app/ide-desktop/lib/dashboard/src/authentication/src/dashboard/service.ts new file mode 100644 index 000000000000..d891e4d9bc8f --- /dev/null +++ b/app/ide-desktop/lib/dashboard/src/authentication/src/dashboard/service.ts @@ -0,0 +1,852 @@ +/** @file Module containing the API client for the Cloud backend API. + * + * Each exported function in the {@link Backend} in this module corresponds to an API endpoint. The + * functions are asynchronous and return a `Promise` that resolves to the response from the API. */ +import * as config from '../config' +import * as http from '../http' +import * as loggerProvider from '../providers/logger' +import * as newtype from '../newtype' + +// ================= +// === Constants === +// ================= + +/** HTTP status indicating that the request was successful. */ +const STATUS_OK = 200 + +/** Default HTTP body for an "open project" request. */ +const DEFAULT_OPEN_PROJECT_BODY: OpenProjectRequestBody = { + forceCreate: false, +} + +/** Relative HTTP path to the "set username" endpoint of the Cloud backend API. */ +const CREATE_USER_PATH = 'users' +/** Relative HTTP path to the "get user" endpoint of the Cloud backend API. */ +const USERS_ME_PATH = 'users/me' +/** Relative HTTP path to the "list directory" endpoint of the Cloud backend API. */ +const LIST_DIRECTORY_PATH = 'directories' +/** Relative HTTP path to the "create directory" endpoint of the Cloud backend API. */ +const CREATE_DIRECTORY_PATH = 'directories' +/** Relative HTTP path to the "list projects" endpoint of the Cloud backend API. */ +const LIST_PROJECTS_PATH = 'projects' +/** Relative HTTP path to the "create project" endpoint of the Cloud backend API. */ +const CREATE_PROJECT_PATH = 'projects' +/** Relative HTTP path to the "list files" endpoint of the Cloud backend API. */ +const LIST_FILES_PATH = 'files' +/** Relative HTTP path to the "upload file" endpoint of the Cloud backend API. */ +const UPLOAD_FILE_PATH = 'files' +/** Relative HTTP path to the "create secret" endpoint of the Cloud backend API. */ +const CREATE_SECRET_PATH = 'secrets' +/** Relative HTTP path to the "list secrets" endpoint of the Cloud backend API. */ +const LIST_SECRETS_PATH = 'secrets' +/** Relative HTTP path to the "create tag" endpoint of the Cloud backend API. */ +const CREATE_TAG_PATH = 'tags' +/** Relative HTTP path to the "list tags" endpoint of the Cloud backend API. */ +const LIST_TAGS_PATH = 'tags' +/** Relative HTTP path to the "list versions" endpoint of the Cloud backend API. */ +const LIST_VERSIONS_PATH = 'versions' +/** Relative HTTP path to the "close project" endpoint of the Cloud backend API. */ +function closeProjectPath(projectId: ProjectId) { + return `projects/${projectId}/close` +} +/** Relative HTTP path to the "get project details" endpoint of the Cloud backend API. */ +function getProjectDetailsPath(projectId: ProjectId) { + return `projects/${projectId}` +} +/** Relative HTTP path to the "open project" endpoint of the Cloud backend API. */ +function openProjectPath(projectId: ProjectId) { + return `projects/${projectId}/open` +} +/** Relative HTTP path to the "project update" endpoint of the Cloud backend API. */ +function projectUpdatePath(projectId: ProjectId) { + return `projects/${projectId}` +} +/** Relative HTTP path to the "delete project" endpoint of the Cloud backend API. */ +function deleteProjectPath(projectId: ProjectId) { + return `projects/${projectId}` +} +/** Relative HTTP path to the "check resources" endpoint of the Cloud backend API. */ +function checkResourcesPath(projectId: ProjectId) { + return `projects/${projectId}/resources` +} +/** Relative HTTP path to the "delete file" endpoint of the Cloud backend API. */ +function deleteFilePath(fileId: FileId) { + return `files/${fileId}` +} +/** Relative HTTP path to the "get project" endpoint of the Cloud backend API. */ +function getSecretPath(secretId: SecretId) { + return `secrets/${secretId}` +} +/** Relative HTTP path to the "delete secret" endpoint of the Cloud backend API. */ +function deleteSecretPath(secretId: SecretId) { + return `secrets/${secretId}` +} +/** Relative HTTP path to the "delete tag" endpoint of the Cloud backend API. */ +function deleteTagPath(tagId: TagId) { + return `secrets/${tagId}` +} + +// ============= +// === Types === +// ============= + +/** Unique identifier for a user/organization. */ +export type UserOrOrganizationId = newtype.Newtype + +/** Unique identifier for a directory. */ +export type DirectoryId = newtype.Newtype + +/** Unique identifier for a user's project. */ +export type ProjectId = newtype.Newtype + +/** Unique identifier for an uploaded file. */ +export type FileId = newtype.Newtype + +/** Unique identifier for a secret environment variable. */ +export type SecretId = newtype.Newtype + +/** Unique identifier for a file tag or project tag. */ +export type TagId = newtype.Newtype + +/** A URL. */ +export type Address = newtype.Newtype + +/** An email address. */ +export type EmailAddress = newtype.Newtype + +/** An AWS S3 file path. */ +export type S3FilePath = newtype.Newtype + +export type Ami = newtype.Newtype + +export type Subject = newtype.Newtype + +/** An RFC 3339 DateTime string. */ +export type Rfc3339DateTime = newtype.Newtype + +/** A user/organization in the application. These are the primary owners of a project. */ +export interface UserOrOrganization { + id: UserOrOrganizationId + name: string + email: EmailAddress +} + +/** Possible states that a project can be in. */ +export enum ProjectState { + created = 'Created', + new = 'New', + openInProgress = 'OpenInProgress', + opened = 'Opened', + closed = 'Closed', +} + +/** Wrapper around a project state value. */ +export interface ProjectStateType { + type: ProjectState +} + +/** Common `Project` fields returned by all `Project`-related endpoints. */ +export interface BaseProject { + organizationId: string + projectId: ProjectId + name: string +} + +/** A `Project` returned by `createProject`. */ +export interface CreatedProject extends BaseProject { + state: ProjectStateType + packageName: string +} + +/** A `Project` returned by `listProjects`. */ +export interface ListedProject extends CreatedProject { + address: Address | null +} + +/** A `Project` returned by `updateProject`. */ +export interface UpdatedProject extends BaseProject { + ami: Ami | null + ideVersion: VersionNumber | null + engineVersion: VersionNumber | null +} + +/** A user/organization's project containing and/or currently executing code. */ +export interface Project extends ListedProject { + ideVersion: VersionNumber | null + engineVersion: VersionNumber | null +} + +/** Metadata describing an uploaded file. */ +export interface File { + // eslint-disable-next-line @typescript-eslint/naming-convention + file_id: FileId + // eslint-disable-next-line @typescript-eslint/naming-convention + file_name: string | null + path: S3FilePath +} + +/** Metadata uniquely identifying an uploaded file. */ +export interface FileInfo { + /* TODO: Should potentially be S3FilePath, + * but it's just string on the backend. */ + path: string + id: FileId +} + +/** A secret environment variable. */ +export interface Secret { + id: SecretId + value: string +} + +/** A secret environment variable and metadata uniquely identifying it. */ +export interface SecretAndInfo { + id: SecretId + name: string + value: string +} + +/** Metadata uniquely identifying a secret environment variable. */ +export interface SecretInfo { + name: string + id: SecretId +} + +export enum TagObjectType { + file = 'File', + project = 'Project', +} + +/** A file tag or project tag. */ +export interface Tag { + /* eslint-disable @typescript-eslint/naming-convention */ + organization_id: UserOrOrganizationId + id: TagId + name: string + value: string + object_type: TagObjectType + object_id: string + /* eslint-enable @typescript-eslint/naming-convention */ +} + +/** Metadata uniquely identifying a file tag or project tag. */ +export interface TagInfo { + id: TagId + name: string + value: string +} + +/** Type of application that a {@link Version} applies to. + * + * We keep track of both backend and IDE versions, so that we can update the two independently. + * However the format of the version numbers is the same for both, so we can use the same type for + * both. We just need this enum to disambiguate. */ +export enum VersionType { + backend = 'Backend', + ide = 'Ide', +} + +/** Stability of an IDE or backend version. */ +export enum VersionLifecycle { + stable = 'Stable', + releaseCandidate = 'ReleaseCandidate', + nightly = 'Nightly', + development = 'Development', +} + +/** Version number of an IDE or backend. */ +export interface VersionNumber { + value: string + lifecycle: VersionLifecycle +} + +/** A version describing a release of the backend or IDE. */ +export interface Version { + number: VersionNumber + ami: Ami | null + created: Rfc3339DateTime + // This does not follow our naming convention because it's defined this way in the backend, + // so we need to match it. + // eslint-disable-next-line @typescript-eslint/naming-convention + version_type: VersionType +} + +/** Resource usage of a VM. */ +export interface ResourceUsage { + /** Percentage of memory used. */ + memory: number + /** Percentage of CPU time used since boot. */ + cpu: number + /** Percentage of disk space used. */ + storage: number +} + +export interface User { + /* eslint-disable @typescript-eslint/naming-convention */ + pk: Subject + user_name: string + user_email: EmailAddress + organization_id: UserOrOrganizationId + /* eslint-enable @typescript-eslint/naming-convention */ +} + +export enum PermissionAction { + own = 'Own', + execute = 'Execute', + edit = 'Edit', + read = 'Read', +} + +export interface UserPermission { + user: User + permission: PermissionAction +} + +/** Metadata uniquely identifying a directory entry. + * Thes can be Projects, Files, Secrets, or other directories. */ +interface BaseAsset { + title: string + id: string + parentId: string + permissions: UserPermission[] | null +} + +export enum AssetType { + project = 'project', + file = 'file', + secret = 'secret', + directory = 'directory', +} + +export interface IdType { + [AssetType.project]: ProjectId + [AssetType.file]: FileId + [AssetType.secret]: SecretId + [AssetType.directory]: DirectoryId +} + +/** Metadata uniquely identifying a directory entry. + * Thes can be Projects, Files, Secrets, or other directories. */ +export interface Asset extends BaseAsset { + type: Type + id: IdType[Type] +} + +// This is an alias. +// It should be a separate type because it is the return value of one of the APIs. +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface Directory extends Asset {} + +// ================= +// === Endpoints === +// ================= + +/** HTTP request body for the "set username" endpoint. */ +export interface CreateUserRequestBody { + userName: string + userEmail: EmailAddress +} + +/** HTTP request body for the "create directory" endpoint. */ +export interface CreateDirectoryRequestBody { + title: string + parentId?: DirectoryId +} + +/** HTTP request body for the "create project" endpoint. */ +export interface CreateProjectRequestBody { + projectName: string + projectTemplateName?: string + parentDirectoryId?: DirectoryId +} + +/** + * HTTP request body for the "project update" endpoint. + * Only updates of the `projectName` or `ami` are allowed. + */ +export interface ProjectUpdateRequestBody { + projectName: string | null + ami: Ami | null + ideVersion: VersionNumber | null +} + +/** HTTP request body for the "open project" endpoint. */ +export interface OpenProjectRequestBody { + forceCreate: boolean +} + +/** HTTP request body for the "create secret" endpoint. */ +export interface CreateSecretRequestBody { + secretName: string + secretValue: string + parentDirectoryId?: DirectoryId +} + +/** HTTP request body for the "create tag" endpoint. */ +export interface CreateTagRequestBody { + name: string + value: string + objectType: TagObjectType + objectId: string +} + +export interface ListDirectoryRequestParams { + parentId?: string +} + +/** URL query string parameters for the "upload file" endpoint. */ +export interface UploadFileRequestParams { + fileId?: string + fileName?: string + parentDirectoryId?: DirectoryId +} + +/** URL query string parameters for the "list tags" endpoint. */ +export interface ListTagsRequestParams { + tagType: TagObjectType +} + +/** URL query string parameters for the "list versions" endpoint. */ +export interface ListVersionsRequestParams { + versionType: VersionType + default: boolean +} + +/** HTTP response body for the "list projects" endpoint. */ +interface ListDirectoryResponseBody { + assets: BaseAsset[] +} + +/** HTTP response body for the "list projects" endpoint. */ +interface ListProjectsResponseBody { + projects: ListedProject[] +} + +/** HTTP response body for the "list files" endpoint. */ +interface ListFilesResponseBody { + files: File[] +} + +/** HTTP response body for the "list secrets" endpoint. */ +interface ListSecretsResponseBody { + secrets: SecretInfo[] +} + +/** HTTP response body for the "list tag" endpoint. */ +interface ListTagsResponseBody { + tags: Tag[] +} + +/** HTTP response body for the "list versions" endpoint. */ +interface ListVersionsResponseBody { + versions: Version[] +} + +// =================== +// === Type guards === +// =================== + +export function assetIsType(type: Type) { + return (asset: Asset): asset is Asset => asset.type === type +} + +// =============== +// === Backend === +// =============== + +/** Class for sending requests to the Cloud backend API endpoints. */ +export class Backend { + /** Creates a new instance of the {@link Backend} API client. + * + * @throws An error if the `Authorization` header is not set on the given `client`. */ + constructor( + private readonly client: http.Client, + private readonly logger: loggerProvider.Logger + ) { + // All of our API endpoints are authenticated, so we expect the `Authorization` header to be + // set. + if (!this.client.defaultHeaders?.has('Authorization')) { + return this.throw('Authorization header not set.') + } else { + return + } + } + + throw(message: string): never { + this.logger.error(message) + throw new Error(message) + } + + /** Sets the username of the current user, on the Cloud backend API. */ + async createUser(body: CreateUserRequestBody): Promise { + const response = await this.post(CREATE_USER_PATH, body) + return await response.json() + } + + /** Returns organization info for the current user, from the Cloud backend API. + * + * @returns `null` if status code 401 or 404 was received. */ + async usersMe(): Promise { + const response = await this.get(USERS_ME_PATH) + if (response.status !== STATUS_OK) { + return null + } else { + return await response.json() + } + } + + /** Returns a list of assets in a directory, from the Cloud backend API. + * + * @throws An error if status code 401 or 404 was received. + */ + async listDirectory(query: ListDirectoryRequestParams): Promise { + const response = await this.get( + LIST_DIRECTORY_PATH + + '?' + + new URLSearchParams({ + // eslint-disable-next-line @typescript-eslint/naming-convention + ...(query.parentId ? { parent_id: query.parentId } : {}), + }).toString() + ) + if (response.status !== STATUS_OK) { + if (query.parentId) { + return this.throw(`Unable to list directory with ID '${query.parentId}'.`) + } else { + return this.throw('Unable to list root directory.') + } + } else { + return (await response.json()).assets.map( + // This type assertion is safe; it is only needed to convert `type` to a newtype. + // eslint-disable-next-line no-restricted-syntax + asset => ({ ...asset, type: asset.id.match(/^(.+?)-/)?.[1] } as Asset) + ) + } + } + + /** Creates a directory, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async createDirectory(body: CreateDirectoryRequestBody): Promise { + const response = await this.post(CREATE_DIRECTORY_PATH, body) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to create directory with name '${body.title}'.`) + } else { + return await response.json() + } + } + + /** Returns a list of projects belonging to the current user, from the Cloud backend API. + * + * @throws An error if status code 401 or 404 was received. + */ + async listProjects(): Promise { + const response = await this.get(LIST_PROJECTS_PATH) + if (response.status !== STATUS_OK) { + return this.throw('Unable to list projects.') + } else { + return (await response.json()).projects + } + } + + /** Creates a project for the current user, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async createProject(body: CreateProjectRequestBody): Promise { + const response = await this.post(CREATE_PROJECT_PATH, body) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to create project with name '${body.projectName}'.`) + } else { + return await response.json() + } + } + + /** Closes the project identified by the given project ID, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async closeProject(projectId: ProjectId): Promise { + const response = await this.post(closeProjectPath(projectId), {}) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to close project with ID '${projectId}'.`) + } else { + return + } + } + + /** Returns project details for the specified project ID, from the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async getProjectDetails(projectId: ProjectId): Promise { + const response = await this.get(getProjectDetailsPath(projectId)) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to get details of project with ID '${projectId}'.`) + } else { + return await response.json() + } + } + + /** Sets project to an open state, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async openProject( + projectId: ProjectId, + body: OpenProjectRequestBody = DEFAULT_OPEN_PROJECT_BODY + ): Promise { + const response = await this.post(openProjectPath(projectId), body) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to open project with ID '${projectId}'.`) + } else { + return + } + } + + async projectUpdate( + projectId: ProjectId, + body: ProjectUpdateRequestBody + ): Promise { + const response = await this.put(projectUpdatePath(projectId), body) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to update project with ID '${projectId}'.`) + } else { + return await response.json() + } + } + + /** Deletes project, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async deleteProject(projectId: ProjectId): Promise { + const response = await this.delete(deleteProjectPath(projectId)) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to delete project with ID '${projectId}'.`) + } else { + return + } + } + + /** Returns project memory, processor and storage usage, from the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async checkResources(projectId: ProjectId): Promise { + const response = await this.get(checkResourcesPath(projectId)) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to get resource usage for project with ID '${projectId}'.`) + } else { + return await response.json() + } + } + + /** Returns a list of files accessible by the current user, from the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async listFiles(): Promise { + const response = await this.get(LIST_FILES_PATH) + if (response.status !== STATUS_OK) { + return this.throw('Unable to list files.') + } else { + return (await response.json()).files + } + } + + /** Uploads a file, to the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async uploadFile(params: UploadFileRequestParams, body: Blob): Promise { + const response = await this.postBase64( + UPLOAD_FILE_PATH + + '?' + + new URLSearchParams({ + /* eslint-disable @typescript-eslint/naming-convention */ + ...(params.fileName ? { file_name: params.fileName } : {}), + ...(params.fileId ? { file_id: params.fileId } : {}), + ...(params.parentDirectoryId + ? { parent_directory_id: params.parentDirectoryId } + : {}), + /* eslint-enable @typescript-eslint/naming-convention */ + }).toString(), + body + ) + if (response.status !== STATUS_OK) { + if (params.fileName) { + return this.throw(`Unable to upload file with name '${params.fileName}'.`) + } else if (params.fileId) { + return this.throw(`Unable to upload file with ID '${params.fileId}'.`) + } else { + return this.throw('Unable to upload file.') + } + } else { + return await response.json() + } + } + + /** Deletes a file, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async deleteFile(fileId: FileId): Promise { + const response = await this.delete(deleteFilePath(fileId)) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to delete file with ID '${fileId}'.`) + } else { + return + } + } + + /** Creates a secret environment variable, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async createSecret(body: CreateSecretRequestBody): Promise { + const response = await this.post(CREATE_SECRET_PATH, body) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to create secret with name '${body.secretName}'.`) + } else { + return await response.json() + } + } + + /** Returns a secret environment variable, from the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async getSecret(secretId: SecretId): Promise { + const response = await this.get(getSecretPath(secretId)) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to get secret with ID '${secretId}'.`) + } else { + return await response.json() + } + } + + /** Returns the secret environment variables accessible by the user, from the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async listSecrets(): Promise { + const response = await this.get(LIST_SECRETS_PATH) + if (response.status !== STATUS_OK) { + return this.throw('Unable to list secrets.') + } else { + return (await response.json()).secrets + } + } + + /** Deletes a secret environment variable, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async deleteSecret(secretId: SecretId): Promise { + const response = await this.delete(deleteSecretPath(secretId)) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to delete secret with ID '${secretId}'.`) + } else { + return + } + } + + /** Creates a file tag or project tag, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async createTag(body: CreateTagRequestBody): Promise { + const response = await this.post(CREATE_TAG_PATH, { + /* eslint-disable @typescript-eslint/naming-convention */ + tag_name: body.name, + tag_value: body.value, + object_type: body.objectType, + object_id: body.objectId, + /* eslint-enable @typescript-eslint/naming-convention */ + }) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to create create tag with name '${body.name}'.`) + } else { + return await response.json() + } + } + + /** Returns file tags or project tags accessible by the user, from the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async listTags(params: ListTagsRequestParams): Promise { + const response = await this.get( + LIST_TAGS_PATH + + '?' + + new URLSearchParams({ + // eslint-disable-next-line @typescript-eslint/naming-convention + tag_type: params.tagType, + }).toString() + ) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to list tags of type '${params.tagType}'.`) + } else { + return (await response.json()).tags + } + } + + /** Deletes a secret environment variable, on the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async deleteTag(tagId: TagId): Promise { + const response = await this.delete(deleteTagPath(tagId)) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to delete tag with ID '${tagId}'.`) + } else { + return + } + } + + /** Returns list of backend or IDE versions, from the Cloud backend API. + * + * @throws An error if a 401 or 404 status code was received. */ + async listVersions(params: ListVersionsRequestParams): Promise { + const response = await this.get( + LIST_VERSIONS_PATH + + '?' + + new URLSearchParams({ + // eslint-disable-next-line @typescript-eslint/naming-convention + version_type: params.versionType, + default: String(params.default), + }).toString() + ) + if (response.status !== STATUS_OK) { + return this.throw(`Unable to list versions of type '${params.versionType}'.`) + } else { + return (await response.json()).versions + } + } + + /** Sends an HTTP GET request to the given path. */ + private get(path: string) { + return this.client.get(`${config.ACTIVE_CONFIG.apiUrl}/${path}`) + } + + /** Sends a JSON HTTP POST request to the given path. */ + private post(path: string, payload: object) { + return this.client.post(`${config.ACTIVE_CONFIG.apiUrl}/${path}`, payload) + } + + /** Sends a binary HTTP POST request to the given path. */ + private postBase64(path: string, payload: Blob) { + return this.client.postBase64(`${config.ACTIVE_CONFIG.apiUrl}/${path}`, payload) + } + + /** Sends a JSON HTTP PUT request to the given path. */ + private put(path: string, payload: object) { + return this.client.put(`${config.ACTIVE_CONFIG.apiUrl}/${path}`, payload) + } + + /** Sends an HTTP DELETE request to the given path. */ + private delete(path: string) { + return this.client.delete(`${config.ACTIVE_CONFIG.apiUrl}/${path}`) + } +} + +// ===================== +// === createBackend === +// ===================== + +/** Shorthand method for creating a new instance of the backend API, along with the necessary + * headers. */ +/* TODO [NP]: https://github.com/enso-org/cloud-v2/issues/343 + * This is a hack to quickly create the backend in the format we want, until we get the provider + * working. This should be removed entirely in favour of creating the backend once and using it from + * the context. */ +export function createBackend(accessToken: string, logger: loggerProvider.Logger): Backend { + const headers = new Headers() + headers.append('Authorization', `Bearer ${accessToken}`) + const client = new http.Client(headers) + return new Backend(client, logger) +} diff --git a/app/ide-desktop/lib/dashboard/src/authentication/src/dashboard/service.tsx b/app/ide-desktop/lib/dashboard/src/authentication/src/dashboard/service.tsx deleted file mode 100644 index 010184c96902..000000000000 --- a/app/ide-desktop/lib/dashboard/src/authentication/src/dashboard/service.tsx +++ /dev/null @@ -1,110 +0,0 @@ -/** @file Module containing the API client for the Cloud backend API. - * - * Each exported function in the {@link Backend} in this module corresponds to an API endpoint. The - * functions are asynchronous and return a `Promise` that resolves to the response from the API. */ -import * as config from '../config' -import * as http from '../http' -import * as loggerProvider from '../providers/logger' - -// ================= -// === Constants === -// ================= - -/** Relative HTTP path to the "set username" endpoint of the Cloud backend API. */ -const SET_USER_NAME_PATH = 'users' -/** Relative HTTP path to the "get user" endpoint of the Cloud backend API. */ -const GET_USER_PATH = 'users/me' - -// ============= -// === Types === -// ============= - -/** A user/organization in the application. These are the primary owners of a project. */ -export interface Organization { - id: string - userEmail: string - name: string -} - -/** HTTP request body for the "set username" endpoint. */ -export interface SetUsernameRequestBody { - userName: string - userEmail: string -} - -// =============== -// === Backend === -// =============== - -/** Class for sending requests to the Cloud backend API endpoints. */ -export class Backend { - /** Creates a new instance of the {@link Backend} API client. - * - * @throws An error if the `Authorization` header is not set on the given `client`. */ - constructor( - private readonly client: http.Client, - private readonly logger: loggerProvider.Logger - ) { - /** All of our API endpoints are authenticated, so we expect the `Authorization` header to be - * set. */ - if (!this.client.defaultHeaders?.has('Authorization')) { - throw new Error('Authorization header not set.') - } - } - - /** Returns a {@link RequestBuilder} for an HTTP GET request to the given path. */ - get(path: string) { - return this.client.get(`${config.ACTIVE_CONFIG.apiUrl}/${path}`) - } - - /** Returns a {@link RequestBuilder} for an HTTP POST request to the given path. */ - post(path: string, payload: object) { - return this.client.post(`${config.ACTIVE_CONFIG.apiUrl}/${path}`, payload) - } - - /** Logs the error that occurred and throws a new one with a more user-friendly message. */ - errorHandler(message: string) { - return (error: Error) => { - this.logger.error(error.message) - throw new Error(message) - } - } - - /** Sets the username of the current user, on the Cloud backend API. */ - setUsername(body: SetUsernameRequestBody): Promise { - return this.post(SET_USER_NAME_PATH, body).then(response => response.json()) - } - - /** Returns organization info for the current user, from the Cloud backend API. - * - * @returns `null` if status code 401 or 404 was received. */ - getUser(): Promise { - return this.get(GET_USER_PATH).then(response => { - if ( - response.status === http.HttpStatus.unauthorized || - response.status === http.HttpStatus.notFound - ) { - return null - } else { - return response.json() - } - }) - } -} - -// ===================== -// === createBackend === -// ===================== - -/** Shorthand method for creating a new instance of the backend API, along with the necessary - * headers. */ -/* TODO [NP]: https://github.com/enso-org/cloud-v2/issues/343 - * This is a hack to quickly create the backend in the format we want, until we get the provider - * working. This should be removed entirely in favour of creating the backend once and using it from - * the context. */ -export function createBackend(accessToken: string, logger: loggerProvider.Logger): Backend { - const headers = new Headers() - headers.append('Authorization', `Bearer ${accessToken}`) - const client = new http.Client(headers) - return new Backend(client, logger) -} diff --git a/app/ide-desktop/lib/dashboard/src/authentication/src/http.tsx b/app/ide-desktop/lib/dashboard/src/authentication/src/http.tsx index 930a28031be9..95dbeb1c3e8f 100644 --- a/app/ide-desktop/lib/dashboard/src/authentication/src/http.tsx +++ b/app/ide-desktop/lib/dashboard/src/authentication/src/http.tsx @@ -2,18 +2,6 @@ * * Used to build authenticated clients for external APIs, like our Cloud backend API. */ -// ================== -// === HttpStatus === -// ================== - -/** HTTP status codes returned in a HTTP response. */ -export enum HttpStatus { - // eslint-disable-next-line @typescript-eslint/no-magic-numbers - unauthorized = 401, - // eslint-disable-next-line @typescript-eslint/no-magic-numbers - notFound = 404, -} - // ================== // === HttpMethod === // ================== diff --git a/app/ide-desktop/lib/dashboard/src/authentication/src/newtype.ts b/app/ide-desktop/lib/dashboard/src/authentication/src/newtype.ts new file mode 100644 index 000000000000..b3a85ec61518 --- /dev/null +++ b/app/ide-desktop/lib/dashboard/src/authentication/src/newtype.ts @@ -0,0 +1,39 @@ +/** @file TypeScript's closest equivalent of `newtype`s. */ + +interface NewtypeVariant { + // eslint-disable-next-line @typescript-eslint/naming-convention + _$type: TypeName +} + +/** Used to create a "branded type", + * which contains a property that only exists at compile time. + * + * `Newtype` and `Newtype` are not compatible with each other, + * however both are regular `string`s at runtime. + * + * This is useful in parameters that require values from a certain source, + * for example IDs for a specific object type. + * + * It is similar to a `newtype` in other languages. + * Note however because TypeScript is structurally typed, + * a branded type is assignable to its base type: + * `a: string = asNewtype>(b)` successfully typechecks. */ +export type Newtype = NewtypeVariant & T + +interface NotNewtype { + // eslint-disable-next-line @typescript-eslint/naming-convention + _$type?: never +} + +export function asNewtype>( + s: NotNewtype & Omit +): T { + // This cast is unsafe. + // `T` has an extra property `_$type` which is used purely for typechecking + // and does not exist at runtime. + // + // The property name is specifically chosen to trigger eslint's `naming-convention` lint, + // so it should not be possible to accidentally create a value with such a type. + // eslint-disable-next-line no-restricted-syntax + return s as unknown as T +} diff --git a/app/ide-desktop/lib/dashboard/src/authentication/src/utils.ts b/app/ide-desktop/lib/dashboard/src/authentication/src/utils.ts index 31a97a322b8d..a0c293587c5a 100644 --- a/app/ide-desktop/lib/dashboard/src/authentication/src/utils.ts +++ b/app/ide-desktop/lib/dashboard/src/authentication/src/utils.ts @@ -7,31 +7,3 @@ export function handleEvent(callback: () => Promise) { await callback() } } - -/** Used to create a "branded type", which is a type intersected with a `Brand<'Name'>`. - * - * This is useful in parameters that require values from a certain source, - * for example IDs for a specific object type. - * - * It is similar to a `newtype` in other languages, - * however because TypeScript is structurally typed, a branded type is assignable to its base type: - * `a: string = b as (string & Brand<'Name'>)` is valid. */ -export interface Brand { - $brand: T -} - -interface NoBrand { - $brand?: never -} - -export function brand>(s: NoBrand & Omit): T { - // This cast is unsafe. It is possible to use this method to cast a value from a base type to a - // branded type, even if that value is not an instance of the branded type. For example, the - // string "foo" could be cast to the `UserPoolId` branded type, although this string is clearly - // not a valid `UserPoolId`. This is acceptable because the branded type is only used to prevent - // accidental misuse of values, and not to enforce correctness. That is, it is up to the - // programmer to declare the correct type of a value. After that point, it is up to the branded - // type to keep that guarantee by preventing accidental misuse of the value. - // eslint-disable-next-line no-restricted-syntax - return s as unknown as T -} diff --git a/app/ide-desktop/lib/types/globals.d.ts b/app/ide-desktop/lib/types/globals.d.ts index face54c28838..01b2e9d1ba93 100644 --- a/app/ide-desktop/lib/types/globals.d.ts +++ b/app/ide-desktop/lib/types/globals.d.ts @@ -60,7 +60,7 @@ declare global { const BUNDLED_ENGINE_VERSION: string const BUILD_INFO: BuildInfo // eslint-disable-next-line no-restricted-syntax - const PROJECT_MANAGER_IN_BUNDLE_PATH: string | undefined + const PROJECT_MANAGER_IN_BUNDLE_PATH: string const IS_DEV_MODE: boolean /* eslint-disable @typescript-eslint/naming-convention */ } diff --git a/app/ide-desktop/package-lock.json b/app/ide-desktop/package-lock.json index b384a73b73bb..13059399c42c 100644 --- a/app/ide-desktop/package-lock.json +++ b/app/ide-desktop/package-lock.json @@ -31,12 +31,14 @@ "dependencies": { "@types/mime-types": "^2.1.1", "@types/opener": "^1.4.0", + "@types/tar": "^6.1.4", "chalk": "^5.2.0", "create-servers": "^3.2.0", "electron-is-dev": "^2.0.0", "mime-types": "^2.1.35", "opener": "^1.5.2", "string-length": "^5.0.1", + "tar": "^6.1.13", "yargs": "17.6.2" }, "devDependencies": { @@ -4688,6 +4690,15 @@ "license": "MIT", "peer": true }, + "node_modules/@types/tar": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@types/tar/-/tar-6.1.4.tgz", + "integrity": "sha512-Cp4oxpfIzWt7mr2pbhHT2OTXGMAL0szYCzuf8lRWyIMCgsx6/Hfc3ubztuhvzXHXgraTQxyOCmmg7TDGIMIJJQ==", + "dependencies": { + "@types/node": "*", + "minipass": "^4.0.0" + } + }, "node_modules/@types/to-ico": { "version": "1.1.1", "dev": true, @@ -8404,6 +8415,28 @@ "node": ">=10" } }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "license": "ISC" @@ -11420,6 +11453,37 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.5.tgz", + "integrity": "sha512-+yQl7SX3bIT83Lhb4BVorMAHVuqsskxRdlmO9kTpyukp8vsm2Sn/fUOV9xlnG8/a5JsypJzap21lz/y3FBMJ8Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/mixin-deep": { "version": "1.3.2", "license": "MIT", @@ -14560,6 +14624,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/tar": { + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^4.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/tar-fs": { "version": "2.1.1", "dev": true, @@ -14586,6 +14666,25 @@ "node": ">=6" } }, + "node_modules/tar/node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "engines": { + "node": ">=10" + } + }, + "node_modules/tar/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/temp": { "version": "0.8.3", "engines": [ @@ -15601,7 +15700,6 @@ }, "node_modules/yallist": { "version": "4.0.0", - "dev": true, "license": "ISC" }, "node_modules/yaml": { @@ -18574,6 +18672,15 @@ "version": "2.0.1", "peer": true }, + "@types/tar": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@types/tar/-/tar-6.1.4.tgz", + "integrity": "sha512-Cp4oxpfIzWt7mr2pbhHT2OTXGMAL0szYCzuf8lRWyIMCgsx6/Hfc3ubztuhvzXHXgraTQxyOCmmg7TDGIMIJJQ==", + "requires": { + "@types/node": "*", + "minipass": "^4.0.0" + } + }, "@types/to-ico": { "version": "1.1.1", "dev": true, @@ -20121,6 +20228,7 @@ "@esbuild/windows-x64": "^0.17.0", "@types/mime-types": "^2.1.1", "@types/opener": "^1.4.0", + "@types/tar": "^6.1.4", "chalk": "^5.2.0", "create-servers": "^3.2.0", "crypto-js": "4.1.1", @@ -20137,6 +20245,7 @@ "opener": "^1.5.2", "portfinder": "^1.0.32", "string-length": "^5.0.1", + "tar": "^6.1.13", "tsx": "^3.12.6", "yargs": "17.6.2" }, @@ -21327,6 +21436,24 @@ "universalify": "^2.0.0" } }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "requires": { + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "requires": { + "yallist": "^4.0.0" + } + } + } + }, "fs.realpath": { "version": "1.0.0" }, @@ -23376,6 +23503,30 @@ "minimist": { "version": "1.2.7" }, + "minipass": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.5.tgz", + "integrity": "sha512-+yQl7SX3bIT83Lhb4BVorMAHVuqsskxRdlmO9kTpyukp8vsm2Sn/fUOV9xlnG8/a5JsypJzap21lz/y3FBMJ8Q==" + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "requires": { + "yallist": "^4.0.0" + } + } + } + }, "mixin-deep": { "version": "1.3.2", "peer": true, @@ -25394,6 +25545,31 @@ } } }, + "tar": { + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==", + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^4.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "dependencies": { + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + } + } + }, "tar-fs": { "version": "2.1.1", "dev": true, @@ -26100,8 +26276,7 @@ "version": "5.0.8" }, "yallist": { - "version": "4.0.0", - "dev": true + "version": "4.0.0" }, "yaml": { "version": "1.10.2", diff --git a/app/ide-desktop/utils.ts b/app/ide-desktop/utils.ts index 9f296a15e3f9..c3944223b267 100644 --- a/app/ide-desktop/utils.ts +++ b/app/ide-desktop/utils.ts @@ -3,6 +3,17 @@ import * as fs from 'node:fs' import * as path from 'node:path' import process from 'node:process' +// ================= +// === Constants === +// ================= + +/** Indent size for outputting JSON. */ +export const INDENT_SIZE = 4 + +// =================== +// === Environment === +// =================== + /** * Get the environment variable value. * @@ -45,3 +56,16 @@ export function requireEnvPathExist(name: string) { throw Error(`File with path ${value} read from environment variable ${name} is missing.`) } } + +// ====================== +// === String Helpers === +// ====================== + +/** Get the common prefix of the two strings. */ +export function getCommonPrefix(a: string, b: string): string { + let i = 0 + while (i < a.length && i < b.length && a[i] === b[i]) { + i++ + } + return a.slice(0, i) +} diff --git a/build.sbt b/build.sbt index 2ca97da93ab1..fd55d754097a 100644 --- a/build.sbt +++ b/build.sbt @@ -474,8 +474,8 @@ val scalameterVersion = "0.19" val scalatestVersion = "3.3.0-SNAP3" val shapelessVersion = "2.4.0-M1" val slf4jVersion = "1.7.36" -val slickVersion = "3.3.3" -val sqliteVersion = "3.36.0.3" +val slickVersion = "3.4.1" +val sqliteVersion = "3.41.2.1" val tikaVersion = "2.4.1" val typesafeConfigVersion = "1.4.2" val junitVersion = "4.13.2" @@ -2039,7 +2039,7 @@ lazy val `std-database` = project `database-polyglot-root` / "std-database.jar", libraryDependencies ++= Seq( "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "org.xerial" % "sqlite-jdbc" % "3.36.0.3", + "org.xerial" % "sqlite-jdbc" % sqliteVersion, "org.postgresql" % "postgresql" % "42.4.0", "com.amazon.redshift" % "redshift-jdbc42" % "2.1.0.9", "com.amazonaws" % "aws-java-sdk-core" % "1.12.273", diff --git a/build/base/src/fs/wrappers.rs b/build/base/src/fs/wrappers.rs index 71ed4647b4a0..5f9794c2ed41 100644 --- a/build/base/src/fs/wrappers.rs +++ b/build/base/src/fs/wrappers.rs @@ -5,6 +5,7 @@ use crate::prelude::*; +use std::fs::DirEntry; use std::fs::File; use std::fs::Metadata; @@ -20,6 +21,12 @@ pub fn metadata>(path: P) -> Result { std::fs::metadata(&path).anyhow_err() } +/// See [std::fs::symlink_metadata]. +#[context("Failed to obtain symlink metadata for file: {}", path.as_ref().display())] +pub fn symlink_metadata>(path: P) -> Result { + std::fs::symlink_metadata(&path).anyhow_err() +} + /// See [std::fs::copy]. #[context("Failed to copy file from {} to {}", from.as_ref().display(), to.as_ref().display())] pub fn copy(from: impl AsRef, to: impl AsRef) -> Result { @@ -39,9 +46,15 @@ pub fn read(path: impl AsRef) -> Result> { } /// See [std::fs::read_dir]. -#[context("Failed to read the directory: {}", path.as_ref().display())] -pub fn read_dir(path: impl AsRef) -> Result { - std::fs::read_dir(&path).anyhow_err() +pub fn read_dir(path: impl AsRef) -> Result>> { + let path = path.as_ref().to_path_buf(); + let read_dir = std::fs::read_dir(&path) + .map_err(|e| anyhow!("Failed to read the directory: '{}'. Error: {}", path.display(), e))?; + Ok(read_dir.into_iter().map(move |elem_result| { + elem_result.map_err(|e| { + anyhow!("Failed to read sub-item from the directory '{}'. Error: {}", path.display(), e) + }) + })) } /// See [std::fs::read_to_string]. diff --git a/build/build/src/ide/web.rs b/build/build/src/ide/web.rs index 16a8e1e7924d..de6511226d70 100644 --- a/build/build/src/ide/web.rs +++ b/build/build/src/ide/web.rs @@ -393,7 +393,7 @@ impl IdeDesktop { let icons_build = self.build_icons(&icons_dist); let (icons, _content) = try_join(icons_build, client_build).await?; - let python_path = if TARGET_OS == OS::MacOS { + let python_path = if TARGET_OS == OS::MacOS && !env::PYTHON_PATH.is_set() { // On macOS electron-builder will fail during DMG creation if there is no python2 // installed. It is looked for in `/usr/bin/python` which is not valid place on newer // MacOS versions. diff --git a/build/build/src/project/backend.rs b/build/build/src/project/backend.rs index b71af472220d..633dae0c1e89 100644 --- a/build/build/src/project/backend.rs +++ b/build/build/src/project/backend.rs @@ -88,8 +88,8 @@ pub async fn bundled_engine_versions( let mut ret = vec![]; let mut dir_reader = ide_ci::fs::tokio::read_dir(&project_manager_bundle.dist).await?; - while let Some(entry) = dir_reader.next_entry().await? { - if entry.metadata().await?.is_dir() { + while let Some(entry) = dir_reader.try_next().await? { + if ide_ci::fs::tokio::metadata(&entry.path()).await?.is_dir() { ret.push(Version::from_str(entry.file_name().as_str())?); } } diff --git a/build/ci_utils/Cargo.toml b/build/ci_utils/Cargo.toml index 0e1467022ab8..4ec09d4079c7 100644 --- a/build/ci_utils/Cargo.toml +++ b/build/ci_utils/Cargo.toml @@ -72,6 +72,7 @@ sysinfo = "0.26.2" tar = "0.4.37" tempfile = "3.2.0" tokio = { workspace = true } +tokio-stream = { workspace = true } tokio-util = { workspace = true } tracing = { version = "0.1.37" } tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } diff --git a/build/ci_utils/src/fs/wrappers/tokio.rs b/build/ci_utils/src/fs/wrappers/tokio.rs index 287bc9253fc3..77a048d89271 100644 --- a/build/ci_utils/src/fs/wrappers/tokio.rs +++ b/build/ci_utils/src/fs/wrappers/tokio.rs @@ -11,6 +11,16 @@ pub fn metadata>(path: P) -> BoxFuture<'static, Result>(path: P) -> BoxFuture<'static, Result> { + let path = path.as_ref().to_owned(); + tokio::fs::symlink_metadata(path.clone()) + .with_context(move || { + format!("Failed to obtain symlink metadata for file: {}", path.display()) + }) + .boxed() +} + #[context("Failed to open path for reading: {}", path.as_ref().display())] pub async fn open(path: impl AsRef) -> Result { File::open(&path).await.anyhow_err() @@ -38,9 +48,20 @@ pub async fn create_dir_all(path: impl AsRef) -> Result { tokio::fs::create_dir_all(&path).await.anyhow_err() } -#[context("Failed to read the directory: {}", path.as_ref().display())] -pub async fn read_dir(path: impl AsRef) -> Result { - tokio::fs::read_dir(&path).await.anyhow_err() +pub async fn read_dir( + path: impl AsRef, +) -> Result>> { + let path = path.as_ref().to_path_buf(); + let read_dir = tokio::fs::read_dir(&path) + .await + .with_context(|| format!("Failed to read the directory: {}", path.display()))?; + + let stream = tokio_stream::wrappers::ReadDirStream::new(read_dir); + Ok(stream.map(move |entry| { + entry.with_context(|| { + format!("Failed to get entry when reading the directory: {}", path.display()) + }) + })) } #[context("Failed to remove directory with the subtree: {}", path.as_ref().display())] diff --git a/build/enso-formatter/src/lib.rs b/build/enso-formatter/src/lib.rs index 64ca45f8fd79..634b09418edb 100644 --- a/build/enso-formatter/src/lib.rs +++ b/build/enso-formatter/src/lib.rs @@ -367,8 +367,9 @@ pub struct RustSourcePath { /// uses non-documented API and is slow as well (8 seconds for the whole codebase). It should be /// possible to improve the latter solution to get good performance, but it seems way harder than it /// should be. -pub async fn process_path(path: impl AsRef, action: Action) -> Result { - let paths = discover_paths(&path)?; +#[context("Enso Formatter: failed to process root path '{}'.", path.as_ref().display())] +pub async fn process_path(path: impl AsRef + Copy, action: Action) -> Result { + let paths = discover_paths(path)?; let total = paths.len(); let mut hash_map = HashMap::::new(); for (i, sub_path) in paths.iter().enumerate() { @@ -396,12 +397,14 @@ pub async fn process_path(path: impl AsRef, action: Action) -> Result { } /// Discover all paths containing Rust sources, recursively. +#[context("Discovering Rust paths failed for '{}' failed.", path.as_ref().display())] pub fn discover_paths(path: impl AsRef) -> Result> { let mut vec = Vec::default(); - discover_paths_internal(&mut vec, path, false)?; + discover_paths_internal(&mut vec, &path, false)?; Ok(vec) } +#[context("Discovering paths failed for '{}' failed.", path.as_ref().display())] pub fn discover_paths_internal( vec: &mut Vec, path: impl AsRef, @@ -409,7 +412,11 @@ pub fn discover_paths_internal( ) -> Result { use ide_ci::fs; let path = path.as_ref(); - let md = fs::metadata(path)?; + // Below we use `symlink_metadata` instead of `metadata` because the latter follows symlinks. + // We don't want the formatter to fail if it encounters a symlink to a non-existing file. + // All files to be formatted should be reachable from the repository root without following + // any symlinks. + let md = fs::symlink_metadata(path)?; if md.is_dir() && !path.file_name().contains(&"target") { let dir_name = path.file_name(); // FIXME: This should cover 'tests' folder also, but only the files that contain actual diff --git a/distribution/engine/THIRD-PARTY/NOTICE b/distribution/engine/THIRD-PARTY/NOTICE index 64c485ed4fb7..347630b20491 100644 --- a/distribution/engine/THIRD-PARTY/NOTICE +++ b/distribution/engine/THIRD-PARTY/NOTICE @@ -183,7 +183,7 @@ Copyright notices related to this dependency can be found in the directory `com. 'slick_2.13', licensed under the Two-clause BSD-style license, is distributed with the engine. The license information can be found along with the copyright notices. -Copyright notices related to this dependency can be found in the directory `com.typesafe.slick.slick_2.13-3.3.3`. +Copyright notices related to this dependency can be found in the directory `com.typesafe.slick.slick_2.13-3.4.1`. 'ssl-config-core_2.13', licensed under the Apache-2.0, is distributed with the engine. @@ -356,9 +356,14 @@ The license file can be found at `licenses/CC0`. Copyright notices related to this dependency can be found in the directory `org.reactivestreams.reactive-streams-1.0.3`. +'reactive-streams', licensed under the MIT-0, is distributed with the engine. +The license file can be found at `licenses/MIT-0`. +Copyright notices related to this dependency can be found in the directory `org.reactivestreams.reactive-streams-1.0.4`. + + 'scala-collection-compat_2.13', licensed under the Apache-2.0, is distributed with the engine. The license file can be found at `licenses/APACHE2.0`. -Copyright notices related to this dependency can be found in the directory `org.scala-lang.modules.scala-collection-compat_2.13-2.0.0`. +Copyright notices related to this dependency can be found in the directory `org.scala-lang.modules.scala-collection-compat_2.13-2.8.1`. 'scala-java8-compat_2.13', licensed under the Apache-2.0, is distributed with the engine. @@ -433,7 +438,7 @@ Copyright notices related to this dependency can be found in the directory `org. 'sqlite-jdbc', licensed under the The Apache Software License, Version 2.0, is distributed with the engine. The license information can be found along with the copyright notices. -Copyright notices related to this dependency can be found in the directory `org.xerial.sqlite-jdbc-3.36.0.3`. +Copyright notices related to this dependency can be found in the directory `org.xerial.sqlite-jdbc-3.41.2.1`. 'snakeyaml', licensed under the Apache License, Version 2.0, is distributed with the engine. diff --git a/distribution/engine/THIRD-PARTY/com.typesafe.slick.slick_2.13-3.3.3/LICENSE.txt b/distribution/engine/THIRD-PARTY/com.typesafe.slick.slick_2.13-3.4.1/LICENSE.txt similarity index 97% rename from distribution/engine/THIRD-PARTY/com.typesafe.slick.slick_2.13-3.3.3/LICENSE.txt rename to distribution/engine/THIRD-PARTY/com.typesafe.slick.slick_2.13-3.4.1/LICENSE.txt index 05b7a25dbccd..0fb1082e7314 100644 --- a/distribution/engine/THIRD-PARTY/com.typesafe.slick.slick_2.13-3.3.3/LICENSE.txt +++ b/distribution/engine/THIRD-PARTY/com.typesafe.slick.slick_2.13-3.4.1/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright 2011-2016 Typesafe, Inc. +Copyright 2011-2021 Lightbend, Inc. All rights reserved. diff --git a/distribution/engine/THIRD-PARTY/licenses/MIT-0 b/distribution/engine/THIRD-PARTY/licenses/MIT-0 new file mode 100644 index 000000000000..a4e9dc906188 --- /dev/null +++ b/distribution/engine/THIRD-PARTY/licenses/MIT-0 @@ -0,0 +1,16 @@ +MIT No Attribution + +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.4/NOTICES b/distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.4/NOTICES new file mode 100644 index 000000000000..bd0a85c0c769 --- /dev/null +++ b/distribution/engine/THIRD-PARTY/org.reactivestreams.reactive-streams-1.0.4/NOTICES @@ -0,0 +1,10 @@ +/************************************************************************ + * Licensed under Public Domain (CC0) * + * * + * To the extent possible under law, the person who associated CC0 with * + * this code has waived all copyright and related or neighboring * + * rights to this code. * + * * + * You should have received a copy of the CC0 legalcode along with this * + * work. If not, see .* + ************************************************************************/ diff --git a/distribution/engine/THIRD-PARTY/org.scala-lang.modules.scala-collection-compat_2.13-2.0.0/NOTICES b/distribution/engine/THIRD-PARTY/org.scala-lang.modules.scala-collection-compat_2.13-2.8.1/NOTICES similarity index 100% rename from distribution/engine/THIRD-PARTY/org.scala-lang.modules.scala-collection-compat_2.13-2.0.0/NOTICES rename to distribution/engine/THIRD-PARTY/org.scala-lang.modules.scala-collection-compat_2.13-2.8.1/NOTICES diff --git a/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/NOTICES b/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/NOTICES deleted file mode 100644 index 06d71bd94d4a..000000000000 --- a/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/NOTICES +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2007 David Crawshaw - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - - -/*-------------------------------------------------------------------------- - * Copyright 2007 Taro L. Saito - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ - -/*-------------------------------------------------------------------------- - * Copyright 2016 Magnus Reftel - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *--------------------------------------------------------------------------*/ - -Copyright (c) 2021 Gauthier Roebroeck - -Copyright 2008 Taro L. Saito - -Copyright 2009 Taro L. Saito - -Copyright 2010 Taro L. Saito diff --git a/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/LICENSE b/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/LICENSE similarity index 100% rename from distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/LICENSE rename to distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/LICENSE diff --git a/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/LICENSE.zentus b/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/LICENSE.zentus similarity index 100% rename from distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/LICENSE.zentus rename to distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/LICENSE.zentus diff --git a/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/NOTICES b/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/NOTICES new file mode 100644 index 000000000000..f149f0d20a8e --- /dev/null +++ b/distribution/engine/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/NOTICES @@ -0,0 +1,11 @@ +Copyright (c) 2007 David Crawshaw + +Copyright (c) 2021 Gauthier Roebroeck + +Copyright 2007 Taro L. Saito + +Copyright 2016 Magnus Reftel + +copyright notice and this permission notice appear in all copies. + +this work for additional information regarding copyright ownership. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Any.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Any.enso index 1b88baa07f34..67455926fc36 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Any.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Any.enso @@ -49,7 +49,8 @@ type Any pretty : Text pretty self = @Builtin_Method "Any.pretty" - ## Generic conversion of an arbitrary Enso value to a corresponding short + ## PRIVATE + Generic conversion of an arbitrary Enso value to a corresponding short human-readable representation. > Example diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Array.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Array.enso index 9e80e81f3bf6..657c4e001d59 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Array.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Array.enso @@ -77,19 +77,20 @@ type Array new_4 : Any -> Any -> Any -> Any -> Array new_4 item_1 item_2 item_3 item_4 = @Builtin_Method "Array.new_4" - ## Copies from the source array, beginning at the specified position, to the + ## PRIVATE + Copies from the source array, beginning at the specified position, to the specified position in the destination array. Arguments: - src: The source array. - source_index: The start position in the src array. - - dest: The desination array. + - dest: The destination array. - dest_index: The start position in the that array. A subsequence of array elements are copied from the src array to the dest array. The number of components copied is equal to count. The components at positions source_index through source_index + count - 1 - in the strc array are copied into positions dest_index through + in the src array are copied into positions dest_index through dest_index + count - 1, respectively, of the destination array. If the src and dest arguments refer to the same array, then the copy @@ -150,7 +151,6 @@ type Array self.sort_builtin comparator ## Identity. - This method is implemented purely for completeness with the runtime's primitive array protocol. to_array : Array diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Array_Proxy.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Array_Proxy.enso index 3e1f7091b1a6..da0f8d37052e 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Array_Proxy.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Array_Proxy.enso @@ -3,7 +3,8 @@ import project.Data.Array.Array import project.Data.Numbers.Integer import project.Errors.Illegal_Argument.Illegal_Argument -## A helper type used for creating an array from a length and a callback +## PRIVATE + A helper type used for creating an array from a length and a callback providing its elements. It can be used to create an array from some non-standard underlying storage @@ -13,8 +14,8 @@ import project.Errors.Illegal_Argument.Illegal_Argument vector backed by such custom storage. @Builtin_Type type Array_Proxy - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED Creates a new `Array_Proxy` from a length and a callback. Arguments: @@ -38,8 +39,8 @@ type Array_Proxy new_builtin : Integer -> (Integer -> Any) -> Array new_builtin length at = @Builtin_Method "Array_Proxy.new_builtin" - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED Creates a new `Array_Proxy` from an object providing `length` and `at` methods. from_proxy_object : Any -> Array diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Filter_Condition.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Filter_Condition.enso index 7023cbb006ef..ff3303eef0fc 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Filter_Condition.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Filter_Condition.enso @@ -1,4 +1,5 @@ import project.Any.Any +import project.Data.Boolean.Boolean import project.Data.Text.Case_Sensitivity.Case_Sensitivity import project.Data.Text.Extensions import project.Data.Text.Regex @@ -157,6 +158,7 @@ type Filter_Condition ## Converts a `Filter_Condition` condition into a predicate taking an element and returning a value indicating whether the element should be accepted by the filter. + to_predicate : (Any -> Boolean) to_predicate self = case self of Less value -> <=value diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso index 54c9470d3960..fd71ad369db9 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso @@ -55,10 +55,10 @@ type Json Check the `message` field for detailed information on the specific failure. type Invalid_JSON + ## PRIVATE Error message ## PRIVATE - Converts the error to a display representation. to_display_text : Text to_display_text self = @@ -69,8 +69,7 @@ type Invalid_JSON A failure indicating the inability to marshall a `Json` object into the specified format. type Marshalling_Error - - ## UNSTABLE + ## PRIVATE The `json` object could not be converted into `format`, due to a type mismatch. @@ -82,8 +81,7 @@ type Marshalling_Error This can occur e.g. when trying to reinterpret a number as a `Text`, etc. Type_Mismatch json format - ## UNSTABLE - + ## PRIVATE The `json` object could not be converted into `format`, due to a field missing in the `json` structure. @@ -96,8 +94,7 @@ type Marshalling_Error when the JSON does not contain all the fields required by the atom. Missing_Field json field format - ## UNSTABLE - + ## PRIVATE Convert the marshalling error into a human-readable format. to_display_text : Text to_display_text self = case self of @@ -108,8 +105,10 @@ type Marshalling_Error Marshalling_Error.Missing_Field _ field _ -> "Missing field in Json: the field `" + field.to_text "` was missing in the json." +## PRIVATE type JS_Object - ## Creates a JS_Object from a list of key-value pairs. + ## PRIVATE + Creates a JS_Object from a list of key-value pairs. Keys must be `Text` values. Values will be recursively converted to JSON serializable as needed. from_pairs : Vector -> JS_Object @@ -163,11 +162,13 @@ type JS_Object proxy = Array_Proxy.new keys.length (i-> Pair.new (keys.at i) (self.get (keys.at i))) Vector.from_polyglot_array proxy - ## Convert the object to a JS_Object. + ## PRIVATE + Convert the object to a JS_Object. to_js_object : JS_Object to_js_object self = self - ## Convert to a Text. + ## PRIVATE + Convert to a Text. to_text : Text to_text self = Json.stringify self @@ -175,8 +176,7 @@ type JS_Object to_json : Text to_json self = self.to_text - ## UNSTABLE - + ## PRIVATE Transform the vector into text for displaying as part of its default visualization. to_default_visualization_data : Text @@ -185,6 +185,7 @@ type JS_Object ## PRIVATE type JS_Object_Comparator + ## PRIVATE compare : JS_Object -> JS_Object -> (Ordering|Nothing) compare obj1 obj2 = obj1_keys = obj1.field_names @@ -193,6 +194,7 @@ type JS_Object_Comparator (obj1.get key == obj2.at key).catch No_Such_Key _->False if same_values then Ordering.Equal else Nothing + ## PRIVATE hash : JS_Object -> Integer hash obj = values_hashes = obj.field_names.map field_name-> @@ -203,7 +205,6 @@ type JS_Object_Comparator Comparable.from (_:JS_Object) = JS_Object_Comparator - ## PRIVATE Render the JS_Object to Text with truncated depth. render object depth=0 max_depth=5 max_length=100 = case object of diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json/Extensions.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json/Extensions.enso index eb230954540e..f175cb8e94c4 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json/Extensions.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json/Extensions.enso @@ -28,7 +28,8 @@ Any.to_json self = Json.stringify self Error.to_json : Text Error.to_json self = self.to_js_object.to_text -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. For Nothing, booleans, numbers and strings, this is the value itself. For arrays or vectors, the elements are converted recursively. For atoms and maps, the object is converted to a JS_Object. @@ -37,7 +38,8 @@ Text.to_js_object self = case self of Text -> JS_Object.from_pairs [["type", "Text"]] _ -> self -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. For Nothing, booleans, numbers and strings, this is the value itself. For arrays or vectors, the elements are converted recursively. For atoms and maps, the object is converted to a JS_Object. @@ -53,7 +55,8 @@ Number.to_js_object self = case self of JS_Object.from_pairs [["type", "BigInt"], ["value", self.to_text]] _ -> self -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. For Nothing, booleans, numbers and strings, this is the value itself. For arrays or vectors, the elements are converted recursively. For atoms and maps, the object is converted to a JS_Object. @@ -62,14 +65,16 @@ Boolean.to_js_object self = case self of Boolean -> JS_Object.from_pairs [["type", "Boolean"]] _ -> self -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. For Nothing, booleans, numbers and strings, this is the value itself. For arrays or vectors, the elements are converted recursively. For atoms and maps, the object is converted to a JS_Object. Nothing.to_js_object : Nothing Nothing.to_js_object self = self -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. For Nothing, booleans, numbers and strings, this is the value itself. For arrays or vectors, the elements are converted recursively. For atoms and maps, the object is converted to a JS_Object. @@ -79,7 +84,8 @@ Array.to_js_object self = proxy = Array_Proxy.new stripped.length i-> stripped.at i . to_js_object Vector.from_polyglot_array proxy -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. For Nothing, booleans, numbers and strings, this is the value itself. For arrays or vectors, the elements are converted recursively. For atoms, the object is converted to a JS_Object. @@ -89,7 +95,8 @@ Vector.to_js_object self = proxy = Array_Proxy.new stripped.length i-> stripped.at i . to_js_object Vector.from_polyglot_array proxy -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. For Nothing, booleans, numbers and strings, this is the value itself. For arrays or vectors, the elements are converted recursively. For atoms, the object is converted to a JS_Object. @@ -115,7 +122,8 @@ Any.to_js_object self = JS_Object.from_pairs [["type", type_name], ["constructor", m.name]] _ -> Error.throw ("Cannot convert " + self.to_text + " to JSON") -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. For Nothing, booleans, numbers and strings, this is the value itself. For arrays or vectors, the elements are converted recursively. For atoms, the object is converted to a JS_Object. @@ -127,7 +135,8 @@ Error.to_js_object self = error_message = ["message", caught.to_display_text] JS_Object.from_pairs [error_type, error_content, error_message] -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. Custom serialization for Locale, serializes the language, country and variant. Locale.to_js_object : JS_Object Locale.to_js_object self = @@ -139,7 +148,8 @@ Locale.to_js_object self = b.append ["variant", self.variant] JS_Object.from_pairs b.to_vector -## Converts the given value to a JSON serializable object. +## PRIVATE + Converts the given value to a JSON serializable object. For Map, this is serialized as a Vector of Key-Value pairs. Enso Maps support arbitrary types as map keys, so we cannot serialize them into JS Objects because there only strings are accepted as keys. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/List.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/List.enso index 86f54e65cdf0..ab4cb3ff9df5 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/List.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/List.enso @@ -464,7 +464,8 @@ type List builder.append elem builder.to_vector - ## Generates a human-readable text representation of the list. + ## PRIVATE + Generates a human-readable text representation of the list. to_text : Text to_text self = go l t e = case l of diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Locale.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Locale.enso index 507005f4fb5e..38777dc3207a 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Locale.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Locale.enso @@ -38,8 +38,8 @@ type Locale java_locale = JavaLocale.new language country_text variant_text Locale.Value java_locale - ## ADVANCED - + ## PRIVATE + ADVANCED Convert a java locale to an Enso locale. Arguments: @@ -410,13 +410,7 @@ type Locale disp = self.java_locale.getDisplayVariant if disp.is_empty then Nothing else disp - ## Converts the locale to text. - - > Example - Convert the default locale to text. - - import Standard.Base.Data.Locale.Locale - - example_to_text = Locale.default.to_text + ## PRIVATE + Converts the locale to text. to_text : Text | Nothing to_text self = self.java_locale.toLanguageTag diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Map.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Map.enso index ddda0c25cebb..ad0a4a85f12b 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Map.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Map.enso @@ -349,7 +349,8 @@ type Map key value to_vector : Vector Any to_vector self = @Builtin_Method "Map.to_vector" - ## Returns a text representation of this Map. + ## PRIVATE + Returns a text representation of this Map. to_text : Text to_text self = @Builtin_Method "Map.to_text" diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso index e8a79bbb742f..fa6c46b4ddf9 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso @@ -942,10 +942,10 @@ type Integer A syntax error when parsing a double. @Builtin_Type type Number_Parse_Error + ## PRIVATE Error text - ## UNSTABLE - + ## PRIVATE Pretty print the syntax error. to_display_text : Text to_display_text = diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Ordering.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Ordering.enso index 37744ebd183b..eb71ee6e5c4d 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Ordering.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Ordering.enso @@ -132,9 +132,11 @@ type Comparable comp = Comparable.from atom (comp.is_a Default_Comparator).not -## Default implementation of a _comparator_. +## PRIVATE + Default implementation of a _comparator_. @Builtin_Type type Default_Comparator + ## PRIVATE compare : Any -> Any -> (Ordering|Nothing) compare x y = case Comparable.equals_builtin x y of @@ -149,13 +151,12 @@ type Default_Comparator True -> Ordering.Greater False -> Nothing + ## PRIVATE hash : Number -> Integer hash x = Comparable.hash_builtin x - Comparable.from (_:Any) = Default_Comparator - ## Types representing the ordering of values. @Builtin_Type type Ordering @@ -191,7 +192,6 @@ type Ordering Ordering.Equal -> 0 ## A lexicographical comparison. - and_then : Ordering -> Ordering and_then self ~other = case self of Ordering.Less -> Ordering.Less @@ -211,8 +211,12 @@ type Ordering from_sign sign = if sign == 0 then Ordering.Equal else if sign > 0 then Ordering.Greater else Ordering.Less +## PRIVATE type Ordering_Comparator + ## PRIVATE compare x y = (Comparable.from x.to_sign).compare x.to_sign y.to_sign + + ## PRIVATE hash x = x.to_sign Comparable.from (_:Ordering) = Ordering_Comparator diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Ordering/Sort_Direction.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Ordering/Sort_Direction.enso index f8ad55a1cbfe..c7ce75f8135c 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Ordering/Sort_Direction.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Ordering/Sort_Direction.enso @@ -19,7 +19,8 @@ type Sort_Direction Sort_Direction.Descending Descending - ## Convert into the sign of the direction + ## PRIVATE + Convert into the sign of the direction to_sign : Integer to_sign self = case self of Sort_Direction.Ascending -> 1 diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Regression.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Regression.enso index e92d6c36ef41..e99d545b0c2a 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Regression.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Regression.enso @@ -78,7 +78,8 @@ type Fitted_Model ## Fitted power series (y = a x ^ b). Power a:Number b:Number r_squared:Number=0.0 - ## Display the fitted line. + ## PRIVATE + Display the fitted line. to_text : Text to_text self = equation = case self of diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Statistics.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Statistics.enso index c185bd88a43f..58f1fc1fbd83 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Statistics.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Statistics.enso @@ -127,7 +127,8 @@ type Statistic - predicted: the series to compute the r_squared with. R_Squared (predicted:Vector) - ## Gets the order needed to compute a statistic for a moment based statistic. + ## PRIVATE + Gets the order needed to compute a statistic for a moment based statistic. order : Integer | Nothing order self = case self of Statistic.Sum -> 1 diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text.enso index 15852a2c1a0d..1ef1c3f51af2 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text.enso @@ -111,8 +111,6 @@ type Text not_empty self = self.is_empty.not ## PRIVATE - UNSTABLE - Conversion to Text that overrides the default `to_text` behavior. to_text : Text to_text self = self diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Encoding.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Encoding.enso index ec63f951f205..c9563c45c331 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Encoding.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Encoding.enso @@ -11,7 +11,9 @@ polyglot java import org.enso.base.Text_Utils ## Represents a character encoding. type Encoding - ## Get all available character sets from Java as Encodings. + ## PRIVATE + ADVANCED + Get all available character sets from Java as Encodings. Used to provide auto completion in the UI. all_character_sets : Vector Text all_character_sets = @@ -30,7 +32,8 @@ type Encoding charset = Charset.forName name Encoding.Value charset.name - ## Create a new Encoding object. + ## PRIVATE + Create a new Encoding object. Arguments: - character_set: java.nio.charset name. @@ -76,7 +79,6 @@ type Encoding windows_1251 = Encoding.Value "windows-1251" ## ALIAS ISO-8859-1 - Encoding for Western European (Windows). windows_1252 : Encoding windows_1252 = Encoding.Value "windows-1252" @@ -86,7 +88,6 @@ type Encoding windows_1253 = Encoding.Value "windows-1253" ## ALIAS ISO-8859-9 - Encoding for Turkish (Windows). windows_1254 : Encoding windows_1254 = Encoding.Value "windows-1254" diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Line_Ending_Style.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Line_Ending_Style.enso index 2610a725f0b3..db678c13fcad 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Line_Ending_Style.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Line_Ending_Style.enso @@ -13,7 +13,8 @@ type Line_Ending_Style The text equivalent is `'\r\n'`. Mac_Legacy - ## Returns the text equivalent of the line ending. + ## PRIVATE + Returns the text equivalent of the line ending. to_text : Text to_text self = case self of Line_Ending_Style.Unix -> '\n' diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex.enso index 269a58461cb0..cf20040a833f 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex.enso @@ -40,9 +40,7 @@ compile self expression case_insensitive=Nothing = Pattern.Value internal_regex_object -## ADVANCED - - Escape the special characters in `expression` such that the result is a +## Escape the special characters in `expression` such that the result is a valid literal pattern for the original string. Arguments: @@ -63,10 +61,10 @@ escape self expression = Regex_Utils.regexQuote expression Arguments: - id: The identifier of the group that was asked for but does not exist. type No_Such_Group + ## PRIVATE Error (id : Text | Integer) ## PRIVATE - Provides a human-readable representation of the `No_Such_Group`. to_display_text : Text to_display_text self = case self.id of diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Match.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Match.enso index 243ec3d1a96d..c6ffb7488ffa 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Match.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Match.enso @@ -261,9 +261,9 @@ type Match ## match.named_groups.keys.sort == ["empty", "letters"] named_groups : Any -> Map Text (Text | Any) named_groups self default=Nothing = - named_group_names = self.pattern.group_names - spans = named_group_names.map name-> self.text name default=default - Map.from_vector (named_group_names.zip spans) + pattern_named_groups = self.pattern.named_groups + Map.from_vector <| + pattern_named_groups.map name-> [name, self.text name default=default] ## Gets the grapheme span matched by the group with the provided index, or a default value if the group did not participate in the match. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Pattern.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Pattern.enso index 013b871738ed..a3f9584cd086 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Pattern.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Pattern.enso @@ -102,9 +102,7 @@ type Pattern Helpers.expect_text input <| self.match_all input . map match_to_group_maybe - ## ADVANCED - - Splits the `input` text based on the pattern described by `self`. + ## Splits the `input` text based on the pattern described by `self`. Arguments: - input: The text to split based on the pattern described by `self`. @@ -148,9 +146,7 @@ type Pattern go it.next builder.to_vector - ## ADVANCED - - Takes an input string and returns all the matches as a `Vector Text`. + ## Takes an input string and returns all the matches as a `Vector Text`. If the pattern contains marked groups, the values are concatenated together; otherwise the whole match is returned. Non-participating groups are omitted. @@ -177,9 +173,7 @@ type Pattern tokenize self input = self.match_all input . map (build_tokenization_output_from_match self _) - ## ADVANCED - - Replace all occurrences of the pattern described by `self` in the `input` + ## Replace all occurrences of the pattern described by `self` in the `input` with the specified `replacement`. Arguments: @@ -320,8 +314,8 @@ type Pattern group_count self = self.internal_regex_object.groupCount ## Return a vector of all named group names. - group_names : Map Text Integer - group_names self = + named_groups : Vector Text + named_groups self = map = polyglot_map_to_map self.internal_regex_object.groups map.keys @@ -342,12 +336,15 @@ type Pattern the string, unmatched, as a single Last value. (Used for `replace` with `only_first=True`.) type Match_Iterator + ## PRIVATE new : Pattern -> Text -> Match_Iterator new pattern input = Match_Iterator.Value pattern input 0 + ## PRIVATE Value (pattern : Pattern) (input : Text) (cursor : Integer) - ## Return the next match, or the last filler string if there is no + ## PRIVATE + Return the next match, or the last filler string if there is no additional match. Also returns the next iterator, if there was a match. @@ -368,30 +365,21 @@ type Match_Iterator next_iterator = Match_Iterator.Value self.pattern self.input next_cursor Match_Iterator_Value.Next filler_span match next_iterator - ## Returns the remainder of the string, unmatched. + ## PRIVATE + Returns the remainder of the string, unmatched. early_exit : Match_Iterator_Value early_exit self = filler_range = Range.new self.cursor (Text_Utils.char_length self.input) filler_span = Utf_16_Span.Value filler_range self.input Match_Iterator_Value.Last filler_span - to_text_debug : Vector Text - to_text_debug self = - vb = Vector.new_builder - go it = case it.next of - Match_Iterator_Value.Next filler match next_it -> - vb.append ('\"' + filler.text + '\"') - vb.append ("/" + (match.span 0).text + "/") - go next_it - Match_Iterator_Value.Last filler -> - vb.append ('\"' + filler.text + '\"') - go self - vb.to_vector - ## PRIVATE type Match_Iterator_Value - Next (filler : Span) (match : Match) (next_iterator : Match_Iterator) - Last (filler : Span) + ## PRIVATE + Next (filler : Span) (match : Match) (next_iterator : Match_Iterator) + + ## PRIVATE + Last (filler : Span) ## PRIVATE Convert the polyglot map to a Map. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Replacer.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Replacer.enso index 7a31ceaa002d..52ec26d15f59 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Replacer.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex/Replacer.enso @@ -35,7 +35,8 @@ type Replacer all the strings together to form the full replacement string. Value (replacement : Vector Replacement) - ## Creates a new Replacer. + ## PRIVATE + Creates a new Replacer. Arguments - replacement_string: a string, possibly containing group references, @@ -44,7 +45,8 @@ type Replacer new replacement_string pattern = Replacer.Value (build_replacement_vector_cached replacement_string pattern) - ## Build a replacement string from a match. + ## PRIVATE + Build a replacement string from a match. Arguments: - match: the match from the original string that is to be replaced. @@ -136,9 +138,12 @@ parse_group_number pattern match = case match.text.take 2 of n = Integer.parse <| match.text 2 Replacement.Substitution (pattern.lookup_group n) +## PRIVATE type Replacement - ## A string literal to replace with. + ## PRIVATE + A string literal to replace with. Literal (text : Text) - ## Target group to insert. + ## PRIVATE + Target group to insert. Substitution (group_number : Integer) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Span.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Span.enso index b1b4e326813f..9042434289ea 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Span.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Span.enso @@ -82,7 +82,8 @@ type Span text : Text text self = self.to_utf_16_span.text - ## Converts the span of extended grapheme clusters to a corresponding span + ## ADVANCED + Converts the span of extended grapheme clusters to a corresponding span of UTF-16 code units. > Example diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date.enso index d98c72607f48..2ee29070e594 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date.enso @@ -539,7 +539,8 @@ type Date _ -> Error.throw (Illegal_Argument.Error "Illegal period argument") - ## Convert to a JS_Object representing this Date. + ## PRIVATE + Convert to a JS_Object representing this Date. > Example Convert the current date to a JS_Object. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date_Time.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date_Time.enso index 100f3b46db7e..43704d2cf194 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date_Time.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Date_Time.enso @@ -584,18 +584,8 @@ type Date_Time Time_Utils.datetime_adjust self Time_Utils.AdjustOp.MINUS period.internal_period ensure_in_epoch result result - ## Convert this time to text using the default formatter. - - > Example - Convert the current time to text. - - from Standard.Base import Date_Time - - example_to_text = Date_Time.now.to_text - to_text : Text - to_text self = @Builtin_Method "Date_Time.to_text" - - ## Convert to a JavaScript Object representing a Date_Time. + ## PRIVATE + Convert to a JavaScript Object representing a Date_Time. > Example Convert the current time to a JS_Object. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Day_Of_Week.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Day_Of_Week.enso index eb1e7097fbde..b2529b6924f4 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Day_Of_Week.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Day_Of_Week.enso @@ -56,11 +56,15 @@ type Day_Of_Week ## PRIVATE type Day_Of_Week_Comparator + ## PRIVATE + compare : Day_Of_Week -> Day_Of_Week -> Ordering compare x y = x_int = x.to_integer y_int = y.to_integer Comparable.from x_int . compare x_int y_int + ## PRIVATE + hash : Day_Of_Week -> Integer hash x = x.to_integer Comparable.from (_:Day_Of_Week) = Day_Of_Week_Comparator diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Duration.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Duration.enso index 24ce569486e2..0edc9007effe 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Duration.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Duration.enso @@ -244,28 +244,8 @@ type Duration total_hours : Decimal ! Illegal_State total_hours self = self.total_minutes / 60.0 - ## Convert this duration to a Vector of hours, minutes, seconds, milliseconds - and nanoseconds. - - > Example - Convert a duration of one hour to a vector resulting in - `[1, 0, 30, 0, 0]`. - - import Standard.Base.Data.Time.Duration - - example_to_vec = (Duration.new hours=1 seconds=30).to_vector - - > Example - Convert duration of 800 nanoseconds to a vector returning - `[0, 0, 0, 0, 0, 800]` - - import Standard.Base.Data.Time.Duration - - example_to_vec = (Duration.new nanoseconds=800)).to_vector - to_vector : Vector Integer - to_vector self = [self.hours, self.minutes, self.seconds, self.milliseconds, self.nanoseconds] - - ## Convert to a JavaScript Object representing a Duration. + ## PRIVATE + Convert to a JavaScript Object representing a Duration. > Example Convert a duration of 10 seconds to a JS_Object. @@ -282,14 +262,3 @@ type Duration if self.milliseconds==0 . not then b.append ["milliseconds", self.milliseconds] if self.nanoseconds==0 . not then b.append ["nanoseconds", self.nanoseconds] JS_Object.from_pairs b.to_vector - - ## Check if this duration represents an empty time-span. - - > Example - Check if the duration of 10 seconds is empty. - - import Standard.Base.Data.Time.Duration - - example_is_empty = Duration.zero.is_empty - is_empty : Boolean - is_empty self = self.to_vector . all (==0) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Time_Of_Day.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Time_Of_Day.enso index 986e35a65ddf..adcc5591342a 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Time_Of_Day.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Time_Of_Day.enso @@ -284,7 +284,8 @@ type Time_Of_Day duration : Duration -> self.minus_builtin duration _ : Period -> Error.throw (Time_Error.Error "Time_Of_Day does not support date intervals (periods)") - ## Convert to a JavaScript Object representing this Time_Of_Day. + ## PRIVATE + Convert to a JavaScript Object representing this Time_Of_Day. > Example Convert the current time to a JS_Object. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Time_Zone.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Time_Zone.enso index d18dc72fe26e..f9adbf6eff0a 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Time_Zone.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Time/Time_Zone.enso @@ -156,7 +156,8 @@ type Time_Zone zone_id : Text zone_id self = @Builtin_Method "Time_Zone.zone_id" - ## Convert to a JavaScript Object representing this Time_Zone. + ## PRIVATE + Convert to a JavaScript Object representing this Time_Zone. > Example Convert your system's current timezone to a JS_Object. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Vector.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Vector.enso index 280f8f75d885..8c17c0ea85c1 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Vector.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Vector.enso @@ -66,7 +66,8 @@ type Vector a new : Integer -> (Integer -> Any) -> Vector Any new length constructor = @Builtin_Method "Vector.new" - ## ADVANCED + ## PRIVATE + ADVANCED Converts an array into a vector by copying content of the array. @@ -122,7 +123,8 @@ type Vector a new_builder : Integer -> Builder new_builder (capacity=10) = Builder.new capacity - ## ADVANCED + ## PRIVATE + ADVANCED Converts a polyglot value representing an array into a vector. @@ -136,7 +138,8 @@ type Vector a from_polyglot_array : Any -> Vector Any from_polyglot_array array = @Builtin_Method "Vector.from_polyglot_array" - ## ADVANCED + ## PRIVATE + ADVANCED Copies content of a vector into an Array. to_array self = @Builtin_Method "Vector.to_array" @@ -591,16 +594,14 @@ type Vector a reverse : Vector Any reverse self = Vector.new self.length (i -> self.at (self.length - (1 + i))) - ## Generates a human-readable text representation of the vector. - - > Example - Convert a vector of numbers to text. - - [1, 2, 3].to_text == "[1, 2, 3]" + ## PRIVATE + Generates a human-readable text representation of the vector. to_text : Text to_text self = self.map .to_text . join ", " "[" "]" - ## UNSTABLE + ## PRIVATE + ADVANCED + Generates a human-readable text representation of the vector, keeping its length limited. @@ -932,8 +933,8 @@ type Vector a to_list self = self.reverse.fold List.Nil acc-> elem-> List.Cons elem acc +## PRIVATE type Builder - ## PRIVATE A builder type for Enso vectors. @@ -968,7 +969,8 @@ type Builder that `Ref` and then `to_vector` could propagate that error. Value java_builder - ## Creates a new builder. + ## PRIVATE + Creates a new builder. Arguments: - capacity: Initial capacity of the Vector.Builder @@ -1111,13 +1113,10 @@ type Builder changes to the builder will not affect the returned vector. Vector.from_polyglot_array self.java_builder.toArray -## UNSTABLE - +## PRIVATE An error that indicates that the vector is empty. type Empty_Error - - ## UNSTABLE - + ## PRIVATE Pretty prints the empty error. to_display_text : Text to_display_text self = "The vector is empty." diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso index 9693c0228359..dbbea7e711df 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Error.enso @@ -20,7 +20,9 @@ from project.Data.Boolean import Boolean, True, False Enso graph. @Builtin_Type type Error - ## Creates a new dataflow error containing the provided payload. + ## PRIVATE + ADVANCED + Creates a new dataflow error containing the provided payload. Arguments: - payload: The contents of the dataflow error to be created. @@ -49,7 +51,8 @@ type Error get_stack_trace_text : Text get_stack_trace_text self = @Builtin_Method "Error.get_stack_trace_text" - ## Converts an error to a corresponding textual representation. + ## PRIVATE + Converts an error to a corresponding textual representation. > Example Converting a thrown error to text. @@ -58,8 +61,7 @@ type Error to_text : Text to_text self = @Builtin_Method "Error.to_text" - ## UNSTABLE - + ## PRIVATE Returns a human-readable text representing this error. to_display_text : Text to_display_text self = "Error: " + (self.catch Any .to_display_text) @@ -116,7 +118,6 @@ type Error map_error self f = self.catch Any (x -> Error.throw (f x)) ## ADVANCED - UNSTABLE Returns the attached stack trace of the error. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso index 859b78fe8b25..9b47659b005a 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso @@ -24,7 +24,6 @@ type Index_Out_Of_Bounds Error index length ## PRIVATE - Pretty prints an index out of bounds error. to_display_text : Text to_display_text self = @@ -145,15 +144,14 @@ type Arithmetic_Error @Builtin_Type type Incomparable_Values - ## An error that indicates that the two values are not comparable + ## An error that indicates that the two values are not comparable. Arguments: - - left: The left value (first operand) - - right: The right value (second operand) + - left: The left value (first operand). + - right: The right value (second operand). Error left right - ## UNSTABLE - + ## PRIVATE Convert the Incomparable_Values error to a human-readable format. to_display_text : Text to_display_text self = @@ -161,7 +159,8 @@ type Incomparable_Values True -> "Incomparable_Values.Error" False -> "Cannot compare `" + self.left.to_text + "` with `" + self.right.to_text + "`" - ## ADVANCED + ## PRIVATE + ADVANCED Catches possible errors from comparing values and throws an `Incomparable_Values` if any occur. handle_errors ~function = @@ -215,7 +214,7 @@ type Module_Does_Not_Exist @Builtin_Type type Invalid_Conversion_Target ## PRIVATE - An error that occurs when the specified value cannot be converted to a given type + An error that occurs when the specified value cannot be converted to a given type. Arguments: - target: the type trying to be converted to. @@ -224,10 +223,19 @@ type Invalid_Conversion_Target @Builtin_Type type No_Such_Conversion ## PRIVATE - An error that occurs when the conversion from one type to another does not exist + An error that occurs when the conversion from one type to another does not exist. Arguments: - target: the type trying to be converted to. - that: the value to be converted. - conversion: the conversion that was attempted. Error target that conversion + +@Builtin_Type +type Forbidden_Operation + ## PRIVATE + An error that occurs when the action is not allowed to perform the operation in the given context. + + Arguments: + - operation: attempted context that is not allowed. + Error operation diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/File_Error.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/File_Error.enso index 421751a96c52..7da423503550 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/File_Error.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/File_Error.enso @@ -43,8 +43,7 @@ type File_Error is not in the expected format. Corrupted_Format (file : File) (message : Text) (cause : Any | Nothing = Nothing) - ## UNSTABLE - + ## PRIVATE Convert the File error to a human-readable format. to_display_text : Text to_display_text self = case self of diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Problem_Behavior.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Problem_Behavior.enso index 8ce32c61da85..2b05336ffe73 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Problem_Behavior.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Problem_Behavior.enso @@ -24,8 +24,8 @@ type Problem_Behavior Report the problem as a dataflow error and abort the operation Report_Error - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED Attaches a problem to the given value according to the expected problem behavior. @@ -46,8 +46,8 @@ type Problem_Behavior if decorated_value.is_error then decorated_value else Error.throw problem - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED Attaches a problem to the given value according to the expected problem behavior. @@ -65,8 +65,8 @@ type Problem_Behavior Report_Error -> Error.throw problem - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED Attaches problems to the given value according to the expected problem behavior. @@ -96,8 +96,8 @@ type Problem_Behavior if problems.is_empty then decorated_value else Error.throw problems.first - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED Attaches problems to the given value according to the expected problem behavior. @@ -127,8 +127,8 @@ type Problem_Behavior if decorated_value.is_error || problems.is_empty then decorated_value else Error.throw problems.first - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED A helper method that will handle any errors contained within the result according to the current problem behavior settings. If the `result` does not contain an error, it is returned as-is. Otherwise, if the problem @@ -151,7 +151,8 @@ type Problem_Behavior values = ["Report_Error", "Report_Warning", "Ignore"] Single_Choice (values.map n-> Option n) display=Display.Expanded_Only - ## ADVANCED + ## PRIVATE + ADVANCED Checks any warnings reported by the `action` and reports them according to this problem behavior - they may be kept as-is, ignored or escalated to errors. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Unimplemented.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Unimplemented.enso index c16f6bbaa06f..bcc2316c7c05 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Unimplemented.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Unimplemented.enso @@ -2,6 +2,7 @@ import project.Data.Text.Text import project.Nothing.Nothing import project.Panic.Panic +@Builtin_Type type Unimplemented ## UNSTABLE @@ -12,12 +13,12 @@ type Unimplemented Error message ## PRIVATE - Converts the unimplemented error to a human-readable error message. to_display_text : Text to_display_text self = "An implementation is missing: " + self.message - ## ADVANCED + ## PRIVATE + ADVANCED A function that can be used to indicate that something hasn't been implemented yet. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/IO.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/IO.enso index 82466041400f..900a61caf36e 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/IO.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/IO.enso @@ -2,7 +2,9 @@ import project.Any.Any import project.Data.Text.Text import project.Nothing.Nothing -## Prints the provided message to standard error. +## PRIVATE + ADVANCED + Prints the provided message to standard error. Arguments: - message: The message to print. It will have to_text called on it to @@ -15,7 +17,9 @@ import project.Nothing.Nothing print_err : Any -> Nothing print_err message = @Builtin_Method "IO.print_err" -## Prints the provided message to standard output. +## PRIVATE + ADVANCED + Prints the provided message to standard output. Arguments: - message: The message to print. It will have to_text called on it to @@ -28,7 +32,9 @@ print_err message = @Builtin_Method "IO.print_err" println : Any -> Nothing println message = @Builtin_Method "IO.println" -## Reads a line from standard input. +## PRIVATE + ADVANCED + Reads a line from standard input. > Example Read a line from standard input. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso index 1cf329e5a0fb..3d3749e1893d 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso @@ -108,6 +108,7 @@ import project.Data.Time.Time_Period.Time_Period import project.Data.Time.Time_Zone.Time_Zone import project.Errors.Problem_Behavior.Problem_Behavior import project.Network.Extensions +import project.Network.HTTP.Header.Header import project.Network.HTTP.HTTP import project.Network.HTTP.HTTP_Method.HTTP_Method import project.Network.HTTP.HTTP_Status_Code.HTTP_Status_Code @@ -156,6 +157,7 @@ export project.Data.Time.Time_Period.Time_Period export project.Data.Time.Time_Zone.Time_Zone export project.Errors.Problem_Behavior.Problem_Behavior export project.Network.Extensions +export project.Network.HTTP.Header.Header export project.Network.HTTP.HTTP export project.Network.HTTP.HTTP_Method.HTTP_Method export project.Network.HTTP.HTTP_Status_Code.HTTP_Status_Code diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Meta.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Meta.enso index f6c830db827b..890efd9f16ce 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Meta.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Meta.enso @@ -18,7 +18,7 @@ import project.Error.Error as Base_Error from project.Data.Boolean import Boolean, True, False type Type - ## UNSTABLE + ## PRIVATE ADVANCED Type meta-representation. @@ -27,16 +27,14 @@ type Type - value: The value of the type in the meta representation. Value value - ## UNSTABLE - ADVANCED + ## ADVANCED Returns a vector of `Meta.Constructor` for this type constructors : Vector constructors self = Vector.from_polyglot_array (get_type_constructors self.value Meta.Constructor.Value) - ## UNSTABLE - ADVANCED + ## ADVANCED Returns a vector of `Meta.Constructor` for this type methods : Vector @@ -44,7 +42,7 @@ type Type Vector.from_polyglot_array (get_type_methods self.value) type Atom - ## UNSTABLE + ## PRIVATE ADVANCED An Atom meta-representation. @@ -53,15 +51,13 @@ type Atom - value: The value of the atom in the meta representation. Value value - ## UNSTABLE - ADVANCED + ## ADVANCED Returns a vector of field values of the given atom. fields : Vector fields self = Vector.from_polyglot_array (get_atom_fields self.value) - ## UNSTABLE - ADVANCED + ## ADVANCED Returns a constructor value of the given atom. constructor : Constructor @@ -70,7 +66,7 @@ type Atom Meta.Constructor.Value java_constructor type Constructor - ## UNSTABLE + ## PRIVATE ADVANCED A constructor meta-representation. @@ -79,8 +75,7 @@ type Constructor - value: The value of the constructor in the meta representation. Value value - ## UNSTABLE - ADVANCED + ## ADVANCED Returns a vector of field names defined by a constructor. fields : Vector @@ -88,8 +83,7 @@ type Constructor c = self.value ... Vector.from_polyglot_array (get_constructor_fields c) - ## UNSTABLE - ADVANCED + ## ADVANCED Returns the name of a constructor. name : Text @@ -97,8 +91,7 @@ type Constructor c = self.value ... get_constructor_name c - ## UNSTABLE - ADVANCED + ## ADVANCED Creates a new atom of the given constructor. @@ -111,7 +104,7 @@ type Constructor new_atom ctor fields.to_array type Primitive - ## UNSTABLE + ## PRIVATE ADVANCED A primitive value meta-representation. @@ -121,7 +114,7 @@ type Primitive Value value type Unresolved_Symbol - ## UNSTABLE + ## PRIVATE ADVANCED An unresolved symbol meta-representation. @@ -130,7 +123,7 @@ type Unresolved_Symbol - value: The value of the unresolved symbol in the meta representation. Value value - ## UNSTABLE + ## PRIVATE ADVANCED Returns a new unresolved symbol with its name changed to the provided @@ -142,22 +135,20 @@ type Unresolved_Symbol rename self new_name = create_unresolved_symbol new_name self.scope - ## UNSTABLE - ADVANCED + ## ADVANCED Returns the name of an unresolved symbol. name : Text name self = get_unresolved_symbol_name self.value - ## UNSTABLE - ADVANCED + ## ADVANCED Returns the definition scope of an unresolved symbol. scope : Any scope self = get_unresolved_symbol_scope self.value type Error - ## UNSTABLE + ## PRIVATE ADVANCED An error meta-representation, containing the payload of a dataflow error. @@ -167,7 +158,7 @@ type Error Value value type Polyglot - ## UNSTABLE + ## PRIVATE ADVANCED A polyglot value meta-representation. @@ -176,8 +167,7 @@ type Polyglot - value: The polyglot value contained in the meta representation. Value value - ## UNSTABLE - ADVANCED + ## ADVANCED Returns the language with which a polyglot value is associated. get_language : Language @@ -185,8 +175,7 @@ type Polyglot lang_str = get_polyglot_language self.value if lang_str == "java" then Language.Java else Language.Unknown -## UNSTABLE - ADVANCED +## ADVANCED Checks whether `self` represents the same underlying reference as `value`. @@ -196,8 +185,7 @@ type Polyglot Any.is_same_object_as : Any -> Boolean Any.is_same_object_as self value = is_same_object self value -## UNSTABLE - ADVANCED +## ADVANCED Checks if `self` is an instance of `typ`. @@ -206,8 +194,7 @@ Any.is_same_object_as self value = is_same_object self value Any.is_a : Any -> Boolean Any.is_a self typ = is_a self typ -## UNSTABLE - ADVANCED +## ADVANCED Checks if `self` is an instance of `typ`. @@ -329,7 +316,7 @@ new_atom constructor fields = @Builtin_Method "Meta.new_atom" atom_with_hole : (Any -> Atom) -> Any atom_with_hole factory = @Builtin_Method "Meta.atom_with_hole_builtin" -## UNSTABLE +## PRIVATE ADVANCED Returns a meta-representation of a given runtime entity. @@ -345,7 +332,7 @@ meta value = if is_atom value then Atom.Value value else if is_type value then Type.Value value.catch else Primitive.Value value -## UNSTABLE +## PRIVATE ADVANCED Checks whether two objects are represented by the same underlying reference. @@ -356,7 +343,7 @@ meta value = if is_atom value then Atom.Value value else is_same_object : Any -> Any -> Boolean is_same_object value_1 value_2 = @Builtin_Method "Meta.is_same_object" -## UNSTABLE +## PRIVATE ADVANCED Checks if `value` is an instance of `typ`. @@ -373,7 +360,7 @@ java_instance_check value typ = typ_java = get_polyglot_language typ == "java" val_java && typ_java && Java.is_instance value typ -## UNSTABLE +## PRIVATE ADVANCED Returns the type of the given value. @@ -383,7 +370,7 @@ java_instance_check value typ = type_of : Any -> Any type_of value = @Builtin_Method "Meta.type_of" -## UNSTABLE +## PRIVATE ADVANCED Given a type object, method name and a parameter name, return the associated annotation if it exists. @@ -395,16 +382,16 @@ type_of value = @Builtin_Method "Meta.type_of" get_annotation : Any -> Text -> Text -> Any | Nothing get_annotation target method_name parameter_name = @Builtin_Method "Meta.get_annotation" -## Represents a polyglot language. +## PRIVATE + Represents a polyglot language. type Language - - ## UNSTABLE + ## PRIVATE ADVANCED The Java language. Java - ## UNSTABLE + ## PRIVATE ADVANCED An unknown language. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Meta/Enso_Project.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Meta/Enso_Project.enso index 9286499adca3..1ccb2edd5a3c 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Meta/Enso_Project.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Meta/Enso_Project.enso @@ -4,7 +4,8 @@ import project.System.File.File ## Functionality for inspecting the current project. @Builtin_Type type Project_Description - ## A representation of an Enso project. + ## PRIVATE + A representation of an Enso project. Arguments: - prim_root_file: The primitive root file of the project. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso index f0eb6b35b09f..63921feb9f4e 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP.enso @@ -61,7 +61,6 @@ type HTTP new (timeout = (Duration.new seconds=10)) (follow_redirects = True) (proxy = Proxy.System) (version = HTTP_Version.HTTP_1_1) = HTTP.Value timeout follow_redirects proxy version - ## ALIAS Fetch Data Send the Get request and return the body. @@ -335,7 +334,8 @@ type HTTP req = Request.delete uri headers self.request req - ## Create a request + ## ADVANCED + Create a request Arguments: - req: The HTTP request to send using `self` HTTP client. @@ -466,7 +466,6 @@ type HTTP Response.Value (self.internal_http_client.send http_request body_handler) ## PRIVATE - Build an HTTP client. internal_http_client : HttpClient internal_http_client self = @@ -497,18 +496,17 @@ type HTTP # build http client builder.build -## UNSTABLE - +## PRIVATE An error when sending an HTTP request. Arguments: - error_type: The type of the error. - message: The message for the error. type Request_Error + ## PRIVATE Error error_type message ## PRIVATE - Convert a request error to a human-readable form. to_display_text : Text to_display_text self = diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Method.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Method.enso index c097a288f8fe..5b5b88004261 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Method.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Method.enso @@ -31,7 +31,9 @@ type HTTP_Method ## Custom unsupported HTTP method. Custom verb:Text - ## Convert to a Text of the HTTP method name. + ## PRIVATE + ADVANCED + Convert to a Text of the HTTP method name. to_http_method_name : Text to_http_method_name self = case self of HTTP_Method.Options -> "OPTIONS" diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso index 6602176422fd..b931715bc73c 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso @@ -1,3 +1,4 @@ +import project.Data.Numbers.Integer import project.Data.Text.Text import project.Nothing.Nothing @@ -33,8 +34,6 @@ type Header new : Text -> Text -> Header new name value = Header.Value name value - # Accept - ## Create an "Accept" header. Arguments: @@ -92,7 +91,13 @@ type Header authorization_basic user pass = Header.authorization (Http_Utils.header_basic_auth user pass) - # Content-Type + ## Create bearer token auth header. + + Arguments: + - token: The token. + authorization_bearer : Text -> Header + authorization_bearer token = + Header.authorization ("Bearer " + token) ## Create "Content-Type" header. @@ -170,10 +175,14 @@ type Header ## PRIVATE type Header_Comparator + ## PRIVATE + compare : Header -> Header -> Ordering compare x y = if x.name.equals_ignore_case y.name && x.value == y.value then Ordering.Equal else Nothing + ## PRIVATE + hash : Header -> Integer hash x = key = x.name.to_case_insensitive_key + x.value Comparable.from key . hash key diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso index b8577a9c69ad..24865df1f9b6 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response.enso @@ -54,7 +54,8 @@ type Response code : HTTP_Status_Code code self = HTTP_Status_Code.Value self.internal_http_response.statusCode - ## Convert to a JavaScript Object representing this Response. + ## PRIVATE + Convert to a JavaScript Object representing this Response. > Example Convert a response to JS_Object. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response_Body.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response_Body.enso index 9415f743d828..fa0b9281f164 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response_Body.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Response_Body.enso @@ -9,22 +9,17 @@ import project.Nothing.Nothing import project.System.File.File import project.System.File.Write_Extensions +## PRIVATE type Response_Body - - ## Response body + ## PRIVATE + Response body Arguments: - bytes: The body of the response as binary data. Value bytes - ## Convert response body to Text. - - > Example - Convert a response to text. NOTE: This example makes a network request. - - import Standard.Examples - - example_to_text = Examples.get_geo_data.to_text + ## PRIVATE + Convert response body to Text. to_text : Text to_text self = Text.from_utf_8 self.bytes diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso index 19c468fc0c43..90a502fea549 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso @@ -153,48 +153,43 @@ type URI fragment : Text ! Nothing fragment self = handle_nothing self.internal_uri.getFragment - ## ADVANCED - + ## PRIVATE + ADVANCED Get the unescaped user info part of this URI. raw_user_info : Text ! Nothing raw_user_info self = handle_nothing self.internal_uri.getRawUserInfo - ## ADVANCED - + ## PRIVATE + ADVANCED Get the unescaped authority part of this URI. raw_authority : Text ! Nothing raw_authority self = handle_nothing self.internal_uri.getRawAuthority - ## ADVANCED - + ## PRIVATE + ADVANCED Get the unescaped path part of this URI. raw_path : Text ! Nothing raw_path self = handle_nothing self.internal_uri.getRawPath - ## ADVANCED - + ## PRIVATE + ADVANCED Get the unescaped query part of this URI. raw_query : Text ! Nothing raw_query self = handle_nothing self.internal_uri.getRawQuery - ## ADVANCED - + ## PRIVATE + ADVANCED Get the unescaped fragment part of this URI. raw_fragment : Text ! Nothing raw_fragment self = handle_nothing self.internal_uri.getRawFragment - ## Convert this URI to text. - - > Example - Convert a URI to text. - - import Standard.Examples - - example_to_text = Examples.uri.to_text + ## PRIVATE + Convert this URI to text. to_text : Text to_text self = self.internal_uri.toString - ## Convert to a JavaScript Object representing this URI. + ## PRIVATE + Convert to a JavaScript Object representing this URI. > Example Convert a URI to a JS_Object. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Panic.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Panic.enso index 7da31f42acfd..5c391a3a1b83 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Panic.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Panic.enso @@ -21,8 +21,9 @@ polyglot java import java.lang.Throwable to be handled through non-linear control flow mechanisms. @Builtin_Type type Panic - - ## Throws a new panic with the provided payload. + ## PRIVATE + ADVANCED + Throws a new panic with the provided payload. Arguments: - payload: The contents of the panic to be thrown. If the payload is a @@ -54,7 +55,6 @@ type Panic primitive_get_attached_stack_trace throwable = @Builtin_Method "Panic.primitive_get_attached_stack_trace" ## ADVANCED - UNSTABLE Returns the attached stack trace of the given throwable. Can be used to get an Enso friendly stack trace from native Java exceptions. @@ -70,7 +70,9 @@ type Panic stack_with_prims = Vector.from_polyglot_array prim_stack stack_with_prims.map Runtime.wrap_primitive_stack_trace_element - ## Takes any value, and if it is a dataflow error, throws it as a Panic, + ## PRIVATE + ADVANCED + Takes any value, and if it is a dataflow error, throws it as a Panic, otherwise, returns the original value unchanged. Arguments: @@ -160,7 +162,9 @@ type Panic True -> caught_panic.convert_to_dataflow_error False -> Panic.throw caught_panic - ## If a dataflow error had occurred, wrap it in a `Wrapped_Dataflow_Error` and promote to a Panic. + ## PRIVATE + ADVANCED + If a dataflow error had occurred, wrap it in a `Wrapped_Dataflow_Error` and promote to a Panic. Arguments: - value: value to return if not an error, or rethrow as a Panic. @@ -168,7 +172,9 @@ type Panic throw_wrapped_if_error ~value = if value.is_error then Panic.throw (Wrapped_Dataflow_Error.Error value.catch) else value - ## Catch any `Wrapped_Dataflow_Error` Panic and rethrow it as a dataflow error. + ## PRIVATE + ADVANCED + Catch any `Wrapped_Dataflow_Error` Panic and rethrow it as a dataflow error. Arguments: - action: The code to execute that potentially raised a Wrapped_Dataflow_Error. @@ -177,9 +183,11 @@ type Panic Panic.catch Wrapped_Dataflow_Error action caught_panic-> Error.throw caught_panic.payload.payload +## PRIVATE @Builtin_Type type Caught_Panic - ## A wrapper for a caught panic. + ## PRIVATE + A wrapper for a caught panic. Arguments: - payload: the payload carried by the error. @@ -201,9 +209,9 @@ type Caught_Panic Wraps a dataflow error lifted to a panic, making possible to distinguish it from other panics. type Wrapped_Dataflow_Error + ## PRIVATE Error payload ## PRIVATE Throws the original error. unwrap self = Error.throw self.payload - diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Polyglot.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Polyglot.enso index 2be4336d7ace..cb565243b6f2 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Polyglot.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Polyglot.enso @@ -12,21 +12,27 @@ import project.Runtime.Source_Location.Source_Location @Builtin_Type type Polyglot - ## Reads the number of elements in a given polyglot array object. + ## PRIVATE + ADVANCED + Reads the number of elements in a given polyglot array object. Arguments: - array: a polyglot array object, originating in any supported language. get_array_size : Any -> Integer get_array_size array = @Builtin_Method "Polyglot.get_array_size" - ## Reads the element in a given polyglot array object. + ## PRIVATE + ADVANCED + Reads the element in a given polyglot array object. Arguments: - index: The index to get the element from. read_array_element : Any -> Integer -> Any read_array_element array index = @Builtin_Method "Polyglot.read_array_element" - ## Executes a polyglot function object (e.g. a lambda). + ## PRIVATE + ADVANCED + Executes a polyglot function object (e.g. a lambda). Arguments: - callable: The polyglot function object to execute. @@ -34,7 +40,9 @@ type Polyglot execute : Any -> Vector -> Any execute callable arguments = @Builtin_Method "Polyglot.execute" - ## Performs a by-name lookup for a member in a polyglot object. + ## PRIVATE + ADVANCED + Performs a by-name lookup for a member in a polyglot object. Arguments: - object: The polyglot object on which to perform the member lookup. @@ -46,7 +54,9 @@ type Polyglot get_member : Any -> Text -> Any get_member object member_name = @Builtin_Method "Polyglot.get_member" - ## Returns a polyglot array of all of the members of the provided object. + ## PRIVATE + ADVANCED + Returns a polyglot array of all of the members of the provided object. Arguments: - object: The object from which to get a list of member names. @@ -57,7 +67,9 @@ type Polyglot get_members : Any -> Array get_members object = @Builtin_Method "Polyglot.get_members" - ## Instantiates a polyglot object using the provided constructor. + ## PRIVATE + ADVANCED + Instantiates a polyglot object using the provided constructor. Arguments: - constructor: The constructor with which to instantiate the object. @@ -70,7 +82,9 @@ type Polyglot new : Any -> Vector -> Any new constructor arguments = @Builtin_Method "Polyglot.new" - ## Invokes a method on a polyglot object by name. + ## PRIVATE + ADVANCED + Invokes a method on a polyglot object by name. Arguments: - target: The polyglot object on which to call the method. @@ -79,45 +93,48 @@ type Polyglot invoke : Any -> Text -> Vector -> Any invoke target name arguments = @Builtin_Method "Polyglot.invoke" - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED - Checks if `value` defines a source location. + Checks if `value` defines a source location. - Source locations are typically exposed by functions, classes, sometimes - also other objects to specify their allocation sites. + Source locations are typically exposed by functions, classes, sometimes + also other objects to specify their allocation sites. has_source_location : Any -> Boolean has_source_location value = @Builtin_Method "Polyglot.has_source_location" - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED - Gets the source location of `value`. + Gets the source location of `value`. - Source locations are typically exposed by functions, classes, sometimes - also other objects to specify their allocation sites. - This method will throw a polyglot exception if - `Polyglot.has_source_location value` returns `False`. + Source locations are typically exposed by functions, classes, sometimes + also other objects to specify their allocation sites. + This method will throw a polyglot exception if + `Polyglot.has_source_location value` returns `False`. get_source_location : Any -> Source_Location get_source_location value = @Builtin_Method "Polyglot.get_source_location" - ## Checks if a polyglot language is installed in the runtime environment. + ## PRIVATE + ADVANCED + Checks if a polyglot language is installed in the runtime environment. - Arguments: - - langauge_name: The name of the language to test + Arguments: + - language_name: The name of the language to test. is_language_installed : Text -> Boolean is_language_installed language_name = @Builtin_Method "Polyglot.is_language_installed" - ## ADVANCED - UNSTABLE - + ## PRIVATE + ADVANCED Returns the executable name of a polyglot object. get_executable_name : Any -> Text get_executable_name value = @Builtin_Method "Polyglot.get_executable_name" ## Utilities for working with Java polyglot objects. type Java - ## Adds the provided entry to the host class path. + ## PRIVATE + ADVANCED + Adds the provided entry to the host class path. Arguments: - path: The java classpath entry to add. @@ -132,7 +149,9 @@ type Java add_to_class_path : Text -> Nothing add_to_class_path path = @Builtin_Method "Java.add_to_class_path" - ## Looks up a java symbol on the classpath by name. + ## PRIVATE + ADVANCED + Looks up a java symbol on the classpath by name. Arguments: - name: The name of the java symbol to look up. @@ -148,7 +167,6 @@ type Java lookup_class name = @Builtin_Method "Java.lookup_class" ## PRIVATE - Checks whether an object is an instance of a given class. Arguments: diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime.enso index 0357fdafcd39..f9df864d60f9 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime.enso @@ -1,12 +1,17 @@ import project.Any.Any import project.Data.Array.Array +import project.Data.Boolean.Boolean import project.Data.Text.Text import project.Data.Vector.Vector +import project.Errors.Common.Forbidden_Operation +import project.Function.Function import project.Nothing.Nothing +import project.Panic.Panic import project.Polyglot.Polyglot import project.Runtime.Source_Location.Source_Location from project.Data.Index_Sub_Range.Index_Sub_Range import First, Last +from project.Runtime.Context import Input,Output ## Utilities for interacting with the runtime. @@ -17,8 +22,8 @@ from project.Data.Index_Sub_Range.Index_Sub_Range import First, Last primitive_get_stack_trace : Array primitive_get_stack_trace = @Builtin_Method "Runtime.primitive_get_stack_trace" -## ADVANCED - UNSTABLE +## PRIVATE + ADVANCED Returns the execution stack trace of its call site. The ordering of the resulting vector is such that the top stack frame is the first element. @@ -30,7 +35,8 @@ get_stack_trace = stack = stack_with_prims.drop (First 2) stack.map wrap_primitive_stack_trace_element -## ADVANCED +## PRIVATE + ADVANCED Suggests that the runtime perform garbage collection. @@ -44,7 +50,8 @@ get_stack_trace = gc : Nothing gc = @Builtin_Method "Runtime.gc" -## ADVANCED +## PRIVATE + ADVANCED Executes the provided action without allowing it to inline. @@ -62,8 +69,8 @@ gc = @Builtin_Method "Runtime.gc" no_inline : Any -> Any no_inline ~action = @Builtin_Method "Runtime.no_inline" -## ADVANCED - UNSTABLE +## PRIVATE + ADVANCED Applies the following function to the given argument, without allowing them to inline. @@ -88,8 +95,8 @@ wrap_primitive_stack_trace_element el = name = Polyglot.get_executable_name el Stack_Trace_Element.Value name loc -## ADVANCED - UNSTABLE +## PRIVATE + ADVANCED Represents a single stack frame in an Enso stack trace. type Stack_Trace_Element @@ -104,31 +111,112 @@ type Stack_Trace_Element loc -> loc.formatted_coordinates "at "+self.name+" ("+loc+")" -## ADVANCED +## PRIVATE + ADVANCED - Types indicating allowed IO operations -type IO_Permissions + Type indicating allowed execution context. + +@Builtin_Type +type Context + ## PRIVATE + ADVANCED Input + ## PRIVATE + ADVANCED Output -## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED - Allows an action in the `Input` context to be performed in the given `env`, - regardless of the Env configuration. + Returns the name of the context. - This can be used to enable certain nodes to run their actions in the - interactive mode, even if the configuration forbids it. -allow_input_in : Text -> Any -> Any -allow_input_in env ~action = @Builtin_Method "Runtime.allow_input_in" + name : Text + name self = + case self of + Input -> "Input" + Output -> "Output" -## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED - Allows an action in the `Output` context to be performed in the given `env`, - regardless of the Env configuration. + Checks whether the context is enabled. If it is, evaluates the provided + function and returns the result. If not, panics. - This can be used to enable certain nodes to run their actions in the - interactive mode, even if the configuration forbids it. -allow_output_in : Text -> Any -> Any -allow_output_in env ~action = @Builtin_Method "Runtime.allow_output_in" + Arguments: + - environment: Name of the execution environment. + - context: The context to enable. + - action: Action to be performed with the context enabled. + if_enabled : Function -> Text -> Any + if_enabled self ~action environment=Runtime.current_execution_environment = + if self.is_enabled environment then action else Panic.throw (Forbidden_Operation.Error self.name) + + ## PRIVATE + ADVANCED + + Checks whether the permission is enabled in the given environment. + + Arguments: + - environment: Name of the execution environment. + - context: The context to enable. + is_enabled : Text -> Boolean + is_enabled self environment=Runtime.current_execution_environment = @Builtin_Method "Context.is_enabled" + + +## PRIVATE + ADVANCED + + Returns the name of the current execution environment. +current_execution_environment : Text +current_execution_environment = @Builtin_Method "Runtime.current_execution_environment" + +## PRIVATE + ADVANCED + + Enables a specific context in the provided runtime environment for the duration of the execution of the action. + + Arguments: + - environment: Name of the execution environment. + - context: The context to enable. + - action: Action to be performed with the context enabled. +with_enabled_context : Context -> Function -> Text -> Any +with_enabled_context context ~action environment=Runtime.current_execution_environment = with_enabled_context_builtin context action environment + +## PRIVATE + ADVANCED + + Enables a specific context in the provided runtime environment for the duration of the execution of the action. + + This method is internal, using `with_enabled_context` is preferred as it provides correct defaults. + + Arguments: + - environment: Name of the execution environment. + - context: The context to enable. + - action: Action to be performed with the context enabled. +with_enabled_context_builtin : Context -> Function -> Text -> Any +with_enabled_context_builtin context ~action environment = @Builtin_Method "Runtime.with_enabled_context_builtin" + +## PRIVATE + ADVANCED + + Disables a specific context in the provided runtime environment for the duration of the execution of the action. + + Arguments: + - environment: Name of the execution environment. + - context: The context to disable. + - action: Action to be performed with the context disabled. +with_disabled_context : Context -> Function -> Text -> Any +with_disabled_context context ~action environment=Runtime.current_execution_environment = with_disabled_context_builtin context action environment + +## PRIVATE + ADVANCED + + Disables a specific context in the provided runtime environment for the duration of the execution of the action. + + This method is internal, using `with_disabled_context` is preferred as it provides correct defaults. + + Arguments: + - environment: Name of the execution environment. + - context: The context to disable. + - action: Action to be performed with the context disabled. +with_disabled_context_builtin : Context -> Function -> Text -> Any +with_disabled_context_builtin context ~action environment = @Builtin_Method "Runtime.with_disabled_context_builtin" diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Debug.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Debug.enso index 9666a5cd9f9b..f69dba53d650 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Debug.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Debug.enso @@ -19,7 +19,9 @@ import project.Nothing.Nothing breakpoint : Nothing breakpoint = @Builtin_Method "Debug.breakpoint" -## Evaluates the provided Enso code in the caller frame. +## PRIVATE + ADVANCED + Evaluates the provided Enso code in the caller frame. Arguments: - expression: The enso code to evaluate. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Lazy.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Lazy.enso deleted file mode 100644 index 388eb0984d0d..000000000000 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Lazy.enso +++ /dev/null @@ -1,75 +0,0 @@ -import project.Any.Any -import project.Error.Error -import project.Nothing.Nothing -import project.Panic.Caught_Panic -import project.Panic.Panic -import project.Runtime.Ref.Ref - -## Holds a value that is computed on first access. -type Lazy - ## PRIVATE - Lazy (cached_ref : Ref) (builder : Nothing -> Any) - - ## PRIVATE - Eager (value : Any) - - ## Creates a new lazy value. - new : Any -> Lazy - new ~lazy_computation = - builder _ = lazy_computation - cached_ref = Ref.new Lazy_Not_Computed_Mark - Lazy.Lazy cached_ref builder - - ## Creates a pre-computed lazy value. - This can be useful if a value needs to admit the Lazy type API, but is - known beforehand. - new_eager value = Lazy.Eager value - - ## Returns the stored value. - - The value will be computed on first access and cached. - get : Any - get self = case self of - Lazy.Lazy cached_ref builder -> case cached_ref.get of - Lazy_Not_Computed_Mark -> - cached_value = Cached_Value.freeze builder - cached_ref.put cached_value - cached_value.get - cached_value -> cached_value.get - Lazy.Eager value -> value - -## PRIVATE - This is a special value that should never be returned from a lazy computation - as it will prevent the lazy value from being cached. -type Lazy_Not_Computed_Mark - -## PRIVATE -type Cached_Value - ## PRIVATE - Value value - - ## PRIVATE - Error error - - ## PRIVATE - Panic (caught_panic : Caught_Panic) - - ## PRIVATE - Accesses the cached value as if it was just computed - any stored errors - or panics will be propagated. - get : Any - get self = case self of - Cached_Value.Value value -> value - Cached_Value.Error error -> Error.throw error - Cached_Value.Panic caught_panic -> Panic.throw caught_panic - - ## PRIVATE - Runs the provided `builder` with a `Nothing` argument, handling any - errors or panics and saving them as a `Cached_Value`. - freeze : (Nothing -> Any) -> Cached_Value - freeze builder = - save_panic caught_panic = Cached_Value.Panic caught_panic - Panic.catch Any handler=save_panic <| - result = Cached_Value.Value (builder Nothing) - result.catch Any dataflow_error-> - Cached_Value.Error dataflow_error diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Managed_Resource.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Managed_Resource.enso index b31c50c5be9d..86381044f5a0 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Managed_Resource.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Managed_Resource.enso @@ -16,7 +16,8 @@ import project.Nothing.Nothing abstractions, and is not expected to be used by end-users. @Builtin_Type type Managed_Resource - ## ADVANCED + ## PRIVATE + ADVANCED Acquires a resource, performs an action on it, and destroys it safely, even in the presence of panics. @@ -29,7 +30,8 @@ type Managed_Resource bracket : Any -> (Any -> Nothing) -> (Any -> Any) -> Any bracket ~constructor ~destructor ~action = @Builtin_Method "Resource.bracket" - ## ADVANCED + ## PRIVATE + ADVANCED Registers a resource with the resource manager to be cleaned up using function once it is no longer in use. @@ -40,14 +42,16 @@ type Managed_Resource register : Any -> (Any -> Nothing) -> Managed_Resource register resource function = @Builtin_Method "Managed_Resource.register" - ## ADVANCED + ## PRIVATE + ADVANCED Forces finalization of a managed resource using the registered finalizer, even if the resource is still reachable. finalize : Nothing finalize self = @Builtin_Method "Managed_Resource.finalize" - ## ADVANCED + ## PRIVATE + ADVANCED Executes the provided action on the resource managed by the managed resource object. @@ -58,7 +62,8 @@ type Managed_Resource with : (Any -> Any) -> Any with self ~action = @Builtin_Method "Managed_Resource.with" - ## ADVANCED + ## PRIVATE + ADVANCED Takes the value held by the managed resource and unregisters the finalization step for this resource, effectively removing it from the diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Ref.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Ref.enso index c72bef84ef7f..a06d0422a577 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Ref.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Ref.enso @@ -1,9 +1,13 @@ import project.Any.Any -## A mutable reference type. +## PRIVATE + ADVANCED + A mutable reference type. @Builtin_Type type Ref - ## Creates a new reference containing the provided value. + ## PRIVATE + ADVANCED + Creates a new reference containing the provided value. Arguments: - value: The value to be contained in the ref. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Source_Location.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Source_Location.enso index 240c33055504..1e514ac13cac 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Source_Location.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Source_Location.enso @@ -4,47 +4,37 @@ import project.System.File.File from project.Data.Boolean import Boolean, True, False -## ADVANCED - UNSTABLE +## PRIVATE + ADVANCED Represents a source location in Enso code. Contains information about the source file and code position within it. type Source_Location ## PRIVATE Value prim_location - ## UNSTABLE + ## PRIVATE Pretty prints the location. to_text : Text to_text self = '(Source_Location ' + self.formatted_coordinates + ')' - ## UNSTABLE - - Returns the 1-based line index of the start of this code range. + ## Returns the 1-based line index of the start of this code range. start_line : Integer start_line self = self.prim_location.getStartLine - ## UNSTABLE - - Returns the 1-based line index of the end of this code range. + ## Returns the 1-based line index of the end of this code range. end_line : Integer end_line self = self.prim_location.getEndLine - ## UNSTABLE - - Returns the 1-based column index of the start of this code range. + ## Returns the 1-based column index of the start of this code range. start_column : Integer start_column self = self.prim_location.getStartColumn - ## UNSTABLE - - Returns the 1-based column index of the end of this code range. + ## Returns the 1-based column index of the end of this code range. end_column : Integer end_column self = self.prim_location.getEndColumn - ## UNSTABLE - - Returns a pretty-printed location (file and line info). + ## Returns a pretty-printed location (file and line info). formatted_coordinates : Text formatted_coordinates self = start_line = self.start_line @@ -64,8 +54,6 @@ type Source_Location _ -> file.path formatted_file + ":" + indices - ## UNSTABLE - - Return the source file corresponding to this location. + ## Return the source file corresponding to this location. file : File file self = File.new self.prim_location.getSource.getPath diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/State.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/State.enso index ba3b8c943087..1082acf2926f 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/State.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/State.enso @@ -6,7 +6,9 @@ import project.Any.Any from project.Errors.Common import Uninitialized_State -## Executes a stateful computation in a local state environment. +## PRIVATE + ADVANCED + Executes a stateful computation in a local state environment. Arguments: - key: The key to associate your local_state with in the environment. @@ -23,7 +25,9 @@ from project.Errors.Common import Uninitialized_State run : Any -> Any -> Any -> Any run key local_state ~computation = @Builtin_Method "State.run" -## Returns the current value for the provided key contained in the monadic +## PRIVATE + ADVANCED + Returns the current value for the provided key contained in the monadic state. Arguments: @@ -40,7 +44,9 @@ run key local_state ~computation = @Builtin_Method "State.run" get : Any -> Any ! Uninitialized_State get key = @Builtin_Method "State.get" -## Associates a new_state with the provided key in the runtime's monadic +## PRIVATE + ADVANCED + Associates a new_state with the provided key in the runtime's monadic state, returning the provided state. Arguments: diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Thread.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Thread.enso index 5c2d5ba38f04..50d858cc3476 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Thread.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Runtime/Thread.enso @@ -3,8 +3,8 @@ import project.Any.Any -## ADVANCED - +## PRIVATE + ADVANCED Executes an action with a handler for the executing thread being interrupted. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System.enso index 7494908bf18f..ae8d3534888e 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System.enso @@ -26,7 +26,9 @@ polyglot java import java.lang.System as Java_System create_process : Text -> Array -> Text -> Boolean -> Boolean -> Boolean -> System_Process_Result create_process command arguments input redirect_in redirect_out redirect_err = @Builtin_Method "System.create_process" -## Exits the Enso program, returning the provided code to the parent +## PRIVATE + ADVANCED + Exits the Enso program, returning the provided code to the parent process. Arguments: @@ -39,7 +41,9 @@ create_process command arguments input redirect_in redirect_out redirect_err = @ exit : Integer -> Nothing exit code = @Builtin_Method "System.exit" -## Gets the nanosecond resolution system time at the moment of the call. +## PRIVATE + ADVANCED + Gets the nanosecond resolution system time at the moment of the call. > Example Getting the current value of the nanosecond timer. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso index fdd245a1c4a5..e681d8e85d96 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso @@ -90,7 +90,9 @@ type File home : File home = @Builtin_Method "File.home" - ## Creates a new output stream for this file and runs the specified action + ## PRIVATE + ADVANCED + Creates a new output stream for this file and runs the specified action on it. Arguments: @@ -117,8 +119,7 @@ type File Managed_Resource.bracket (self.new_output_stream open_options) (_.close) action ## PRIVATE - - ## Creates a new output stream for this file. Recommended to use + Creates a new output stream for this file. Recommended to use `File.with_output_stream` instead, which does resource management. Arguments: @@ -128,8 +129,7 @@ type File output_stream self options = @Builtin_Method "File.output_stream" ## PRIVATE - - ## Creates a new input stream for this file. Recommended to use + Creates a new input stream for this file. Recommended to use `File.with_input_stream` instead, which does resource management. Arguments: @@ -138,7 +138,9 @@ type File input_stream : Vector File_Access -> Input_Stream input_stream self options = @Builtin_Method "File.input_stream" - ## Creates a new input stream for this file and runs the specified action + ## PRIVATE + ADVANCED + Creates a new input stream for this file and runs the specified action on it. Arguments: @@ -279,7 +281,8 @@ type File resolve : File resolve self = @Builtin_Method "File.resolve" - ## Convert the file descriptor to a JS_Object. + ## PRIVATE + Convert the file descriptor to a JS_Object. > Example Convert a file to a JS_Object. @@ -532,8 +535,8 @@ type File delete_if_exists : Nothing ! File_Error delete_if_exists self = if self.exists then self.delete else Nothing - ## ADVANCED - + ## PRIVATE + ADVANCED Returns a new input stream for this file. Arguments: @@ -550,8 +553,8 @@ type File resource = Managed_Resource.register stream close_stream Input_Stream.Value self resource - ## ADVANCED - + ## PRIVATE + ADVANCED Returns a new output stream for this file. Arguments: @@ -580,7 +583,6 @@ type File Output_Stream.Value self resource ## PRIVATE - Reads last `n` bytes from the file (or less if the file is too small) and returns a vector of bytes. read_last_bytes : Integer -> Vector ! File_Error @@ -655,32 +657,27 @@ type File matcher.matches (Path.of pathStr) filtered - ## UNSTABLE - - Checks if `self` is a child path of `other`. + ## Checks if `self` is a child path of `other`. is_child_of : File -> Boolean is_child_of self other = self.starts_with other - ## UNSTABLE - - Transforms `child` to a relative path with respect to `self`. + ## Transforms `child` to a relative path with respect to `self`. relativize : File -> Boolean relativize self child = @Builtin_Method "File.relativize" ## PRIVATE - Utility function that lists immediate children of a directory. list_immediate_children : Vector File list_immediate_children self = Vector.from_polyglot_array (self.list_immediate_children_array) ## PRIVATE - Return the absolute path of this File to_text : Text to_text self = self.absolute . path -## An output stream, allowing for interactive writing of contents into an +## PRIVATE + An output stream, allowing for interactive writing of contents into an open file. type Output_Stream ## PRIVATE @@ -694,8 +691,8 @@ type Output_Stream stream. Value file stream_resource - ## ADVANCED - + ## PRIVATE + ADVANCED Writes a vector of bytes into the file at the current stream position. Arguments: @@ -719,8 +716,8 @@ type Output_Stream java_stream.flush Nothing - ## ADVANCED - + ## PRIVATE + ADVANCED Closes this stream. Even though Streams are closed automatically upon garbage collection, it @@ -741,7 +738,7 @@ type Output_Stream close self = self.stream_resource . finalize ## PRIVATE - + ADVANCED Exposes operations on the underlying Java output stream. Arguments: @@ -753,6 +750,7 @@ type Output_Stream with_java_stream self f = self.stream_resource . with f ## PRIVATE + ADVANCED Runs an action with a `ReportingStreamEncoder` encoding data to the output stream with the specified encoding. with_stream_encoder : Encoding -> Problem_Behavior -> (ReportingStreamEncoder -> Any) -> Any @@ -769,7 +767,8 @@ type Output_Stream problems = Vector.from_polyglot_array results.problems . map Encoding_Error.Error on_problems.attach_problems_after results.result problems -## An input stream, allowing for interactive reading of contents from an open +## PRIVATE + An input stream, allowing for interactive reading of contents from an open file. type Input_Stream @@ -784,8 +783,8 @@ type Input_Stream stream. Value file stream_resource - ## ADVANCED - + ## PRIVATE + ADVANCED Reads all the bytes in this file into a vector of bytes. > Example @@ -805,8 +804,8 @@ type Input_Stream File_Error.handle_java_exceptions self.file <| Vector.from_polyglot_array java_stream.readAllBytes - ## ADVANCED - + ## PRIVATE + ADVANCED Reads _up to_ the provided number of bytes from the stream. Arguments: @@ -836,8 +835,8 @@ type Input_Stream bytes = java_stream.readNBytes n Vector.from_polyglot_array bytes - ## ADVANCED - + ## PRIVATE + ADVANCED Reads the next byte from the stream. The returned value is an integer in the range 0-255 representing the @@ -860,8 +859,8 @@ type Input_Stream File_Error.handle_java_exceptions self.file <| java_stream.read - ## ADVANCED - + ## PRIVATE + ADVANCED Closes this stream. Even though Streams are closed automatically upon garbage collection, it diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/File_Permissions.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/File_Permissions.enso index ffb3b82f94d6..314f6bd1125d 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/File_Permissions.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File/File_Permissions.enso @@ -14,10 +14,13 @@ type Permission Execute type File_Permissions - ## Access permissions for a file. + ## PRIVATE + Access permissions for a file. Value (owner : Vector Permission) (group : Vector Permission) (others : Vector Permission) - ## Converts the Enso atom to its Java enum counterpart. + ## PRIVATE + ADVANCED + Converts the Enso atom to its Java enum counterpart. to_java : Vector PosixFilePermission to_java self = result = Vector.new_builder @@ -77,7 +80,9 @@ type File_Permissions others_execute : Boolean others_execute self = self.others.contains Permission.Execute - ## Converts a Java `Set` of Java `PosixFilePermission` to `File_Permissions`. + ## PRIVATE + ADVANCED + Converts a Java `Set` of Java `PosixFilePermission` to `File_Permissions`. from_java_set java_set = owner = Vector.new_builder group = Vector.new_builder diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso index 5c34bb762c79..5cbface4a5f6 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso @@ -35,7 +35,6 @@ get_format callback = @Tail_Call reader (idx + 1) reader 0 - type Auto_Detect ## PRIVATE Implements the `File.read` for this `File_Format` @@ -55,7 +54,6 @@ type Auto_Detect get_web_parser content_type uri = get_format f-> f.for_web content_type uri - type File_Format ## Gets all the currently available file formats. @@ -73,7 +71,8 @@ type File_Format type Plain_Text_Format Plain_Text (encoding:Encoding=Encoding.utf_8) - ## If the File_Format supports reading from the file, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the file, return a configured instance. for_file : File -> Plain_Text_Format | Nothing for_file file = case file.extension of @@ -81,7 +80,8 @@ type Plain_Text_Format ".log" -> Plain_Text_Format.Plain_Text _ -> Nothing - ## If the File_Format supports reading from the web response, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the web response, return a configured instance. for_web : Text -> URI -> Plain_Text_Format | Nothing for_web content_type _ = parts = content_type.split ";" . map .trim @@ -107,14 +107,16 @@ type Plain_Text_Format Text.from_bytes response.body.bytes self.encoding type Bytes - ## If the File_Format supports reading from the file, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the file, return a configured instance. for_file : File -> Bytes | Nothing for_file file = case file.extension of ".dat" -> Bytes _ -> Nothing - ## If the File_Format supports reading from the web response, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the web response, return a configured instance. As `Bytes`, does not support reading from the web returns `Nothing`. for_web : Text -> URI -> Bytes | Nothing for_web _ _ = Nothing @@ -126,7 +128,8 @@ type Bytes file.read_bytes type JSON_File - ## If the File_Format supports reading from the file, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the file, return a configured instance. for_file : File -> JSON_File | Nothing for_file file = case file.extension of @@ -134,7 +137,8 @@ type JSON_File ".geojson" -> JSON_File _ -> Nothing - ## If the File_Format supports reading from the web response, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the web response, return a configured instance. for_web : Text -> URI -> JSON_File | Nothing for_web content_type _ = first = content_type.split ';' . first . trim diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/Platform.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/Platform.enso index fc58a13086af..cbdba7b4b5f0 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/Platform.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/Platform.enso @@ -4,7 +4,6 @@ import project.System ## A representation of the various operating systems on which Enso can run. type OS - ## The Linux operating system. Linux @@ -33,7 +32,6 @@ is_unix : Boolean is_unix = @Builtin_Method "System.is_unix" ## PRIVATE - Create an Os object from text. from_text : Text -> OS from_text os = case os of diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process.enso index ae4692d3a7c6..e78b63387788 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process.enso @@ -29,7 +29,9 @@ run : Text -> Vector Text -> Text -> Process_Result run command arguments=[] stdin="" = new_builder command arguments stdin . create -## Create a new process builder. +## PRIVATE + ADVANCED + Create a new process builder. Arguments: - command: The command to execute on the system. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process/Exit_Code.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process/Exit_Code.enso index 931511ced3ea..83a88bc9d779 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process/Exit_Code.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process/Exit_Code.enso @@ -2,7 +2,9 @@ import project.Data.Numbers.Integer ## The exit codes that the process can return. type Exit_Code - ## Create exit code from a number. + ## PRIVATE + ADVANCED + Create exit code from a number. Arguments: - code: The exit code you want to create. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process/Process_Builder.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process/Process_Builder.enso index a9d34226860c..6d33b4a212f8 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process/Process_Builder.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/Process/Process_Builder.enso @@ -6,8 +6,8 @@ import project.System.System_Process_Result from project.Data.Boolean import Boolean, True, False -## UNSTABLE - +## PRIVATE + ADVANCED The builder object that is used to create operating system processes. type Process_Builder ## PRIVATE @@ -25,9 +25,7 @@ type Process_Builder set arguments and standard output. It results in much clearer code. Value command arguments stdin - ## UNSTABLE - - Sets the arguments that should be passed to the created process. + ## Sets the arguments that should be passed to the created process. Arguments: - arguments: The arguments to pass to the process. @@ -43,9 +41,7 @@ type Process_Builder set_arguments : Vector Text -> Process_Builder set_arguments self arguments = Process_Builder.Value self.command arguments self.stdin - ## UNSTABLE - - Sets the text that will be used to feed standard input to the created + ## Sets the text that will be used to feed standard input to the created process. Arguments: @@ -62,9 +58,7 @@ type Process_Builder set_stdin : Text -> Process_Builder set_stdin self stdin = Process_Builder.Value self.command self.arguments stdin - ## UNSTABLE - - Create a process using a builder returning the result of execution. + ## Create a process using a builder returning the result of execution. > Example Execute the process contained in the builder. diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Warning.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Warning.enso index f61a2faa4c08..70c8289cc962 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Warning.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Warning.enso @@ -17,28 +17,26 @@ from project.Data.Boolean import Boolean, True, False ## A representation of a dataflow warning attached to a value. @Builtin_Type type Warning - ## UNSTABLE - + ## PRIVATE + ADVANCED Attaches a new warning to the value. attach : Any -> Any -> Any attach warning value = origin = Runtime.get_stack_trace attach_with_stacktrace value warning (origin.drop (Index_Sub_Range.First 1)) - ## UNSTABLE - + ## ADVANCED Are any warnings attached to the value? has_warnings : Any -> Boolean has_warnings value = has_warnings_builtin value - ## UNSTABLE - + ## ADVANCED Gets all the warnings attached to the given value. Warnings are returned in the reverse-chronological order with respect to their attachment time. get_all : Any -> Vector Warning get_all value = Vector.from_polyglot_array (get_all_array value) - ## UNSTABLE + ## PRIVATE ADVANCED Sets a new list of warnings for the given value. Any warnings already present @@ -50,8 +48,7 @@ type Warning set : Any -> Vector Warning -> Any set value warnings = set_array value warnings.to_array - ## UNSTABLE - ADVANCED + ## ADVANCED Returns the provided value with any warnings removed from it. Arguments: @@ -59,7 +56,7 @@ type Warning clear : Any -> Any clear value = Warning.set value [] - ## UNSTABLE + ## PRIVATE ADVANCED Executes the provided function with the given argument with its warnings suspended. @@ -93,8 +90,8 @@ type Warning case arg of _ -> function (Warning.clear arg) - - ## UNSTABLE + ## PRIVATE + ADVANCED Maps warnings attached to a value. Arguments: @@ -115,7 +112,8 @@ type Warning map_attached_warnings mapper value = map_attached_warnings_helper mapper value 1 - ## UNSTABLE + ## PRIVATE + ADVANCED An utility function which applies the mapping function both to any attached warnings and dataflow errors. @@ -143,7 +141,7 @@ type Warning original dataflow error as-is, to preserve its stacktrace. Nothing -> mapped_warnings_or_error - ## UNSTABLE + ## ADVANCED A helper function which selects warnings matching a predicate and returns a pair whose first element is the original value with the matched warnings removed and the second element is the list of matched warnings. @@ -167,7 +165,8 @@ type Warning remaining = result.second Pair.new (Warning.set value remaining) matched - ## UNSTABLE + ## PRIVATE + ADVANCED A helper function which gathers warnings matching some predicate and passes them into a function which can aggregate them. @@ -195,20 +194,17 @@ type Warning new_warnings.fold result.first acc-> warning-> Warning.attach warning acc - ## UNSTABLE - - Returns the warning value – usually its explanation or other contents. + ## Returns the warning value – usually its explanation or other contents. value : Any value self = @Builtin_Method "Warning.value" - ## UNSTABLE - ADVANCED + ## ADVANCED A stack trace for the original warning creation. origin : Vector Stack_Trace_Element origin self = @Builtin_Method "Warning.origin" - ## UNSTABLE + ## PRIVATE ADVANCED A list of locations where the warning was reassigned in the order of diff --git a/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/NOTICE index 3caa2f08e41c..7fa7dc342520 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/NOTICE +++ b/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/NOTICE @@ -83,7 +83,7 @@ Copyright notices related to this dependency can be found in the directory `org. 'sqlite-jdbc', licensed under the The Apache Software License, Version 2.0, is distributed with the Database. The license information can be found along with the copyright notices. -Copyright notices related to this dependency can be found in the directory `org.xerial.sqlite-jdbc-3.36.0.3`. +Copyright notices related to this dependency can be found in the directory `org.xerial.sqlite-jdbc-3.41.2.1`. 'ion-java', licensed under the The Apache License, Version 2.0, is distributed with the Database. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/NOTICES b/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/NOTICES deleted file mode 100644 index 81aa5300d8b1..000000000000 --- a/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/NOTICES +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2007 David Crawshaw - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Copyright (c) 2021 Gauthier Roebroeck - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -Copyright (c) 2007 David Crawshaw - -Copyright 2007 Taro L. Saito - -Copyright 2008 Taro L. Saito - -Copyright 2009 Taro L. Saito - -Copyright 2010 Taro L. Saito - -Copyright 2016 Magnus Reftel diff --git a/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/LICENSE b/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/LICENSE similarity index 100% rename from distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/LICENSE rename to distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/LICENSE diff --git a/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/LICENSE.zentus b/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/LICENSE.zentus similarity index 100% rename from distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/LICENSE.zentus rename to distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/LICENSE.zentus diff --git a/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/NOTICES b/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/NOTICES new file mode 100644 index 000000000000..f149f0d20a8e --- /dev/null +++ b/distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.41.2.1/NOTICES @@ -0,0 +1,11 @@ +Copyright (c) 2007 David Crawshaw + +Copyright (c) 2021 Gauthier Roebroeck + +Copyright 2007 Taro L. Saito + +Copyright 2016 Magnus Reftel + +copyright notice and this permission notice appear in all copies. + +this work for additional information regarding copyright ownership. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso index 1d6893e5d87f..77a94658aef4 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso @@ -33,7 +33,8 @@ type Connection - dialect: the dialect associated with the database we are connected to. Value jdbc_connection dialect - ## Closes the connection releasing the underlying database resources + ## PRIVATE + Closes the connection releasing the underlying database resources immediately instead of waiting for them to be automatically released. The connection is not usable afterwards. @@ -41,18 +42,21 @@ type Connection close self = self.jdbc_connection.close - ## Returns the list of databases (or catalogs) for the connection. + ## PRIVATE + Returns the list of databases (or catalogs) for the connection. databases : Vector Text databases self = self.jdbc_connection.with_metadata metadata-> read_column metadata.getCatalogs "TABLE_CAT" - ## Returns the name of the current database (or catalog). + ## PRIVATE + Returns the name of the current database (or catalog). database : Text database self = self.jdbc_connection.with_connection connection->connection.getCatalog - ## Returns a new Connection with the specified database set as default. + ## PRIVATE + Returns a new Connection with the specified database set as default. Arguments: - database: The name of the database to connect to. @@ -61,18 +65,21 @@ type Connection if database == self.database then self else SQL_Error.throw_sql_error "Changing database is not supported." - ## Returns the list of schemas for the connection within the current database (or catalog). + ## PRIVATE + Returns the list of schemas for the connection within the current database (or catalog). schemas : Vector Text schemas self = self.jdbc_connection.with_metadata metadata-> read_column metadata.getSchemas "TABLE_SCHEM" - ## Returns the name of the current schema. + ## PRIVATE + Returns the name of the current schema. schema : Text schema self = self.jdbc_connection.with_connection .getSchema - ## Returns a new Connection with the specified schema set as default. + ## PRIVATE + Returns a new Connection with the specified schema set as default. Arguments: - schema: The name of the schema to connect to. @@ -81,13 +88,15 @@ type Connection if schema == self.schema then self else SQL_Error.throw_sql_error "Changing schema is not supported." - ## Gets a list of the table types + ## PRIVATE + Gets a list of the table types table_types : [Text] table_types self = self.jdbc_connection.with_metadata metadata-> read_column metadata.getTableTypes "TABLE_TYPE" - ## Returns a materialized Table of all the matching views and tables. + ## PRIVATE + Returns a materialized Table of all the matching views and tables. Arguments: - name_like: The table name pattern to search for. Supports SQL wildcards (`%`, `_`). Defaults to `Nothing` which @@ -107,7 +116,8 @@ type Connection if all_fields then renamed else renamed.select_columns ["Database", "Schema", "Name", "Type", "Description"] - ## Set up a query returning a Table object, which can be used to work with + ## PRIVATE + Set up a query returning a Table object, which can be used to work with data within the database or load it into memory. Arguments: @@ -136,7 +146,7 @@ type Connection Error.throw (Table_Not_Found.Error query sql_error treated_as_query=True) SQL_Query.Raw_SQL raw_sql -> handle_sql_errors <| self.jdbc_connection.ensure_query_has_no_holes raw_sql . if_not_error <| - columns = self.jdbc_connection.fetch_columns raw_sql Statement_Setter.null + columns = self.fetch_columns raw_sql Statement_Setter.null name = if alias == "" then (UUID.randomUUID.to_text) else alias ctx = Context.for_query raw_sql name Database_Table_Module.make_table self name columns ctx @@ -145,12 +155,13 @@ type Connection ctx = Context.for_table name (if alias == "" then name else alias) statement = self.dialect.generate_sql (Query.Select Nothing ctx) statement_setter = self.dialect.get_statement_setter - columns = self.jdbc_connection.fetch_columns statement statement_setter + columns = self.fetch_columns statement statement_setter Database_Table_Module.make_table self name columns ctx result.catch SQL_Error sql_error-> Error.throw (Table_Not_Found.Error name sql_error treated_as_query=False) - ## Execute the query and load the results into memory as a Table. + ## PRIVATE + Execute the query and load the results into memory as a Table. Arguments: - query: name of the table or sql statement to query. @@ -171,14 +182,23 @@ type Connection to ensure that default `ResultSet` metadata is used for these columns. - last_row_only: If set true, only the last row of the query is fetched. Defaults to false. - read_statement : SQL_Statement -> (Nothing | Vector SQL_Type_Reference) -> Materialized_Table + read_statement : SQL_Statement -> (Nothing | Vector SQL_Type_Reference) -> Boolean -> Materialized_Table read_statement self statement column_type_suggestions=Nothing last_row_only=False = type_overrides = self.dialect.get_type_mapping.prepare_type_overrides column_type_suggestions statement_setter = self.dialect.get_statement_setter self.jdbc_connection.with_prepared_statement statement statement_setter stmt-> result_set_to_table stmt.executeQuery self.dialect.make_column_fetcher_for_type type_overrides last_row_only - ## ADVANCED + ## PRIVATE + Given a prepared statement, gets the column names and types for the + result set. + fetch_columns : Text | SQL_Statement -> Statement_Setter -> Any + fetch_columns self statement statement_setter = + needs_execute_query = self.dialect.needs_execute_query_for_type_inference + self.jdbc_connection.raw_fetch_columns statement needs_execute_query statement_setter + + ## PRIVATE + ADVANCED Executes a raw update query. If the query was inserting, updating or deleting rows, the number of affected rows is returned; otherwise it diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection_Options.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection_Options.enso index f8e0db840f2d..5ed8f4c3c58d 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection_Options.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection_Options.enso @@ -1,11 +1,12 @@ from Standard.Base import all - type Connection_Options ## Hold a set of key value pairs used to configure the connection. Value options:Vector=[] - ## Merge the base set of options with the overrides in this object. + ## PRIVATE + ADVANCED + Merge the base set of options with the overrides in this object. merge : Vector -> Vector merge self base_options = base_options.filter x->(self.options.any (y->y.first==x.first) . not) + self.options diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Credentials.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Credentials.enso index f50a660b8bfa..f111dff2e1b9 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Credentials.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Credentials.enso @@ -4,6 +4,7 @@ type Credentials ## Simple username and password type. Username_And_Password username:Text password:Text - ## Override `to_text` to mask the password field. + ## PRIVATE + Override `to_text` to mask the password field. to_text : Text to_text self = 'Credentials ' + self.username + ' *****' diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Options.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Options.enso index 69e77528bd8a..b0157aaf8697 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Options.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Options.enso @@ -28,7 +28,8 @@ type Postgres_Options - client_cert: The client certificate to use or `Nothing` if not needed. Postgres (host:Text=default_postgres_host) (port:Integer=default_postgres_port) (database:Text=default_postgres_database) (schema:Text="") (credentials:(Credentials|Nothing)=Nothing) (use_ssl:SSL_Mode=SSL_Mode.Prefer) (client_cert:(Client_Certificate|Nothing)=Nothing) - ## Build the Connection resource. + ## PRIVATE + Build the Connection resource. Arguments: - options: Overrides for the connection properties. @@ -44,12 +45,14 @@ type Postgres_Options Postgres_Connection.create self.jdbc_url properties make_new - ## Provides the jdbc url for the connection. + ## PRIVATE + Provides the jdbc url for the connection. jdbc_url : Text jdbc_url self = 'jdbc:postgresql://' + self.host + ':' + self.port.to_text + (if self.database == '' then '' else '/' + self.database) - ## Provides the properties for the connection. + ## PRIVATE + Provides the properties for the connection. jdbc_properties : [Pair Text Text] jdbc_properties self = credentials = case self.credentials of diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Redshift_Options.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Redshift_Options.enso index 385a15090783..0cc44d18cb42 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Redshift_Options.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Redshift_Options.enso @@ -25,7 +25,8 @@ type Redshift_Options - client_cert: The client certificate to use or `Nothing` if not needed. Redshift (host:Text) (port:Integer=5439) (schema:Text='') (credentials:Credentials|AWS_Credential|Nothing=Nothing) (use_ssl:SSL_Mode=SSL_Mode.Require) (client_cert:Client_Certificate|Nothing=Nothing) - ## Build the Connection resource. + ## PRIVATE + Build the Connection resource. Arguments: - options: Overrides for the connection properties. @@ -41,7 +42,8 @@ type Redshift_Options jdbc_connection = JDBC_Connection.create self.jdbc_url properties Connection.Value jdbc_connection Dialect.redshift - ## Provides the jdbc url for the connection. + ## PRIVATE + Provides the jdbc url for the connection. jdbc_url : Text jdbc_url self = prefix = case self.credentials of @@ -49,7 +51,8 @@ type Redshift_Options _ -> 'jdbc:redshift://' prefix + self.host + ':' + self.port.to_text + (if self.schema == '' then '' else '/' + self.schema) - ## Provides the properties for the connection. + ## PRIVATE + Provides the properties for the connection. jdbc_properties : [Pair Text Text] jdbc_properties self = credentials = case self.credentials of diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Format.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Format.enso index 7c1f8450881b..4a2dac39a0d8 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Format.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Format.enso @@ -9,7 +9,8 @@ type SQLite_Format ## Read SQLite files For_File - ## If the File_Format supports reading from the file, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the file, return a configured instance. for_file : File -> SQLite_Format | Nothing for_file file = case file.extension of @@ -17,13 +18,15 @@ type SQLite_Format ".sqlite" -> SQLite_Format.For_File _ -> Nothing - ## If the File_Format supports reading from the web response, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the web response, return a configured instance. for_web : Text -> URI -> SQLite_Format | Nothing for_web _ _ = ## Currently not loading SQLite files automatically. Nothing - ## Implements the `File.read` for this `File_Format` + ## PRIVATE + Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any read self file on_problems = _ = [on_problems] diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Options.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Options.enso index 348ad5924803..ae135623e8b9 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Options.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Options.enso @@ -10,7 +10,8 @@ type SQLite_Options - location: Location of the SQLite database to connect to. SQLite (location:(In_Memory|File|Text)) - ## Build the Connection resource. + ## PRIVATE + Build the Connection resource. Arguments: - options: Overrides for the connection properties. @@ -19,13 +20,15 @@ type SQLite_Options properties = options.merge self.jdbc_properties SQLite_Connection.create self.jdbc_url properties - ## Provides the jdbc url for the connection. + ## PRIVATE + Provides the jdbc url for the connection. jdbc_url : Text jdbc_url self = case self.location of In_Memory -> "jdbc:sqlite::memory:" _ -> "jdbc:sqlite:" + ((File.new self.location).absolute.path.replace '\\' '/') - ## Provides the properties for the connection. + ## PRIVATE + Provides the properties for the connection. jdbc_properties : Vector jdbc_properties self = [] diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 944c0cbeebe3..42afd4a5a820 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -8,7 +8,7 @@ import Standard.Table.Internal.Java_Problems import Standard.Table.Internal.Problem_Builder.Problem_Builder import Standard.Table.Internal.Widget_Helpers from Standard.Table import Sort_Column, Data_Formatter, Value_Type, Auto -from Standard.Table.Errors import Floating_Point_Equality, Inexact_Type_Coercion, Invalid_Value_Type +from Standard.Table.Errors import Floating_Point_Equality, Inexact_Type_Coercion, Invalid_Value_Type, Lossy_Conversion import project.Data.SQL_Statement.SQL_Statement import project.Data.SQL_Type.SQL_Type @@ -26,9 +26,7 @@ from project.Errors import Unsupported_Database_Operation, Integrity_Error, Unsu polyglot java import org.enso.table.data.column.operation.map.MapOperationProblemBuilder type Column - - ## UNSTABLE - PRIVATE + ## PRIVATE Represents a single column backed by a database. @@ -47,8 +45,8 @@ type Column able to be combined. Value name:Text connection:Connection sql_type_reference:SQL_Type_Reference expression:SQL_Expression context:Context - ## UNSTABLE - + ## PRIVATE + ADVANCED Returns a text containing an ASCII-art table displaying this data. Arguments: @@ -58,8 +56,8 @@ type Column display self show_rows=10 format_terminal=False = self.to_table.display show_rows format_terminal - ## UNSTABLE - + ## PRIVATE + ADVANCED Prints an ASCII-art table with this data to the standard output. Arguments: @@ -69,22 +67,24 @@ type Column IO.println (self.display show_rows format_terminal=True) IO.println '' - ## UNSTABLE - + ## PRIVATE Converts this column to JS_Object representation. to_js_object : JS_Object to_js_object self = self.to_sql.to_js_object - ## UNSTABLE - - Converts this column into a single-column table. + ## Converts this column into a single-column table. to_table : Table to_table self = Table.Value self.name self.connection [self.as_internal] self.context - ## UNSTABLE + ## Returns a Table describing this column's contents. - Returns a materialized column containing rows of this column. + The table behaves like `Table.info` - it lists the column name, the count + of non-null items and the value type. + info : Table + info self = self.to_table.info + + ## Returns a materialized column containing rows of this column. Arguments: - max_rows: specifies a maximum amount of rows to fetch; if not set, all @@ -93,26 +93,21 @@ type Column read self max_rows=Nothing = self.to_table.read max_rows . at self.name - ## UNSTABLE - - Returns a vector containing all the elements in this column. + ## Returns a vector containing all the elements in this column. to_vector : Vector Any to_vector self = self.to_table.read . at self.name . to_vector - ## UNSTABLE TODO this is a very early prototype that will be revisited later - This implementation is really just so that we can use the types in - `filter`, it does not provide even a decent approximation of the true - type in many cases. It will be improved when the types work is - implemented. + ## Returns the `Value_Type` associated with that column. + + The value type determines what type of values the column is storing and + what operations are permitted. value_type : Value_Type value_type self = mapping = self.connection.dialect.get_type_mapping mapping.sql_type_to_value_type self.sql_type - ## UNSTABLE - - Returns an SQL statement that will be used for materializing this column. + ## Returns an SQL statement that will be used for materializing this column. to_sql : SQL_Statement to_sql self = self.to_table.to_sql @@ -660,7 +655,7 @@ type Column Returns a new column where missing values have been replaced with the provided default. - fill_nothing : Any -> Column + fill_nothing : Column | Any -> Column fill_nothing self default = new_name = self.naming_helpers.function_name "fill_nothing" [self, default] self.make_binary_op "FILL_NULL" default new_name @@ -820,7 +815,7 @@ type Column Applies only to columns that hold the `Date` or `Date_Time` types. Returns a column of `Integer` type. - year : Column -> Column ! Invalid_Value_Type + year : Column ! Invalid_Value_Type year self = Value_Type.expect_has_date self.value_type related_column=self.name <| self.make_unary_op "year" @@ -828,7 +823,7 @@ type Column Applies only to columns that hold the `Date` or `Date_Time` types. Returns a column of `Integer` type. - month : Column -> Column ! Invalid_Value_Type + month : Column ! Invalid_Value_Type month self = Value_Type.expect_has_date self.value_type related_column=self.name <| self.make_unary_op "month" @@ -837,7 +832,7 @@ type Column Applies only to columns that hold the `Date` or `Date_Time` types. Returns a column of `Integer` type. - day : Column -> Column ! Invalid_Value_Type + day : Column ! Invalid_Value_Type day self = Value_Type.expect_has_date self.value_type related_column=self.name <| self.make_unary_op "day" @@ -907,11 +902,68 @@ type Column ## Parsing values is not supported in database columns. @type Widget_Helpers.parse_type_selector - parse : (Auto|Integer|Decimal|Date|Date_Time|Time_Of_Day|Boolean) -> Text | Data_Formatter -> Problem_Behavior -> Column + parse : Value_Type | Auto -> Text | Data_Formatter -> Problem_Behavior -> Column parse self type=Auto format=Data_Formatter.Value on_problems=Report_Warning = _ = [type, format, on_problems] Error.throw <| Unsupported_Database_Operation.Error "`Column.parse` is not implemented yet for the Database backends." + ## PRIVATE + UNSTABLE + Cast the column to a specific type. + + Arguments: + - value_type: The `Value_Type` to cast the column to. + - on_problems: Specifies how to handle problems if they occur, reporting + them as warnings by default. + + TODO [RW] this is a prototype needed for debugging, proper implementation + and testing will come with #6112. + + In the Database backend, this will boil down to a CAST operation. + In the in-memory backend, a conversion will be performed according to + the following rules: + - Anything can be cast into the `Mixed` type. + - Converting to a `Char` type, the elements of the column will be + converted to text. If it is fixed length, the texts will be trimmed or + padded on the right with the space character to match the desired + length. + - Conversion between numeric types will replace values exceeding the + range of the target type with `Nothing`. + - Booleans may also be converted to numbers, with `True` being converted + to `1` and `False` to `0`. The reverse is not supported - use `iif` + instead. + - A `Date_Time` may be converted into a `Date` or `Time` type - the + resulting value will be truncated to the desired type. + - If a `Date` is to be converted to `Date_Time`, it will be set at + midnight of the default system timezone. + + ? Conversion Precision + + In the in-memory backend, if the conversion is lossy, a + `Lossy_Conversion` warning will be reported. The only exception is when + truncating a column which is already a text column - as then the + truncation seems like an intended behaviour, so it is not reported. If + truncating needs to occur when converting a non-text column, a warning + will still be reported. + + Currently, the warning is not reported for Database backends. + + ? Inexact Target Type + + If the backend does not support the requested target type, the closest + supported type is chosen and a `Inexact_Type_Coercion` problem is + reported. + cast : Value_Type -> Problem_Behavior -> Column ! Illegal_Argument | Inexact_Type_Coercion | Lossy_Conversion + cast self value_type=self.value_type on_problems=Problem_Behavior.Report_Warning = + dialect = self.connection.dialect + type_mapping = dialect.get_type_mapping + target_sql_type = type_mapping.value_type_to_sql value_type on_problems + target_sql_type.if_not_error <| + infer_from_database new_expression = + SQL_Type_Reference.new self.connection self.context new_expression + new_column = dialect.make_cast self.as_internal target_sql_type infer_from_database + Column.Value new_column.name self.connection new_column.sql_type_reference new_column.expression self.context + ## ALIAS Transform Column Applies `function` to each item in this column and returns the column @@ -964,7 +1016,8 @@ type Column as_internal : Internal_Column as_internal self = Internal_Column.Value self.name self.sql_type_reference self.expression - ## Provides a simplified text representation for display in the REPL and errors. + ## PRIVATE + Provides a simplified text representation for display in the REPL and errors. to_text : Text to_text self = "(Database Column "+self.name.to_text+")" diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso index e3a0dd8f4837..4733a1948b47 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso @@ -10,14 +10,17 @@ import project.Data.SQL_Statement.SQL_Statement import project.Data.SQL_Type.SQL_Type import project.Data.Table.Table import project.Internal.Column_Fetcher.Column_Fetcher +import project.Internal.IR.Context.Context import project.Internal.IR.From_Spec.From_Spec import project.Internal.IR.Internal_Column.Internal_Column import project.Internal.IR.Order_Descriptor.Order_Descriptor import project.Internal.IR.Query.Query +import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.Postgres.Postgres_Dialect import project.Internal.Redshift.Redshift_Dialect import project.Internal.SQLite.SQLite_Dialect import project.Internal.SQL_Type_Mapping.SQL_Type_Mapping +import project.Internal.SQL_Type_Reference.SQL_Type_Reference import project.Internal.Statement_Setter.Statement_Setter from project.Errors import Unsupported_Database_Operation @@ -58,14 +61,6 @@ type Dialect _ = [internal_column, sort_direction, text_ordering] Unimplemented.throw "This is an interface only." - ## PRIVATE - Prepares a join operation, returning a new table instance encapsulating a - proper query. - prepare_join : Connection -> Join_Kind -> Text -> From_Spec -> From_Spec -> Vector -> Vector -> Vector -> Table - prepare_join self connection join_kind new_table_name left_subquery right_subquery on_expressions where_expressions columns_to_select = - _ = [connection, join_kind, new_table_name, left_subquery, right_subquery, on_expressions, where_expressions, columns_to_select] - Unimplemented.throw "This is an interface only." - ## PRIVATE Prepares a distinct operation. prepare_distinct : Table -> Vector -> Case_Sensitivity -> Problem_Builder -> Table @@ -113,6 +108,61 @@ type Dialect get_statement_setter self = Unimplemented.throw "This is an interface only." + ## PRIVATE + Builds an SQL expression that casts the given expression to the given + target type. + + Arguments: + - column: the input column to transform. + - target_type: the target type. + - infer_result_type_from_database_callback: A callback that can be used + to infer the type of the newly built expression from the Database. It + should be used by default, unless an override is chosen. + make_cast : Internal_Column -> SQL_Type -> (SQL_Expression -> SQL_Type_Reference) -> Internal_Column + make_cast self column target_type infer_result_type_from_database_callback = + _ = [column, target_type, infer_result_type_from_database_callback] + Unimplemented.throw "This is an interface only." + + ## PRIVATE + Specifies if the `fetch_columns` operation needs to execute the query to + get the column types. + + In most backends, the `getMetaData` may be called on a + `PreparedStatement` directly, to infer column types without actually + executing the query. In some however, like SQLite, this is insufficient + and will yield incorrect results, so the query needs to be executed (even + though the full results may not need to be streamed). + needs_execute_query_for_type_inference : Boolean + needs_execute_query_for_type_inference self = + Unimplemented.throw "This is an interface only." + + ## PRIVATE + Specifies if the cast used to reconcile column types should be done after + performing the union. If `False`, the cast will be done before the union. + + Most databases that care about column types will want to do the cast + before the union operation to ensure that types are aligned when merging. + For an SQLite workaround to work, it's better to do the cast after the + union operation. + cast_after_union : Boolean + cast_after_union self = + Unimplemented.throw "This is an interface only." + + ## PRIVATE + Prepares a query that can be used to fetch the type of an expression in + the provided context. + + This method may modify the context to optimize the query while preserving + the types. For example, in most databases, it is fine to add + `WHERE FALSE` to the query - ensuring that the engine will not do any + actual work, but the resulting type will still be the same. There are + exceptions though, like SQLite, where the best we can do is add + `LIMIT 1`. + prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement + prepare_fetch_types_query self expression context = + _ = [expression, context] + Unimplemented.throw "This is an interface only." + ## PRIVATE Checks if the given aggregate is supported. @@ -139,3 +189,8 @@ postgres = Postgres_Dialect.postgres The dialect of Redshift databases. redshift : Dialect redshift = Redshift_Dialect.redshift + +## PRIVATE +default_fetch_types_query dialect expression context = + empty_context = context.add_where_filters [SQL_Expression.Literal "FALSE"] + dialect.generate_sql (Query.Select [["typed_column", expression]] empty_context) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL.enso index e884511c5592..229d210111ea 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL.enso @@ -6,7 +6,7 @@ import Standard.Table.Internal.Vector_Builder.Vector_Builder import project.Data.SQL_Type.SQL_Type import project.Data.SQL_Statement.SQL_Statement -## UNSTABLE +## PRIVATE A fragment of a SQL query. @@ -14,7 +14,7 @@ import project.Data.SQL_Statement.SQL_Statement SQL_Fragment.Interpolation which represents an object that will be interpolated into the query. type SQL_Fragment - ## UNSTABLE + ## PRIVATE A SQL fragment that represents raw SQL code. @@ -22,7 +22,7 @@ type SQL_Fragment - code: A fragment of SQL code. Code_Part code:Text - ## UNSTABLE + ## PRIVATE A SQL fragment that represents an object which will be interpolated into the query. @@ -31,12 +31,17 @@ type SQL_Fragment - object: A value that will be interpolated into the query. Interpolation object:Any +## PRIVATE type Builder - ## Creates a Builder representing and empty code fragment. + ## PRIVATE + ADVANCED + Creates a Builder representing and empty code fragment. empty : Builder empty = Builder.Value (Vector_Builder.empty) - ## Creates a Builder representing a code fragment containing the specified raw + ## PRIVATE + ADVANCED + Creates a Builder representing a code fragment containing the specified raw code. Arguments: @@ -46,7 +51,9 @@ type Builder vec = if text.is_empty then [] else [SQL_Fragment.Code_Part text] Builder.Value (Vector_Builder.from_vector vec) - ## Creates a Builder representing an interpolation of the given object. + ## PRIVATE + ADVANCED + Creates a Builder representing an interpolation of the given object. Arguments: - object: The object to be interpolated into the query as if it has the type @@ -54,7 +61,9 @@ type Builder interpolation : Any -> Builder interpolation object = Builder.Value (Vector_Builder.from_vector [SQL_Fragment.Interpolation object]) - ## Joins a vector of code fragments with the provided separator. + ## PRIVATE + ADVANCED + Joins a vector of code fragments with the provided separator. Arguments: - separator: The separator to use when joining the code fragments. @@ -79,8 +88,8 @@ type Builder end build the actual query in linear time. Value (fragments:(Vector_Builder SQL_Fragment)) - ## UNSTABLE - + ## PRIVATE + ADVANCED Concatenates two code fragments. Arguments: @@ -90,28 +99,28 @@ type Builder text : Text -> if text == "" then self else Builder.Value (self.fragments ++ (Builder.code text).fragments) _ -> Builder.Value (self.fragments ++ other.fragments) - ## UNSTABLE - + ## PRIVATE + ADVANCED Checks if the builder represents an empty code fragment. is_empty : Boolean is_empty self = self.fragments.is_empty - ## UNSTABLE - + ## PRIVATE + ADVANCED Builds a SQL statement. build : SQL_Statement build self = fragments = optimize_fragments self.fragments.build SQL_Statement.Value fragments - ## UNSTABLE - + ## PRIVATE + ADVANCED Wraps the code fragment in parentheses. paren : Builder paren self = Builder.code "(" ++ self ++ ")" - ## UNSTABLE - + ## PRIVATE + ADVANCED If the fragment is non empty, prepends the specified prefix to it. Arguments: diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL_Statement.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL_Statement.enso index 2ab9ba52527a..511635023129 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL_Statement.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL_Statement.enso @@ -2,9 +2,9 @@ from Standard.Base import all import project.Data.SQL.SQL_Fragment +## PRIVATE type SQL_Statement - - ## UNSTABLE + ## PRIVATE Represents a built SQL statement. @@ -15,8 +15,8 @@ type SQL_Statement interpolated for these parameters. Value (internal_fragments:(Vector SQL_Fragment)) - ## UNSTABLE - + ## PRIVATE + ADVANCED A vector of code fragments. Consists of two types of values: @@ -26,10 +26,9 @@ type SQL_Statement fragments : Vector SQL_Fragment fragments self = self.internal_fragments - ## UNSAFE - UNSTABLE + ## PRIVATE ADVANCED - + UNSAFE This function returns a raw SQL string for the query, manually handling the interpolation that is normally handled by the database engine itself. @@ -52,8 +51,8 @@ type SQL_Statement _ -> "'" + obj.to_text.replace "'" "''" + "'" strings.join "" - ## UNSTABLE - + ## PRIVATE + ADVANCED Returns a pair consisting of the SQL code with holes for values and a list for values that should be substituted. prepare self = @@ -67,8 +66,7 @@ type SQL_Statement substitutions = self.fragments.flat_map to_subst [sql, substitutions] - ## UNSTABLE - + ## PRIVATE Returns a JS_Object representation of the statement. to_js_object : JS_Object to_js_object self = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL_Type.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL_Type.enso index 2ef1b8870bb1..499fa6740de3 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL_Type.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/SQL_Type.enso @@ -6,10 +6,11 @@ import project.Data.Column.Column polyglot java import java.sql.Types polyglot java import java.sql.ResultSetMetaData -## Represents an internal SQL data-type. +## PRIVATE + Represents an internal SQL data-type. type SQL_Type - - ## Represents an internal SQL data-type. + ## PRIVATE + Represents an internal SQL data-type. Arguments: - typeid: a numerical type id, as defined in `java.sql.Types`. @@ -17,14 +18,16 @@ type SQL_Type - precision: For character types, specifies their length. See `ResultSetMetaData.getPrecision`. - scale: The scale for fixed precision numeric types. Not applicable for - other types, so it's value is undefined and will usually just be 0. + other types, so it's value is undefined. See `ResultSetMetaData.getScale`. - nullable: Specifies if the given column is nullable. May be `Nothing` if that is unknown / irrelevant for the type. TODO: the precise meaning of this will be revised with #5872. - Value (typeid : Integer) (name : Text) (precision : Nothing | Integer = Nothing) (scale : Integer = 0) (nullable : Boolean | Nothing = Nothing) + Value (typeid : Integer) (name : Text) (precision : Nothing | Integer = Nothing) (scale : Nothing | Integer = Nothing) (nullable : Boolean | Nothing = Nothing) - ## The SQL type representing a null value. + ## PRIVATE + ADVANCED + The SQL type representing a null value. null : SQL_Type null = SQL_Type.Value Types.NULL "NULL" @@ -37,8 +40,9 @@ type SQL_Type 0 -> Nothing p : Integer -> p scale = metadata.getScale ix + effective_scale = if precision.is_nothing && (scale == 0) then Nothing else scale nullable_id = metadata.isNullable ix nullable = if nullable_id == ResultSetMetaData.columnNoNulls then False else if nullable_id == ResultSetMetaData.columnNullable then True else Nothing - SQL_Type.Value typeid typename precision scale nullable + SQL_Type.Value typeid typename precision effective_scale nullable diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index a5cb21fb71fc..aa8b480738f3 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -13,6 +13,7 @@ import Standard.Table.Data.Expression.Expression import Standard.Table.Data.Expression.Expression_Error import Standard.Table.Data.Join_Condition.Join_Condition import Standard.Table.Data.Join_Kind.Join_Kind +import Standard.Table.Data.Match_Columns as Match_Columns_Helpers import Standard.Table.Data.Report_Unmatched.Report_Unmatched import Standard.Table.Data.Row.Row import Standard.Table.Data.Table.Table as Materialized_Table @@ -61,8 +62,8 @@ type Table - context: The context associated with this table. Value name:Text connection:Connection (internal_columns:(Vector Internal_Column)) context:IR.Context - ## UNSTABLE - + ## PRIVATE + ADVANCED Returns a text containing an ASCII-art table displaying this data. Arguments: @@ -74,8 +75,8 @@ type Table all_rows_count = self.row_count display_dataframe df indices_count=0 all_rows_count format_terminal - ## UNSTABLE - + ## PRIVATE + ADVANCED Prints an ASCII-art table with this data to the standard output. Arguments: @@ -85,8 +86,7 @@ type Table IO.println (self.display show_rows format_terminal=True) IO.println '' - ## UNSTABLE - + ## PRIVATE Converts this column to JS_Object representation. to_js_object : JS_Object to_js_object self = case self.internal_columns.is_empty of @@ -952,7 +952,9 @@ type Table Join_Kind.Right_Exclusive -> SQL_Join_Kind.Right problem_builder.attach_problems_before on_problems <| - self.connection.dialect.prepare_join self.connection sql_join_kind new_table_name left_setup.subquery right_setup.subquery on_expressions where_expressions columns_to_select=result_columns + new_from = From_Spec.Join sql_join_kind left_setup.subquery right_setup.subquery on_expressions + new_ctx = Context.for_subquery new_from . set_where_filters where_expressions + Table.Value new_table_name self.connection result_columns new_ctx ## ALIAS Cartesian Join Joins tables by pairing every row of the left table with every row of the @@ -1129,11 +1131,77 @@ type Table regardless of types of other columns. Mixing any other types will result in a `No_Common_Type` problem. If columns of incompatible types are meant to be mixed, at least one of them should be explicitly - retyped to the `Mixed` type to indicate that intention. + retyped to the `Mixed` type to indicate that intention. Note that the + `Mixed` type may not be supported by most Database backends. union : (Table | Vector Table) -> Match_Columns -> Boolean | Report_Unmatched -> Boolean -> Problem_Behavior -> Table union self tables match_columns=Match_Columns.By_Name keep_unmatched_columns=Report_Unmatched allow_type_widening=True on_problems=Report_Warning = - _ = [tables, match_columns, keep_unmatched_columns, allow_type_widening, on_problems] - Error.throw (Unsupported_Database_Operation.Error "Table.union is not implemented yet for the Database backends.") + all_tables = case tables of + v : Vector -> [self] + v + single_table -> [self, single_table] + all_tables.all (check_db_table "tables") . if_not_error <| + problem_builder = Problem_Builder.new + matched_column_sets = Match_Columns_Helpers.match_columns all_tables match_columns keep_unmatched_columns problem_builder + dialect = self.connection.dialect + type_mapping = dialect.get_type_mapping + merged_columns = matched_column_sets.map column_set-> + case Table_Helpers.unify_result_type_for_union column_set all_tables allow_type_widening problem_builder of + Nothing -> Nothing + result_type : Value_Type -> + sql_type = type_mapping.value_type_to_sql result_type Problem_Behavior.Report_Error + sql_type.catch Inexact_Type_Coercion error-> + Panic.throw <| + Illegal_State.Error "Unexpected inexact type coercion in Union. The union logic should only operate in types supported by the given backend. This is a bug in the Database library. The coercion was: "+error.to_display_text cause=error + [column_set, sql_type] + good_columns = merged_columns.filter r-> r.is_nothing.not + if good_columns.is_empty then Error.throw No_Output_Columns else + problem_builder.attach_problems_before on_problems <| + cast_after_union = dialect.cast_after_union + queries = all_tables.map_with_index i-> t-> + columns_to_select = good_columns.map description-> + column_set = description.first + result_type = description.second + column_name = column_set.name + input_column = case column_set.column_indices.at i of + Nothing -> + typ = SQL_Type_Reference.from_constant SQL_Type.null + expr = SQL_Expression.Literal "NULL" + Internal_Column.Value column_name typ expr + corresponding_column_index : Integer -> + t.at corresponding_column_index . as_internal . rename column_name + ## We return `null` return type, as this type should + never be queried - we will just put it into the + union and the overall queried type will be taken + from there. This is just needed to create an + internal representation. + infer_return_type _ = SQL_Type_Reference.null + if cast_after_union then input_column else + dialect.make_cast input_column result_type infer_return_type + pairs = columns_to_select.map c-> + [c.name, c.expression] + Query.Select pairs t.context + + union_alias = all_tables.map .name . join "_" + new_from = From_Spec.Union queries union_alias + new_ctx = Context.for_subquery new_from + ## TODO [RW] The result type is currently fetched + independently for each column, instead we should fetch it + for all columns at once. + See #6118. + infer_return_type expression = + SQL_Type_Reference.new self.connection new_ctx expression + new_columns = good_columns.map description-> + column_set = description.first + result_type = description.second + name = column_set.name + expression = SQL_Expression.Column union_alias name + case cast_after_union of + True -> + input_column = Internal_Column.Value name SQL_Type_Reference.null expression + dialect.make_cast input_column result_type infer_return_type + False -> + Internal_Column.Value name (infer_return_type expression) expression + + Table.Value union_alias self.connection new_columns new_ctx ## ALIAS group, summarize @@ -1316,8 +1384,8 @@ type Table ## Parsing values is not supported in database tables, the table has to be loaded into memory first with `read`. - parse_values : Text | Integer | Column_Selector | Vector (Text | Integer | Column_Selector) -> (Auto|Integer|Decimal|Date|Date_Time|Time_Of_Day|Boolean) -> Text | Data_Formatter -> Boolean -> Problem_Behavior -> Table - parse_values columns=(self.columns . filter (c-> c.value_type.is_text) . map .name) type=Auto format=Data_Formatter.Value error_on_missing_columns=True on_problems=Report_Warning = + parse : Text | Integer | Column_Selector | Vector (Text | Integer | Column_Selector) -> Value_Type | Auto -> Text | Data_Formatter -> Boolean -> Problem_Behavior -> Table + parse columns=(self.columns . filter (c-> c.value_type.is_text) . map .name) type=Auto format=Data_Formatter.Value error_on_missing_columns=True on_problems=Report_Warning = ## Avoid unused arguments warning. We cannot rename arguments to `_`, because we need to keep the API consistent with the in-memory table. _ = [columns, type, format, error_on_missing_columns, on_problems] @@ -1394,6 +1462,63 @@ type Table _ = [column pattern case_sensitivity on_problems] Unimplemented.throw "This is an interface only." + ## PRIVATE + UNSTABLE + Cast the selected columns to a specific type. + + Returns a new table in which the selected columns are replaced with + columns having the new types. + + Arguments: + - columns: The selection of columns to cast. + - value_type: The `Value_Type` to cast the column to. + - on_problems: Specifies how to handle problems if they occur, reporting + them as warnings by default. + + TODO [RW] this is a prototype needed for debugging, proper implementation + and testing will come with #6112. + + In the Database backend, this will boil down to a CAST operation. + In the in-memory backend, a conversion will be performed according to + the following rules: + - Anything can be cast into the `Mixed` type. + - Converting to a `Char` type, the elements of the column will be + converted to text. If it is fixed length, the texts will be trimmed or + padded on the right with the space character to match the desired + length. + - Conversion between numeric types will replace values exceeding the + range of the target type with `Nothing`. + - Booleans may also be converted to numbers, with `True` being converted + to `1` and `False` to `0`. The reverse is not supported - use `iif` + instead. + - A `Date_Time` may be converted into a `Date` or `Time` type - the + resulting value will be truncated to the desired type. + - If a `Date` is to be converted to `Date_Time`, it will be set at + midnight of the default system timezone. + + ? Conversion Precision + + In the in-memory backend, if the conversion is lossy, a + `Lossy_Conversion` warning will be reported. The only exception is when + truncating a column which is already a text column - as then the + truncation seems like an intended behaviour, so it is not reported. If + truncating needs to occur when converting a non-text column, a warning + will still be reported. + + Currently, the warning is not reported for Database backends. + + ? Inexact Target Type + + If the backend does not support the requested target type, the closest + supported type is chosen and a `Inexact_Type_Coercion` problem is + reported. + cast : (Text | Integer | Column_Selector | Vector (Integer | Text | Column_Selector)) -> Value_Type -> Problem_Behavior -> Table ! Illegal_Argument | Inexact_Type_Coercion | Lossy_Conversion + cast self columns=[0] value_type=Value_Type.Char on_problems=Problem_Behavior.Report_Warning = + selected = self.select_columns columns + selected.columns.fold self table-> column_to_cast-> + new_column = column_to_cast.cast value_type on_problems + table.set new_column new_name=column_to_cast.name set_mode=Set_Mode.Update + ## ALIAS dropna ALIAS drop_missing_rows Remove rows which are all blank or containing blank values. @@ -1458,9 +1583,9 @@ type Table ## Returns a Table describing this table's contents. - The table lists all columns, counts of non-null items and storage types - of each column. - info : Table + The table lists all columns, counts of non-null items and value types of + each column. + info : Materialized_Table info self = cols = self.internal_columns count_query = @@ -1564,7 +1689,8 @@ type Table False -> Error.throw <| Illegal_State.Error "The update unexpectedly affected "+affected_rows.to_text+" rows." True -> Nothing - ## Provides a simplified text representation for display in the REPL and errors. + ## PRIVATE + Provides a simplified text representation for display in the REPL and errors. to_text : Text to_text self = "(Database Table "+self.name.to_text+")" @@ -1721,3 +1847,16 @@ freshen_columns used_names columns = type Wrapped_Error ## PRIVATE Value value + +## PRIVATE + Checks if the argument is a proper table and comes from the current backend. + It returns True or throws a dataflow error explaining the issue. + + TODO [RW]: this should check that the tables are on the same connection +check_db_table arg_name table = + if Table_Helpers.is_table table . not then Error.throw (Type_Error.Error Table table arg_name) else + same_backend = table.is_a Table + case same_backend of + False -> + Error.throw (Illegal_Argument.Error "Currently cross-backend operations are not supported. Materialize the table using `.read` before mixing it with an in-memory Table.") + True -> True diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Errors.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Errors.enso index 8e0576202f98..68095b259cb8 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Errors.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Errors.enso @@ -5,25 +5,23 @@ polyglot java import java.sql.SQLException ## Indicates that a requested operation is not supported, for example because a particular database backend does not support it. type Unsupported_Database_Operation + ## PRIVATE Error message - ## UNSTABLE - + ## PRIVATE Convert the SQL error to a textual representation. to_text : Text to_text self = "Unsupported database operation: " + self.message - ## UNSTABLE - + ## PRIVATE Pretty print the error. to_display_text : Text to_display_text self = "Unsupported database operation: " + self.message type SQL_Error - ## UNSTABLE - + ## PRIVATE Indicates an error with executing a query, update or connecting to the database. @@ -33,29 +31,26 @@ type SQL_Error error is related to. Error java_exception related_query=Nothing - ## UNSTABLE - + ## PRIVATE Convert the SQL error to a textual representation. to_text : Text to_text self = query = if self.related_query.is_nothing.not then " [Query was: " + self.related_query + "]" else "" "There was an SQL error: " + self.java_exception.getMessage.to_text + "." + query - ## UNSTABLE - + ## PRIVATE Pretty print the SQL error. to_display_text : Text to_display_text self = self.to_text ## PRIVATE - Throws an error as if a SQL Exception was thrown. throw_sql_error : Text -> SQL_Error throw_sql_error message = Error.throw (SQL_Error.Error (SQLException.new message)) type SQL_Timeout - ## UNSTABLE + ## PRIVATE Indicates that an operation has timed out. @@ -65,24 +60,20 @@ type SQL_Timeout error is related to. Error java_exception related_query=Nothing - ## UNSTABLE - + ## PRIVATE Convert the timeout error to a textual representation. to_text : Text to_text self = query = if self.related_query.is_nothing.not then " [Query was: " + query + "]" else "" "The SQL connection timed out: " + self.java_exception.getMessage + "." + query - ## UNSTABLE - + ## PRIVATE Pretty print the timeout error. to_display_text : Text to_display_text self = self.to_text -## UNSTABLE - - Signals that a name for a column or table is not supported. +## Signals that a name for a column or table is not supported. Arguments: - text: The name that is not supported. @@ -91,25 +82,23 @@ type SQL_Timeout underscore. This is a temporary limitation simplifying name handling. It will be removed in a future version. type Unsupported_Name + ## PRIVATE Error text ## PRIVATE - Creates a human-readable representation of the unsupported name error. to_text : Text to_text self = "The name " + self.text + " is not currently supported by the Database library." - ## UNSTABLE - + ## PRIVATE Pretty print the unsupported name error. to_display_text : Text to_display_text self = self.to_text type Integrity_Error - ## UNSTABLE - + ## PRIVATE Signalizes that an operation tried using objects coming from different contexts. @@ -122,14 +111,14 @@ type Integrity_Error to_text : Text to_text self = self.object_description + " comes from a different context." - ## UNSTABLE - + ## PRIVATE Pretty print the integrity error. to_display_text : Text to_display_text self = self.to_text type Table_Not_Found - ## Indicates that a table was not found in the database. + ## PRIVATE + Indicates that a table was not found in the database. Arguments: - table_name: The name of the table that was not found. @@ -139,6 +128,8 @@ type Table_Not_Found string. Error (name:Text) (related_query_error:SQL_Error) (treated_as_query:Boolean) + ## PRIVATE + Pretty print the table not found error. to_display_text : Text to_display_text self = case self.treated_as_query of True -> "The name " + self.name + " was treated as a query, but the query failed with the following error: " + self.related_query_error.to_display_text + "; if you want to force to use that as a table name, wrap it in `SQL_Query.Table_Name`." diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Aggregate_Helper.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Aggregate_Helper.enso index 4f1794b2b91c..f94eb2a20650 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Aggregate_Helper.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Aggregate_Helper.enso @@ -5,10 +5,10 @@ import Standard.Table.Data.Aggregate_Column.Aggregate_Column from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all import project.Data.Dialect.Dialect -import project.Data.SQL_Type.SQL_Type import project.Data.Table.Table import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.IR.Internal_Column.Internal_Column +import project.Internal.SQL_Type_Reference.SQL_Type_Reference from project.Errors import Unsupported_Database_Operation @@ -22,7 +22,7 @@ from project.Errors import Unsupported_Database_Operation - infer_return_type: A function that takes 3 arguments (name of the operation, list of input columns and a raw SQL IR Expression) and returns the inferred type for the aggregation. -make_aggregate_column : Aggregate_Column -> Text -> Dialect -> SQL_Expression +make_aggregate_column : Aggregate_Column -> Text -> Dialect -> (Any -> Any -> Any -> SQL_Type_Reference) -> SQL_Expression make_aggregate_column aggregate new_name dialect infer_return_type = is_non_empty_selector v = v.is_nothing.not && v.not_empty simple_aggregate op_kind columns = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index 2740dabe802a..723ba07e48b9 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -169,7 +169,7 @@ base_dialect = fun = name -> [name, make_function name] arith = [["ADD_NUMBER", make_binary_op "+"], ["ADD_TEXT", make_binary_op "||"], bin "-", bin "*", bin "/", bin "%", ["mod", make_function "MOD"], ["^", make_function "POWER"]] - logic = [bin "AND", bin "OR", unary "NOT", ["IIF", make_iif], ["TRUE", make_constant "TRUE"], ["FALSE", make_constant "FALSE"]] + logic = [bin "AND", bin "OR", unary "NOT", ["IIF", make_iif]] eq = lift_binary_op "==" make_equals neq = lift_binary_op "!=" make_not_equals compare = [eq, neq, bin "<", bin ">", bin "<=", bin ">=", ["BETWEEN", make_between]] @@ -179,7 +179,8 @@ base_dialect = text = [is_empty, bin "LIKE", simple_equals_ignore_case, fold_case, make_case_sensitive] nulls = [["IS_NULL", make_right_unary_op "IS NULL"], ["FILL_NULL", make_function "COALESCE"]] contains = [["IS_IN", make_is_in], ["IS_IN_COLUMN", make_is_in_column]] - base_map = Map.from_vector (arith + logic + compare + functions + agg + counts + text + nulls + contains) + types = [simple_cast] + base_map = Map.from_vector (arith + logic + compare + functions + agg + counts + text + nulls + contains + types) Internal_Dialect.Value base_map wrap_in_quotes ## PRIVATE @@ -199,6 +200,10 @@ make_iif arguments = case arguments.length of _ -> Error.throw <| Illegal_State.Error ("Invalid amount of arguments for operation IIF") +## PRIVATE +simple_cast = Base_Generator.lift_binary_op "CAST" a-> b-> + Builder.code "CAST(" ++ a ++ " AS " ++ b ++ ")" + ## PRIVATE make_between : Vector Builder -> Builder make_between arguments = case arguments.length of @@ -249,6 +254,7 @@ generate_expression dialect expr = case expr of SQL_Expression.Column origin name -> dialect.wrap_identifier origin ++ '.' ++ dialect.wrap_identifier name SQL_Expression.Constant value -> Builder.interpolation value + SQL_Expression.Literal value -> Builder.code value SQL_Expression.Operation kind arguments -> op = dialect.operation_map.get kind (Error.throw <| Unsupported_Database_Operation.Error kind) parsed_args = arguments.map (generate_expression dialect) @@ -287,6 +293,14 @@ generate_from_part dialect from_spec = case from_spec of right = generate_from_part dialect right_spec ons = Builder.join " AND " (on.map (generate_expression dialect)) . prefix_if_present " ON " left ++ (" " + kind.to_sql + " ") ++ right ++ ons + From_Spec.Union queries as_name -> + built_queries = queries.map query-> + case query of + Query.Select _ _ -> Nothing + _ -> Panic.throw <| Illegal_State.Error "Only SELECT queries can be used in a UNION. This is a bug in the Database library." + generate_query dialect query + joined = Builder.join " UNION ALL " built_queries + joined.paren ++ alias dialect as_name From_Spec.Sub_Query columns context as_name -> sub = generate_query dialect (Query.Select columns context) sub.paren ++ alias dialect as_name diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Database_Join_Helper.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Database_Join_Helper.enso index ba71048a63c6..bcd871267e45 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Database_Join_Helper.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Common/Database_Join_Helper.enso @@ -15,12 +15,6 @@ import project.Internal.IR.Internal_Column.Internal_Column import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.SQL_Type_Reference.SQL_Type_Reference -## PRIVATE -default_prepare_join connection join_kind new_table_name left_subquery right_subquery on_expressions where_expressions columns_to_select = - new_from = From_Spec.Join join_kind left_subquery right_subquery on_expressions - new_ctx = Context.for_subquery new_from . set_where_filters where_expressions - Table.Value new_table_name connection columns_to_select new_ctx - ## PRIVATE make_join_helpers left_table right_table left_column_mapping right_column_mapping = ## Resolves the column in the original table and finds the expression @@ -94,12 +88,12 @@ prepare_subqueries left right needs_left_indicator needs_right_indicator = renamer = Unique_Name_Strategy.new renamer.mark_used (left.internal_columns.map .name) # This is an operation, not a constant to avoid adding unnecessary interpolations to the query. - [Internal_Column.Value (renamer.make_unique "left_indicator") SQL_Type_Reference.null (SQL_Expression.Operation "TRUE" [])] + [Internal_Column.Value (renamer.make_unique "left_indicator") SQL_Type_Reference.null (SQL_Expression.Literal "TRUE")] right_indicators = if needs_right_indicator.not then [] else renamer = Unique_Name_Strategy.new renamer.mark_used (right.internal_columns.map .name) - [Internal_Column.Value (renamer.make_unique "right_indicator") SQL_Type_Reference.null (SQL_Expression.Operation "TRUE" [])] + [Internal_Column.Value (renamer.make_unique "right_indicator") SQL_Type_Reference.null (SQL_Expression.Literal "TRUE")] # Create subqueries that encapsulate the original queries and provide needed columns. # The generated new sets of columns refer to the encapsulated expressions within the subquery and are diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/From_Spec.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/From_Spec.enso index fc2cae33f505..3567d9395ffa 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/From_Spec.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/From_Spec.enso @@ -1,6 +1,7 @@ from Standard.Base import all import project.Internal.IR.Context.Context +import project.Internal.IR.Query.Query import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.IR.SQL_Join_Kind.SQL_Join_Kind @@ -46,6 +47,22 @@ type From_Spec sources. Join (kind : SQL_Join_Kind) (left_spec : From_Spec) (right_spec : From_Spec) (on : Vector SQL_Expression) + ## PRIVATE + + A query source that performs a union operation on multiple sources. + + This maps to the SQL operation `UNION ALL`, keeping any duplicate rows. + + Arguments: + - queries: the list of queries to be unioned. Eachn query shold have the + same number of columns, as these will be merged by position. Ideally, + corresponding columns should have the same names too, as the outer + query will be referring to columns of the union by names of the columns + from the first query. + - alias: the name for the consolidated query, to be used by column + references, referring to columns of the union. + Union (queries : Vector Query) (alias : Text) + ## PRIVATE A query source consisting of a sub-query. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso index ea037db99d68..5f155e62a9b3 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/SQL_Expression.enso @@ -30,6 +30,12 @@ type SQL_Expression values depends on the database backend. Constant (value : Any) + ## PRIVATE + + The internal representation of an SQL literal that should be inserted + as-is into a query. + Literal (value : Text) + ## PRIVATE The internal representation of an SQL expression built from an operation diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso index aa9ca84b8a62..ed8226546ca3 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso @@ -92,10 +92,11 @@ type JDBC_Connection ## PRIVATE Given a prepared statement, gets the column names and types for the result set. - fetch_columns : Text | SQL_Statement -> Statement_Setter -> Any - fetch_columns self statement statement_setter = + raw_fetch_columns : Text | SQL_Statement -> Boolean -> Statement_Setter -> Any + raw_fetch_columns self statement execute_query statement_setter = self.with_prepared_statement statement statement_setter stmt-> - metadata = stmt.executeQuery.getMetaData + metadata = if execute_query then stmt.executeQuery.getMetaData else + stmt.getMetaData resolve_column ix = name = metadata.getColumnLabel ix+1 diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 5c98ef140b64..a698436946f6 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -9,6 +9,7 @@ import Standard.Table.Internal.Problem_Builder.Problem_Builder from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all import project.Connection.Connection.Connection +import project.Data.Dialect import project.Data.SQL.Builder import project.Data.SQL_Statement.SQL_Statement import project.Data.SQL_Type.SQL_Type @@ -20,14 +21,15 @@ import project.Internal.Common.Database_Distinct_Helper import project.Internal.Common.Database_Join_Helper import project.Internal.IR.Context.Context import project.Internal.IR.From_Spec.From_Spec -import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.IR.Internal_Column.Internal_Column -import project.Internal.IR.Order_Descriptor.Order_Descriptor import project.Internal.IR.Nulls_Order.Nulls_Order +import project.Internal.IR.Order_Descriptor.Order_Descriptor +import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.IR.SQL_Join_Kind.SQL_Join_Kind import project.Internal.IR.Query.Query import project.Internal.Postgres.Postgres_Type_Mapping.Postgres_Type_Mapping import project.Internal.SQL_Type_Mapping.SQL_Type_Mapping +import project.Internal.SQL_Type_Reference.SQL_Type_Reference import project.Internal.Statement_Setter.Statement_Setter from project.Errors import Unsupported_Database_Operation @@ -78,13 +80,6 @@ type Postgres_Dialect prepare_order_descriptor self internal_column sort_direction text_ordering = make_order_descriptor internal_column sort_direction text_ordering - ## PRIVATE - Prepares a join operation, returning a new table instance encapsulating a - proper query. - prepare_join : Connection -> SQL_Join_Kind -> Text -> From_Spec -> From_Spec -> Vector -> Vector -> Vector -> Table - prepare_join self connection join_kind new_table_name left_subquery right_subquery on_expressions where_expressions columns_to_select = - Database_Join_Helper.default_prepare_join connection join_kind new_table_name left_subquery right_subquery on_expressions where_expressions columns_to_select - ## PRIVATE Prepares a distinct operation. prepare_distinct : Table -> Vector -> Case_Sensitivity -> Problem_Builder -> Table @@ -132,6 +127,35 @@ type Postgres_Dialect get_statement_setter : Statement_Setter get_statement_setter self = postgres_statement_setter + ## PRIVATE + make_cast : Internal_Column -> SQL_Type -> (SQL_Expression -> SQL_Type_Reference) -> Internal_Column + make_cast self column target_type infer_result_type_from_database_callback = + mapping = self.get_type_mapping + source_type = mapping.sql_type_to_value_type column.sql_type_reference.get + target_value_type = mapping.sql_type_to_value_type target_type + # Boolean to Numeric casts need special handling: + transformed_expression = case source_type.is_boolean && target_value_type.is_numeric of + True -> + SQL_Expression.Operation "IIF" [column.expression, SQL_Expression.Literal "1", SQL_Expression.Literal "0"] + False -> column.expression + target_type_sql_text = mapping.sql_type_to_text target_type + new_expression = SQL_Expression.Operation "CAST" [transformed_expression, SQL_Expression.Literal target_type_sql_text] + new_sql_type_reference = infer_result_type_from_database_callback new_expression + Internal_Column.Value column.name new_sql_type_reference new_expression + + ## PRIVATE + needs_execute_query_for_type_inference : Boolean + needs_execute_query_for_type_inference self = False + + ## PRIVATE + cast_after_union : Boolean + cast_after_union self = False + + ## PRIVATE + prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement + prepare_fetch_types_query self expression context = + Dialect.default_fetch_types_query self expression context + ## PRIVATE check_aggregate_support : Aggregate_Column -> Boolean ! Unsupported_Database_Operation check_aggregate_support self aggregate = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso index d8b2de8c4c27..0fd8d1f66205 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso @@ -7,6 +7,7 @@ from Standard.Table.Errors import Inexact_Type_Coercion import project.Data.SQL_Type.SQL_Type import project.Internal.IR.SQL_Expression.SQL_Expression +import project.Internal.SQL_Type_Mapping import project.Internal.SQL_Type_Reference.SQL_Type_Reference polyglot java import java.sql.Types @@ -48,9 +49,7 @@ type Postgres_Type_Mapping type_name = if with_timezone then "timestamptz" else "timestamp" SQL_Type.Value Types.TIMESTAMP type_name Value_Type.Binary _ _ -> - # This is the maximum size that JDBC driver reports for Postgres. - max_int4 = 2147483647 - SQL_Type.Value Types.BINARY "bytea" precision=max_int4 + SQL_Type.Value Types.BINARY "bytea" precision=max_precision Value_Type.Mixed -> Error.throw (Illegal_Argument.Error "Postgres tables do not support Mixed types.") Value_Type.Unsupported_Data_Type type_name underlying_type -> @@ -74,6 +73,19 @@ type Postgres_Type_Mapping Nothing -> on_unknown_type sql_type builder -> builder sql_type + ## PRIVATE + sql_type_to_text : SQL_Type -> Text + sql_type_to_text sql_type = + if sql_type.name == "bool" then "bool" else + variable_length_types = [Types.VARCHAR, Types.BINARY] + ## If the type is variable length and the maximum is provided, we treat + it as unbounded, otherwise too big max length may be not accepted by + Postgres. + skip_precision = (variable_length_types.contains sql_type.typeid) && (sql_type.precision == max_precision) + case skip_precision of + True -> sql_type.name + False -> SQL_Type_Mapping.default_sql_type_to_text sql_type + ## PRIVATE The Postgres type mapping always relies on the return type determined by the database backend. @@ -95,7 +107,6 @@ type Postgres_Type_Mapping simple_types_map = Map.from_vector <| ints = [[Types.SMALLINT, Value_Type.Integer Bits.Bits_16], [Types.BIGINT, Value_Type.Integer Bits.Bits_64], [Types.INTEGER, Value_Type.Integer Bits.Bits_32]] floats = [[Types.DOUBLE, Value_Type.Float Bits.Bits_64], [Types.REAL, Value_Type.Float Bits.Bits_32]] - # TODO Bit1, Date_Time other = [[Types.DATE, Value_Type.Date], [Types.TIME, Value_Type.Time]] ints + floats + other @@ -127,3 +138,8 @@ complex_types_map = Map.from_vector <| ## PRIVATE on_unknown_type sql_type = Value_Type.Unsupported_Data_Type sql_type.name sql_type + +## PRIVATE + This is the maximum size that JDBC driver reports for 'unbounded' types in + Postgres. +max_precision = 2147483647 diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Redshift/Redshift_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Redshift/Redshift_Dialect.enso index ae3519574b72..fa5521dc5cd2 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Redshift/Redshift_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Redshift/Redshift_Dialect.enso @@ -4,14 +4,17 @@ import Standard.Table.Internal.Naming_Helpers.Naming_Helpers from Standard.Table import Aggregate_Column import project.Connection.Connection.Connection +import project.Data.Dialect import project.Data.SQL_Statement.SQL_Statement import project.Data.SQL_Type.SQL_Type import project.Data.Table.Table import project.Internal.Base_Generator import project.Internal.Column_Fetcher.Column_Fetcher import project.Internal.Column_Fetcher as Column_Fetcher_Module +import project.Internal.IR.Context.Context import project.Internal.IR.From_Spec.From_Spec import project.Internal.IR.Internal_Column.Internal_Column +import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.IR.SQL_Join_Kind.SQL_Join_Kind import project.Internal.IR.Order_Descriptor.Order_Descriptor import project.Internal.IR.Query.Query @@ -19,6 +22,7 @@ import project.Internal.Postgres.Postgres_Dialect import project.Internal.Common.Database_Join_Helper import project.Internal.Postgres.Postgres_Type_Mapping.Postgres_Type_Mapping import project.Internal.SQL_Type_Mapping.SQL_Type_Mapping +import project.Internal.SQL_Type_Reference.SQL_Type_Reference import project.Internal.Statement_Setter.Statement_Setter from project.Errors import Unsupported_Database_Operation @@ -69,13 +73,6 @@ type Redshift_Dialect prepare_order_descriptor self internal_column sort_direction text_ordering = Postgres_Dialect.make_order_descriptor internal_column sort_direction text_ordering - ## PRIVATE - Prepares a join operation, returning a new table instance encapsulating a - proper query. - prepare_join : Connection -> SQL_Join_Kind -> Text -> From_Spec -> From_Spec -> Vector -> Vector -> Vector -> Table - prepare_join self connection join_kind new_table_name left_subquery right_subquery on_expressions where_expressions columns_to_select = - Database_Join_Helper.default_prepare_join connection join_kind new_table_name left_subquery right_subquery on_expressions where_expressions columns_to_select - ## PRIVATE A heuristic used by `Connection.query` to determine if a given text looks like a SQL query for the given dialect or is rather a table name. @@ -108,6 +105,28 @@ type Redshift_Dialect get_statement_setter : Statement_Setter get_statement_setter self = Postgres_Dialect.postgres_statement_setter + ## PRIVATE + make_cast : Internal_Column -> SQL_Type -> (SQL_Expression -> SQL_Type_Reference) -> Internal_Column + make_cast self column target_type infer_result_type_from_database_callback = + mapping = self.get_type_mapping + sql_type_text = mapping.sql_type_to_text target_type + new_expression = SQL_Expression.Operation "CAST" [column.expression, SQL_Expression.Literal sql_type_text] + new_sql_type_reference = infer_result_type_from_database_callback new_expression + Internal_Column.Value column.name new_sql_type_reference new_expression + + ## PRIVATE + needs_execute_query_for_type_inference : Boolean + needs_execute_query_for_type_inference self = False + + ## PRIVATE + cast_after_union : Boolean + cast_after_union self = False + + ## PRIVATE + prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement + prepare_fetch_types_query self expression context = + Dialect.default_fetch_types_query self expression context + ## PRIVATE check_aggregate_support : Aggregate_Column -> Boolean ! Unsupported_Database_Operation check_aggregate_support self aggregate = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Mapping.enso index 24f740d9c429..560c1a63cd2f 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Mapping.enso @@ -2,13 +2,16 @@ from Standard.Base import all import Standard.Base.Errors.Unimplemented.Unimplemented import Standard.Table.Data.Type.Value_Type.Value_Type +from Standard.Table.Errors import Inexact_Type_Coercion import project.Data.SQL_Type.SQL_Type import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.SQL_Type_Reference.SQL_Type_Reference type SQL_Type_Mapping - ## Converts the given Value_Type to its corresponding SQL_Type. + ## PRIVATE + ADVANCED + Converts the given Value_Type to its corresponding SQL_Type. Some SQL dialects may not support all Value_Types (in fact most will have at least a few exceptions, and some like SQLite may have very @@ -19,18 +22,30 @@ type SQL_Type_Mapping If the conversion is exact, it should be reversible, i.e. `sql_type_to_value_type (value_type_to_sql x Problem_Behavior.Report_Error) = x`. - value_type_to_sql : Value_Type -> Problem_Behavior -> SQL_Type + value_type_to_sql : Value_Type -> Problem_Behavior -> SQL_Type ! Inexact_Type_Coercion value_type_to_sql value_type on_problems = _ = [value_type, on_problems] Unimplemented.throw "This is an interface only." - ## Converts the given SQL_Type to its corresponding Value_Type. + ## PRIVATE + ADVANCED + Converts the given SQL_Type to its corresponding Value_Type. sql_type_to_value_type : SQL_Type -> Value_Type sql_type_to_value_type sql_type = _ = sql_type Unimplemented.throw "This is an interface only." - ## Returns a `SQL_Type_Reference` that will resolve to the resulting type of + ## Converts an SQL_Type to a Text representation compatible with the related + SQL dialect that can be used in SQL expressions like CAST or column + definitions. + sql_type_to_text : SQL_Type -> Text + sql_type_to_text sql_type = + _ = sql_type + Unimplemented.throw "This is an interface only." + + ## PRIVATE + ADVANCED + Returns a `SQL_Type_Reference` that will resolve to the resulting type of the given operation. In most cases this will just delegate to `infer_from_database_callback` @@ -70,3 +85,10 @@ type SQL_Type_Mapping prepare_type_overrides column_type_suggestions = _ = column_type_suggestions Unimplemented.throw "This is an interface only." + +## PRIVATE +default_sql_type_to_text sql_type = + suffix = if sql_type.precision.is_nothing then "" else + if sql_type.scale.is_nothing then "(" + sql_type.precision.to_text + ")" else + " (" + sql_type.precision.to_text + "," + sql_type.scale.to_text + ")" + sql_type.name.trim + suffix diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Reference.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Reference.enso index 11e02d9b82ac..7aa5d8a87905 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Reference.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Reference.enso @@ -1,6 +1,5 @@ from Standard.Base import all import Standard.Base.Errors.Illegal_State.Illegal_State -import Standard.Base.Runtime.Lazy.Lazy import project.Connection.Connection.Connection import project.Data.SQL_Type.SQL_Type @@ -14,7 +13,7 @@ type SQL_Type_Reference Since fetching this type requires querying the database, it is computed lazily and cached. - Computed_By_Database (lazy_ref : Lazy) + Computed_By_Database (~lazy_ref : SQL_Type) ## Refers to an SQL type that is overridden by the dialect's type system. Overridden (value : SQL_Type) @@ -25,7 +24,7 @@ type SQL_Type_Reference This may perform a database query on first access. get : SQL_Type get self = case self of - SQL_Type_Reference.Computed_By_Database lazy_ref -> lazy_ref.get + SQL_Type_Reference.Computed_By_Database lazy_ref -> lazy_ref SQL_Type_Reference.Overridden value -> value ## PRIVATE @@ -45,13 +44,12 @@ type SQL_Type_Reference new : Connection -> Context -> SQL_Expression -> SQL_Type_Reference new connection context expression = do_fetch = - empty_context = context.add_where_filters [SQL_Expression.Constant False] - statement = connection.dialect.generate_sql (Query.Select [["typed_column", expression]] empty_context) + statement = connection.dialect.prepare_fetch_types_query expression context statement_setter = connection.dialect.get_statement_setter - columns = connection.jdbc_connection.fetch_columns statement statement_setter + columns = connection.fetch_columns statement statement_setter only_column = columns.first only_column.second - SQL_Type_Reference.Computed_By_Database (Lazy.new do_fetch) + SQL_Type_Reference.Computed_By_Database do_fetch ## PRIVATE Creates a new `SQL_Type_Reference` that should never be used. @@ -61,7 +59,7 @@ type SQL_Type_Reference null = getter = Error.throw (Illegal_State.Error "Getting the SQL_Type from SQL_Type_Reference.null is not allowed. This indicates a bug in the Database library.") - SQL_Type_Reference.Computed_By_Database (Lazy.new getter) + SQL_Type_Reference.Computed_By_Database getter ## PRIVATE Turns this reference into a type override. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index afd86463409e..41dd8548206b 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -18,12 +18,14 @@ import project.Internal.Column_Fetcher as Column_Fetcher_Module import project.Internal.IR.Context.Context import project.Internal.IR.From_Spec.From_Spec import project.Internal.IR.Internal_Column.Internal_Column -import project.Internal.IR.SQL_Join_Kind.SQL_Join_Kind import project.Internal.IR.Order_Descriptor.Order_Descriptor import project.Internal.IR.Query.Query +import project.Internal.IR.SQL_Expression.SQL_Expression +import project.Internal.IR.SQL_Join_Kind.SQL_Join_Kind import project.Internal.Common.Database_Distinct_Helper import project.Internal.Common.Database_Join_Helper import project.Internal.SQL_Type_Mapping.SQL_Type_Mapping +import project.Internal.SQL_Type_Reference.SQL_Type_Reference import project.Internal.SQLite.SQLite_Type_Mapping.SQLite_Type_Mapping import project.Internal.Statement_Setter.Statement_Setter from project.Errors import Unsupported_Database_Operation @@ -88,22 +90,6 @@ type SQLite_Dialect True -> Order_Descriptor.Value internal_column.expression sort_direction collation="NOCASE" - ## PRIVATE - Prepares a join operation, returning a new table instance encapsulating a - proper query. - prepare_join : Connection -> SQL_Join_Kind -> Text -> From_Spec -> From_Spec -> Vector -> Vector -> Vector -> Table - prepare_join self connection join_kind new_table_name left_subquery right_subquery on_expressions where_expressions columns_to_select = case join_kind of - SQL_Join_Kind.Right -> - # We just do a left join with swapped order of sub-queries, while keeping the original order of columns. - Database_Join_Helper.default_prepare_join connection SQL_Join_Kind.Left new_table_name right_subquery left_subquery on_expressions where_expressions columns_to_select - SQL_Join_Kind.Full -> - ## TODO workaround for full joins by left outer + right-anti union - https://www.pivotaltracker.com/story/show/184090548 - Error.throw (Unsupported_Database_Operation.Error "Full outer joins are not YET supported by the SQLite backend. You may need to materialize the Table to perform this operation.") - _ -> - # Other kinds of joins just fall back to the default logic. - Database_Join_Helper.default_prepare_join connection join_kind new_table_name left_subquery right_subquery on_expressions where_expressions columns_to_select - ## PRIVATE Prepares a distinct operation. prepare_distinct : Table -> Vector -> Case_Sensitivity -> Problem_Builder -> Table @@ -151,6 +137,31 @@ type SQLite_Dialect get_statement_setter : Statement_Setter get_statement_setter self = Statement_Setter.default + ## PRIVATE + make_cast : Internal_Column -> SQL_Type -> (SQL_Expression -> SQL_Type_Reference) -> Internal_Column + make_cast self column target_type infer_result_type_from_database_callback = + _ = infer_result_type_from_database_callback + mapping = self.get_type_mapping + sql_type_text = mapping.sql_type_to_text target_type + new_expression = SQL_Expression.Operation "CAST" [column.expression, SQL_Expression.Literal sql_type_text] + # We override the type here, because SQLite gets it wrong if the column starts with NULL values. + new_sql_type_reference = SQL_Type_Reference.from_constant target_type + Internal_Column.Value column.name new_sql_type_reference new_expression + + ## PRIVATE + needs_execute_query_for_type_inference : Boolean + needs_execute_query_for_type_inference self = True + + ## PRIVATE + cast_after_union : Boolean + cast_after_union self = True + + ## PRIVATE + prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement + prepare_fetch_types_query self expression context = + minimized_context = context.set_limit 1 + self.generate_sql (Query.Select [["typed_column", expression]] minimized_context) + ## PRIVATE check_aggregate_support : Aggregate_Column -> Boolean ! Unsupported_Database_Operation check_aggregate_support self aggregate = case aggregate of diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso index 2f01d584096a..797cd709a1b5 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso @@ -5,12 +5,14 @@ import Standard.Base.Errors.Illegal_State.Illegal_State import Standard.Table.Data.Type.Enso_Types import Standard.Table.Data.Type.Value_Type.Value_Type import Standard.Table.Data.Type.Value_Type.Bits +import Standard.Table.Data.Type.Value_Type_Helpers from Standard.Table.Errors import Inexact_Type_Coercion import project.Data.Column.Column import project.Data.SQL_Type.SQL_Type import project.Internal.IR.Internal_Column.Internal_Column import project.Internal.IR.SQL_Expression.SQL_Expression +import project.Internal.SQL_Type_Mapping import project.Internal.SQL_Type_Reference.SQL_Type_Reference polyglot java import java.sql.Types @@ -77,6 +79,10 @@ type SQLite_Type_Mapping Value_Type.Unsupported_Data_Type sql_type.name sql_type simple_types_map.get sql_type.typeid on_not_found + ## PRIVATE + sql_type_to_text : SQL_Type -> Text + sql_type_to_text sql_type = SQL_Type_Mapping.default_sql_type_to_text sql_type + ## PRIVATE The SQLite type mapping takes special measures to keep boolean columns boolean even if the Database will say that they are numeric. @@ -113,7 +119,11 @@ type SQLite_Type_Mapping Panic.throw (Illegal_State.Error "Impossible: IIF must have 3 arguments. This is a bug in the Database library.") inputs_types = arguments.drop 1 . map find_type if inputs_types.first == inputs_types.second then return inputs_types.first else - infer_default_type + case Value_Type_Helpers.reconcile_types inputs_types.first inputs_types.second of + ## Inference failed, fall back to default type. + Ideally, should never happen. To be handled in #6106. + Value_Type.Mixed -> infer_default_type + common -> return common always_boolean_ops = ["==", "!=", "equals_ignore_case", ">=", "<=", "<", ">", "BETWEEN", "AND", "OR", "NOT", "IS_NULL", "IS_NAN", "IS_EMPTY", "LIKE", "IS_IN", "starts_with", "ends_with", "contains"] always_text_ops = ["ADD_TEXT", "CONCAT", "CONCAT_QUOTE_IF_NEEDED"] @@ -136,7 +146,8 @@ type SQLite_Type_Mapping Nothing -> Nothing _ : Vector -> column_type_suggestions.map .get -## The types that SQLite JDBC driver will report are: BOOLEAN, TINYINT, +## PRIVATE + The types that SQLite JDBC driver will report are: BOOLEAN, TINYINT, SMALLINT, BIGINT, INTEGER, DECIMAL, DOUBLE, REAL, FLOAT, NUMERIC, DATE, TIMESTAMP, CHAR, VARCHAR, BINARY, BLOB, CLOB. @@ -155,6 +166,7 @@ simple_types_map = Map.from_vector <| special_types = [[Types.BOOLEAN, Value_Type.Boolean]] ints + floats + numerics + strings + blobs + special_types +## PRIVATE type SQLite_Types ## PRIVATE text = SQL_Type.Value Types.VARCHAR "TEXT" diff --git a/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Histogram.enso b/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Histogram.enso index 4fa74d34edfd..900f4a509d00 100644 --- a/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Histogram.enso +++ b/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Histogram.enso @@ -1,10 +1,9 @@ from Standard.Base import all -## UNSTABLE +## PRIVATE type Histogram ## PRIVATE - The histogram of a single image channel. Arguments: @@ -12,8 +11,7 @@ type Histogram - data: The histogram data. Value channel data - ## UNSTABLE - + ## PRIVATE Convert histogram data to a JS_Object. > Example diff --git a/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Image.enso b/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Image.enso index 6887daa51a49..6ac86b49e2fb 100644 --- a/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Image.enso +++ b/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Image.enso @@ -108,7 +108,7 @@ type Image Panic.catch JException (Java_Codecs.write path self.opencv_mat int_flags) _-> Error.throw (File_Error.IO_Error (File.new path) 'Failed to write to the file') - ## UNSTABLE + ## PRIVATE The image data type. @@ -120,9 +120,7 @@ type Image Pixel values are normalized in a range [0.0 .. 1.0]. Value opencv_mat - ## UNSTABLE - - Return the number of image rows. + ## Return the number of image rows. > Example Get the number of rows from an image. @@ -133,9 +131,7 @@ type Image rows : Integer rows self = self.opencv_mat.rows - ## UNSTABLE - - Return the number of image columns. + ## Return the number of image columns. > Example Get the number of columns from an image. @@ -146,9 +142,7 @@ type Image columns : Integer columns self = self.opencv_mat.cols - ## UNSTABLE - - Return the number of image channels. + ## Return the number of image channels. > Example Get the number of channels from an image. @@ -159,9 +153,7 @@ type Image channels : Integer channels self = self.opencv_mat.channels - ## UNSTABLE - - Get the pixel value indexed by row and column. + ## Get the pixel value indexed by row and column. Arguments: - row: the row index. @@ -180,9 +172,7 @@ type Image arr = Java_Image.get self.opencv_mat row column Vector.from_polyglot_array arr - ## UNSTABLE - - Calculates the per-element sum of an image and a scalar or a matrix. + ## Calculates the per-element sum of an image and a scalar or a matrix. Arguments: - value: A value can be a number, a vector of numbers, or a matrix. The @@ -231,9 +221,7 @@ type Image + : (Number | Vector | Matrix) -> Image ! Matrix_Error + self value = Panic.recover Any (core_op self.opencv_mat value (Java_Image.add _ _ _)) . catch Any core_op_handler - ## UNSTABLE - - Calculates the per-element difference between an image and a scalar or a matrix. + ## Calculates the per-element difference between an image and a scalar or a matrix. Arguments: - value: A value can be a number, a vector of numbers, or a matrix. The @@ -282,9 +270,7 @@ type Image - : (Number | Vector | Matrix) -> Image ! Matrix_Error - self value = Panic.recover Any (core_op self.opencv_mat value (Java_Image.subtract _ _ _)) . catch Any core_op_handler - ## UNSTABLE - - Calculates the per-element product of an image and a scalar or a matrix. + ## Calculates the per-element product of an image and a scalar or a matrix. Arguments: - value: A value can be a number, a vector of numbers, or a matrix. The @@ -333,9 +319,7 @@ type Image * : (Number | Vector | Matrix) -> Image ! Matrix_Error * self value = Panic.recover Any (core_op self.opencv_mat value (Java_Image.multiply _ _ _)) . catch Any core_op_handler - ## UNSTABLE - - Performs per-element division of an image and a scalar or a matrix. + ## Performs per-element division of an image and a scalar or a matrix. Arguments: - value: A value can be a number, a vector of numbers, or a matrix. The @@ -384,9 +368,7 @@ type Image / : (Number | Vector | Matrix) -> Image ! Matrix_Error / self value = Panic.recover Any (core_op self.opencv_mat value (Java_Image.divide _ _ _)) . catch Any core_op_handler - ## UNSTABLE - - Convert the image to a vector. + ## Convert the image to a vector. > Example Convert an image to a vector. @@ -399,8 +381,7 @@ type Image arr = Java_Image.to_vector self.opencv_mat Vector.from_polyglot_array arr - ## UNSTABLE - + ## PRIVATE Convert the image to a JS_Object. > Example @@ -414,9 +395,7 @@ type Image base64 = Java_Image.to_base64 self.opencv_mat JS_Object.from_pairs [["mediaType", "image/png"], ["base64", base64]] - ## UNSTABLE - - Convert the image to a Matrix. + ## Convert the image to a Matrix. > Example Convert an image to a matrix. @@ -427,9 +406,7 @@ type Image to_matrix : Matrix to_matrix self = Matrix.from_vector self.to_vector self.rows self.channels - ## UNSTABLE - - Create a histogram for the specified channel of the image. + ## Create a histogram for the specified channel of the image. Arguments: - channel: the channel number. @@ -480,10 +457,14 @@ core_op_handler error = ## PRIVATE type Image_Comparator + ## PRIVATE + compare : Image -> Image -> Ordering compare x y = if Java_Image.is_equals x.opencv_mat y.opencv_mat then Ordering.Equal else Nothing + ## PRIVATE + hash : Image -> Integer hash x = x.opencv_mat.hashCode Comparable.from (_:Image) = Image_Comparator diff --git a/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Matrix.enso b/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Matrix.enso index 67085cb8d39f..ec137273c55f 100644 --- a/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Matrix.enso +++ b/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Matrix.enso @@ -11,7 +11,6 @@ polyglot java import org.opencv.core.Scalar ## UNSTABLE type Matrix - ## PRIVATE The matrix data type. @@ -99,9 +98,7 @@ type Matrix from_vector values rows=1 channels=1 = Matrix.Value (Java_Matrix.from_vector values.to_array channels rows) - ## UNSTABLE - - Return the number of matrix rows. + ## Return the number of matrix rows. > Example Get the number of rows in this matrix. @@ -112,9 +109,7 @@ type Matrix rows : Integer rows self = self.opencv_mat.rows - ## UNSTABLE - - Return the number of matrix columns. + ## Return the number of matrix columns. > Example Get the number of columns in this matrix. @@ -136,9 +131,7 @@ type Matrix channels : Integer channels self = self.opencv_mat.channels - ## UNSTABLE - - Get the matrix value at the specified row and column. + ## Get the matrix value at the specified row and column. Arguments: - row: the row index. @@ -157,9 +150,7 @@ type Matrix arr = Java_Matrix.get self.opencv_mat row column Vector.from_polyglot_array arr - ## UNSTABLE - - Reshape the matrix specifying new number of rows and channels. + ## Reshape the matrix specifying new number of rows and channels. Arguments: - rows: the new number of rows. @@ -177,9 +168,7 @@ type Matrix Nothing -> Matrix.Value (self.opencv_mat.reshape self.channels rows) _ -> Matrix.Value (self.opencv_mat.reshape channels rows) - ## UNSTABLE - - Calculates the per-element sum of two matrices or a matrix and a scalar. + ## Calculates the per-element sum of two matrices or a matrix and a scalar. Arguments: - value: A value can be a number, a vector of numbers, or a matrix. The @@ -215,9 +204,7 @@ type Matrix + : (Number | Vector | Matrix) -> Matrix ! Matrix_Error + self value = Panic.recover Any (core_op self.opencv_mat value (Java_Matrix.add _ _ _)) . catch core_op_handler - ## UNSTABLE - - Calculates the per-element difference of two matrices or of a matrix and + ## Calculates the per-element difference of two matrices or of a matrix and a scalar. Arguments: @@ -254,9 +241,7 @@ type Matrix - : (Number | Vector | Matrix) -> Matrix ! Matrix_Error - self value = Panic.recover Any (core_op self.opencv_mat value (Java_Matrix.subtract _ _ _)) . catch core_op_handler - ## UNSTABLE - - Calculates the per-element product of two matrices or a matrix and a + ## Calculates the per-element product of two matrices or a matrix and a scalar. Arguments: @@ -298,9 +283,7 @@ type Matrix * : (Number | Vector | Matrix) -> Matrix ! Matrix_Error * self value = Panic.recover Any (core_op self.opencv_mat value (Java_Matrix.multiply _ _ _)) . catch core_op_handler - ## UNSTABLE - - Performs per-element division of two matrices or a matrix and a scalar. + ## Performs per-element division of two matrices or a matrix and a scalar. Arguments: - value: A value can be a number, a vector of numbers, or a matrix. The @@ -336,9 +319,7 @@ type Matrix / : (Number | Vector | Matrix) -> Matrix ! Matrix_Error / self value = Panic.recover Any (core_op self.opencv_mat value (Java_Matrix.divide _ _ _)) . catch core_op_handler - ## UNSTABLE - - Normalize the matrix into a range of [min_value .. max_value] so that the + ## Normalize the matrix into a range of [min_value .. max_value] so that the minimum value of the matrix becomes `min_value` and the maximum value of the matrix becomes `max_value`. @@ -370,9 +351,7 @@ type Matrix normalize self min_value=0.0 max_value=1.0 = Matrix.Value (Java_Matrix.normalize self.opencv_mat min_value max_value) - ## UNSTABLE - - Convert this matrix to an image. + ## Convert this matrix to an image. > Example Convert a matrix to an image. @@ -383,9 +362,7 @@ type Matrix to_image : Image to_image self = Image (Image.from_vector self.normalize.to_vector self.rows self.channels) - ## UNSTABLE - - Get the elements of this matrix as a vector. + ## Get the elements of this matrix as a vector. > Example Convert a matrix to a vector. @@ -398,8 +375,7 @@ type Matrix arr = Java_Matrix.to_vector self.opencv_mat Vector.from_polyglot_array arr - ## UNSTABLE - + ## PRIVATE Convert this matrix to a JS_Object. > Example @@ -412,7 +388,6 @@ type Matrix to_js_object self = self.opencv_mat.to_text ## PRIVATE - Apply a core matrix operation. Arguments: @@ -433,7 +408,6 @@ core_op mat value function = Matrix.Value result ## PRIVATE - Handles errors in `core_op`. Arguments: @@ -445,10 +419,14 @@ core_op_handler error = ## PRIVATE type Matrix_Comparator + ## PRIVATE + compare : Matrix -> Matrix -> Ordering compare x y = if Java_Matrix.is_equals x.opencv_mat y.opencv_mat then Ordering.Equal else Nothing + ## PRIVATE + hash : Matrix -> Number hash x = x.opencv_mat.hashCode Comparable.from (_:Matrix) = Matrix_Comparator diff --git a/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Matrix_Error.enso b/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Matrix_Error.enso index aee4e1a6577e..b967de0fd6a3 100644 --- a/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Matrix_Error.enso +++ b/distribution/lib/Standard/Image/0.0.0-dev/src/Data/Matrix_Error.enso @@ -1,10 +1,8 @@ from Standard.Base import all -## UNSTABLE +## PRIVATE type Matrix_Error - - ## UNSTABLE - + ## PRIVATE Indicates that a matrix has been accessed with an illegal index. Arguments: @@ -13,14 +11,12 @@ type Matrix_Error - index: The requested index in the matrix. Index_Out_Of_Bounds rows columns index - ## UNSTABLE - + ## PRIVATE An error indicating that an operation has failed due to a mismatch of matrix dimensions. Dimensions_Not_Equal - ## UNSTABLE - + ## PRIVATE Pretty-prints a matrix error to be readable by the users. to_display_text : Text to_display_text self = case self of diff --git a/distribution/lib/Standard/Image/0.0.0-dev/src/Image_File_Format.enso b/distribution/lib/Standard/Image/0.0.0-dev/src/Image_File_Format.enso index 68cf8eecaaf5..d5e80347ff50 100644 --- a/distribution/lib/Standard/Image/0.0.0-dev/src/Image_File_Format.enso +++ b/distribution/lib/Standard/Image/0.0.0-dev/src/Image_File_Format.enso @@ -12,19 +12,22 @@ type Image_File_Format ## File_Format to read Image files For_File - ## If the File_Format supports reading from the file, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the file, return a configured instance. for_file : File -> Image_File_Format | Nothing for_file file = extension = file.extension if supported.contains extension then Image_File_Format.For_File else Nothing - ## If the File_Format supports reading from the web response, return a configured instance. + ## PRIVATE + If the File_Format supports reading from the web response, return a configured instance. for_web : Text -> URI -> Image_File_Format | Nothing for_web _ _ = ## Currently not loading Image files automatically. This should be supported later. Nothing - ## Implements the `File.read` for this `File_Format` + ## PRIVATE + Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any read self file on_problems = _ = [on_problems] diff --git a/distribution/lib/Standard/Table/0.0.0-dev/package.yaml b/distribution/lib/Standard/Table/0.0.0-dev/package.yaml index 3a275b837ee9..383e63698216 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/package.yaml +++ b/distribution/lib/Standard/Table/0.0.0-dev/package.yaml @@ -14,6 +14,7 @@ component-groups: exports: - Standard.Table.Data.Table.Table.new - Standard.Table.Data.Table.Table.from_rows + - Standard.Table.Data.Table.Table.from_objects - Standard.Table.Data.Column.Column.from_vector - Standard.Base.Select: exports: diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso index b65cb9db4d2c..b8b59be0f611 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Column.enso @@ -16,7 +16,6 @@ import project.Internal.Widget_Helpers from project.Data.Table import print_table from project.Data.Type.Value_Type import Value_Type, Auto -from project.Data.Type.Value_Type_Helpers import ensure_valid_parse_target from project.Errors import No_Index_Set_Error, Floating_Point_Equality, Invalid_Value_Type polyglot java import org.enso.table.data.column.operation.map.MapOperationProblemBuilder @@ -50,7 +49,9 @@ type Column Illegal_Argument.handle_java_exception <| Column.Value (Java_Column.new name storage) - ## Creates a new column given a name and a vector of elements repeated over and over. + ## PRIVATE + ADVANCED + Creates a new column given a name and a vector of elements repeated over and over. Arguments: - name: The name of the column to create. @@ -69,7 +70,9 @@ type Column - java_column: The internal representation of the column. Value java_column - ## Returns a text containing an ASCII-art table displaying this data. + ## PRIVATE + ADVANCED + Returns a text containing an ASCII-art table displaying this data. Arguments: - show_rows: the number of initial rows that should be displayed. @@ -98,7 +101,9 @@ type Column missing = '\n\u2026 and ' + (num_rows - display_rows).to_text + ' hidden rows.' table + missing - ## TEXT_ONLY + ## PRIVATE + ADVANCED + TEXT_ONLY Prints an ASCII-art table with this data to the standard output. @@ -912,7 +917,7 @@ type Column Applies only to columns that hold the `Date` or `Date_Time` types. Returns a column of `Integer` type. - year : Column -> Column ! Invalid_Value_Type + year : Column ! Invalid_Value_Type year self = Value_Type.expect_has_date self.value_type related_column=self.name <| simple_unary_op self "year" @@ -921,7 +926,7 @@ type Column Applies only to columns that hold the `Date` or `Date_Time` types. Returns a column of `Integer` type. - month : Column -> Column ! Invalid_Value_Type + month : Column ! Invalid_Value_Type month self = Value_Type.expect_has_date self.value_type related_column=self.name <| simple_unary_op self "month" @@ -930,7 +935,7 @@ type Column Applies only to columns that hold the `Date` or `Date_Time` types. Returns a column of `Integer` type. - day : Column -> Column ! Invalid_Value_Type + day : Column ! Invalid_Value_Type day self = Value_Type.expect_has_date self.value_type related_column=self.name <| simple_unary_op self "day" @@ -1023,15 +1028,16 @@ type Column example_contains = Examples.text_column_1.parse Boolean 'Yes|No' @type Widget_Helpers.parse_type_selector - parse : (Auto|Integer|Decimal|Date|Date_Time|Time_Of_Day|Boolean) -> Text | Data_Formatter -> Problem_Behavior -> Column + parse : Value_Type | Auto -> Text | Data_Formatter -> Problem_Behavior -> Column parse self type=Auto format=Data_Formatter.Value on_problems=Report_Warning = - Value_Type.expect_text self.value_type related_column=self.name <| ensure_valid_parse_target type <| + Value_Type.expect_text self.value_type related_column=self.name <| formatter = case format of _ : Text -> Data_Formatter.Value.with_format type format - _ -> format + _ : Data_Formatter -> format + _ -> Error.throw (Illegal_Argument.Error "Invalid format type. Expected Text or Data_Formatter.") - parser = if type == Auto then formatter.make_auto_parser else formatter.make_datatype_parser type + parser = formatter.make_value_type_parser type storage = self.java_column.getStorage new_storage_and_problems = parser.parseColumn self.name storage @@ -1206,8 +1212,7 @@ type Column storage_type = self.java_column.getStorage.getType Storage.to_value_type storage_type - ## UNSTABLE - + ## PRIVATE Converts this column to JS_Object representation. > Example @@ -1235,6 +1240,13 @@ type Column to_table : Table to_table self = Table.Value self.java_column.toTable + ## Returns a Table describing this column's contents. + + The table behaves like `Table.info` - it lists the column name, the count + of non-null items and the value type. + info : Table + info self = self.to_table.info + ## UNSTABLE Sorts the column according to the specified rules. @@ -1320,9 +1332,7 @@ type Column limit = Math.max (Math.min (end - offset) (length - offset)) 0 Column.Value (self.java_column.slice offset limit) - ## UNSTABLE - - Returns the first element in the column, if it exists. + ## Returns the first element in the column, if it exists. If the column is empty, this method will return a dataflow error containing an `Index_Out_Of_Bounds`. @@ -1336,9 +1346,7 @@ type Column first : Any ! Index_Out_Of_Bounds first self = self.at 0 - ## UNSTABLE - - Returns the last element in the column, if it exists. + ## Returns the last element in the column, if it exists. If the column is empty, this method will return a dataflow error containing an `Index_Out_Of_Bounds`. @@ -1352,9 +1360,7 @@ type Column last : Any ! Index_Out_Of_Bounds last self = self.at (self.length - 1) - ## UNSTABLE - - Returns a column containing the values of `self` column with their order + ## Returns a column containing the values of `self` column with their order reversed. > Example @@ -1368,9 +1374,7 @@ type Column mask = OrderBuilder.buildReversedMask self.length Column.Value (self.java_column.applyMask mask) - ## UNSTABLE - - Returns a column of numbers, in which every entry denotes how many times + ## Returns a column of numbers, in which every entry denotes how many times the value at the given position occured before. > Example @@ -1382,7 +1386,8 @@ type Column duplicate_count : Column duplicate_count self = Column.Value self.java_column.duplicateCount - ## Provides a simplified text representation for display in the REPL and errors. + ## PRIVATE + Provides a simplified text representation for display in the REPL and errors. to_text : Text to_text self = "(In-Memory Column "+self.name.to_text+")" diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Data_Formatter.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Data_Formatter.enso index 1e6f16ab9a49..91b7190ddd3b 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Data_Formatter.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Data_Formatter.enso @@ -2,7 +2,7 @@ from Standard.Base import all import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import project.Internal.Parse_Values_Helper -from project.Data.Type.Value_Type import Value_Type, Auto +from project.Data.Type.Value_Type import Value_Type, Auto, Bits polyglot java import org.enso.table.parsing.IntegerParser polyglot java import org.enso.table.parsing.DecimalParser @@ -61,25 +61,28 @@ type Data_Formatter - false_values: Values representing False. Value trim_values:Boolean=True allow_leading_zeros:Boolean=False decimal_point:Text='.' thousand_separator:Text='' allow_exponential_notation:Boolean=False datetime_formats:(Vector Text)=["ENSO_ZONED_DATE_TIME"] date_formats:(Vector Text)=["ISO_LOCAL_DATE"] time_formats:(Vector Text)=["ISO_LOCAL_TIME"] datetime_locale:Locale=Locale.default true_values:(Vector Text)=["True","true","TRUE"] false_values:(Vector Text)=["False","false","FALSE"] - ## Parse a Text into a value. + ## PRIVATE + ADVANCED + Parse a Text into a value. Arguments: - text: Text value to parse. - - datatype: Text value to parse. + - datatype: The expected Enso type to parse the value into. If set to + `Auto`, the type will be inferred automatically. - on_problems: Specifies the behavior when a problem occurs. By default, a warning is issued, but the operation proceeds. If set to `Report_Error`, the operation fails with a dataflow error. If set to `Ignore`, the operation proceeds without errors or warnings. parse : Text -> (Auto|Integer|Number|Date|Date_Time|Time_Of_Day|Boolean) -> Problem_Behavior -> Any parse self text datatype=Auto on_problems=Problem_Behavior.Report_Warning = - parser = case datatype of - Auto -> self.make_auto_parser - _ -> self.make_datatype_parser datatype + parser = self.make_datatype_parser datatype result = parser.parseIndependentValue text problems = Vector.from_polyglot_array result.problems . map (Parse_Values_Helper.translate_parsing_problem datatype) on_problems.attach_problems_after result.value problems - ## Format a value into a Text. + ## PRIVATE + ADVANCED + Format a value into a Text. Arguments: - value: Value to format. @@ -141,23 +144,25 @@ type Data_Formatter It is mostly a convenience function to easily specify a datatype format. Arguments: - - type: The datatype for which to change the format. The format can be - changed only for Date_Time, Date, Time_Of_Day and Boolean types. + - type: The value type for which to change the format. The format can be + changed only for `Date_Time`, `Date`, `Time` and `Boolean` value types. - format: The new format string to set. For dates, it is the usual date format notation, and for booleans it should be two values that represent true and false, separated by a `|`. - with_format : (Auto|Integer|Number|Date|Date_Time|Time_Of_Day|Boolean) -> Text -> Data_Formatter + with_format : Value_Type | Auto -> Text -> Data_Formatter with_format self type format = case type of - Auto -> Error.throw (Illegal_Argument.Error "Cannot specify a `format` with type `Auto`.") - Integer -> Error.throw (Illegal_Argument.Error "Cannot specify a `format` with type `Integer`.") - Decimal -> Error.throw (Illegal_Argument.Error "Cannot specify a `format` with type `Decimal`.") - Date -> self.with_datetime_formats date_formats=[format] - Date_Time -> self.with_datetime_formats datetime_formats=[format] - Time_Of_Day -> self.with_datetime_formats time_formats=[format] - Boolean -> + Value_Type.Date -> self.with_datetime_formats date_formats=[format] + Value_Type.Time -> self.with_datetime_formats time_formats=[format] + Value_Type.Date_Time _ -> + self.with_datetime_formats datetime_formats=[format] + Value_Type.Boolean -> formats = format.split "|" if formats.length != 2 then Error.throw (Illegal_Argument.Error "The `format` for Booleans must be a string with two values separated by `|`, for example: 'Yes|No'.") else self.with_boolean_values true_values=[formats.at 0] false_values=[formats.at 1] + Auto -> + Error.throw (Illegal_Argument.Error "Cannot specify a `format` with type `Auto`.") + _ : Value_Type -> + Error.throw (Illegal_Argument.Error "Cannot specify a `format` for type `"+type.to_text+"`.") ## PRIVATE Clone the instance with some properties overridden. @@ -212,7 +217,26 @@ type Data_Formatter Date -> self.make_date_parser Date_Time -> self.make_date_time_parser Time_Of_Day -> self.make_time_of_day_parser - _ -> Error.throw (Illegal_Argument.Error "Unsupported datatype: "+datatype.to_text) + Auto -> self.make_auto_parser + _ -> + type_name = case datatype.to_text of + text : Text -> text + _ -> Meta.meta datatype . to_text + Error.throw (Illegal_Argument.Error "Unsupported datatype: "+type_name) + + ## PRIVATE + make_value_type_parser self value_type = case value_type of + # TODO once we implement #5159 we will need to add checks for bounds here and support 16/32-bit ints + Value_Type.Integer Bits.Bits_64 -> self.make_integer_parser + # TODO once we implement #6109 we can support 32-bit floats + Value_Type.Float Bits.Bits_64 -> self.make_decimal_parser + Value_Type.Boolean -> self.make_boolean_parser + Value_Type.Date -> self.make_date_parser + Value_Type.Date_Time True -> self.make_date_time_parser + Value_Type.Time -> self.make_time_of_day_parser + Auto -> self.make_auto_parser + _ -> + Error.throw (Illegal_Argument.Error "Unsupported value type: "+value_type.to_display_text) ## PRIVATE get_specific_type_parsers self = diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Expression.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Expression.enso index 029f9c235ca2..cf231b0e236e 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Expression.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Expression.enso @@ -5,7 +5,9 @@ polyglot java import java.lang.IllegalArgumentException polyglot java import java.lang.UnsupportedOperationException type Expression - ## Evaluates an expression and returns the result + ## PRIVATE + ADVANCED + Evaluates an expression and returns the result Arguments: - expression: the expression to evaluate @@ -29,15 +31,21 @@ type Expression ExpressionVisitorImpl.evaluate expression get_column make_constant module_name type_name var_args_functions.to_array type Expression_Error - ## The expression supplied could not be parsed due to a syntax error. + ## PRIVATE + The expression supplied could not be parsed due to a syntax error. Syntax_Error message:Text line:Integer column:Integer - ## Expression error when a function could not be found on the target type. + ## PRIVATE + Expression error when a function could not be found on the target type. Unsupported_Operation name:Text - ## Expression error when the number of arguments for a function is incorrect. + ## PRIVATE + Expression error when the number of arguments for a function is incorrect. Argument_Mismatch message:Text + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = case self of Expression_Error.Syntax_Error _ _ _ -> "Expression.Syntax_Error: " + self.message + " (line " + self.line.to_text + ", column " + self.column.to_text + ")." diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Row.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Row.enso index 47916939a8dd..aa057e9768bd 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Row.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Row.enso @@ -35,6 +35,7 @@ type Row to_vector : Vector to_vector self = Vector.from_polyglot_array (Array_Proxy.from_proxy_object self) - ## Converts this row into a JS_Object. + ## PRIVATE + Converts this row into a JS_Object. to_js_object : Vector to_js_object self = self.to_vector.to_js_object diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index c9d534a53e17..f489a70e8202 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -39,7 +39,6 @@ import project.Data.Expression.Expression_Error import project.Delimited.Delimited_Format.Delimited_Format from project.Data.Type.Value_Type import Value_Type, Auto -from project.Data.Type.Value_Type_Helpers import ensure_valid_parse_target from project.Internal.Rows_View import Rows_View from project.Errors import all @@ -117,7 +116,9 @@ type Table - java_table: The internal java representation of the table. Value java_table - ## Returns a text containing an ASCII-art table displaying this data. + ## PRIVATE + ADVANCED + Returns a text containing an ASCII-art table displaying this data. Arguments: - show_rows: the number of initial rows that should be displayed. @@ -146,7 +147,9 @@ type Table missing = '\n\u2026 and ' + (num_rows - display_rows).to_text + ' hidden rows.' table + missing - ## Prints an ASCII-art table with this data to the standard output. + ## PRIVATE + ADVANCED + Prints an ASCII-art table with this data to the standard output. Arguments: - show_rows: the number of initial rows that should be displayed. @@ -162,7 +165,8 @@ type Table IO.println (self.display show_rows format_terminal=True) IO.println '' - ## Converts this table into a JS_Object. + ## PRIVATE + Converts this table into a JS_Object. > Example Convert a table to a corresponding JavaScript JS_Object representation. @@ -792,31 +796,31 @@ type Table > Example Parse the first and last columns containing Yes/No values as booleans. - table.parse_values columns=[0, -1] type=Boolean format="Yes|No" + table.parse columns=[0, -1] type=Boolean format="Yes|No" > Example Parse dates in a column in the format `yyyy-MM-dd` (the default format). - table.parse_values "birthday" Date + table.parse "birthday" Date > Example Parse dates in a column in the format `dd/MM/yyyy`. - table.parse_values "birthday" Date 'dd/MM/yyyy' + table.parse "birthday" Date 'dd/MM/yyyy' > Example Parse all columns inferring their types, using `,` as the decimal point for numbers. - table.parse_values format=(Data_Formatter.Value.with_number_formatting decimal_point=',') - parse_values : Text | Integer | Column_Selector | Vector (Text | Integer | Column_Selector) -> (Auto|Integer|Decimal|Date|Date_Time|Time_Of_Day|Boolean) -> Text | Data_Formatter -> Boolean -> Problem_Behavior -> Table - parse_values self columns=(self.columns . filter (c-> c.value_type.is_text) . map .name) type=Auto format=Data_Formatter.Value error_on_missing_columns=True on_problems=Report_Warning = ensure_valid_parse_target type <| + table.parse format=(Data_Formatter.Value.with_number_formatting decimal_point=',') + parse : Text | Integer | Column_Selector | Vector (Text | Integer | Column_Selector) -> Value_Type | Auto -> Text | Data_Formatter -> Boolean -> Problem_Behavior -> Table + parse self columns=(self.columns . filter (c-> c.value_type.is_text) . map .name) type=Auto format=Data_Formatter.Value error_on_missing_columns=True on_problems=Report_Warning = formatter = case format of _ : Text -> Data_Formatter.Value.with_format type format - _ -> format + _ : Data_Formatter -> format + _ -> Error.throw (Illegal_Argument.Error "Invalid format type. Expected Text or Data_Formatter.") - parser = if type == Auto then formatter.make_auto_parser else - formatter.make_datatype_parser type + parser = formatter.make_value_type_parser type select_problem_builder = Problem_Builder.new error_on_missing_columns=error_on_missing_columns selected_columns = self.columns_helper.select_columns_helper columns reorder=True select_problem_builder @@ -1553,16 +1557,14 @@ type Table regardless of types of other columns. Mixing any other types will result in a `No_Common_Type` problem. If columns of incompatible types are meant to be mixed, at least one of them should be explicitly - retyped to the `Mixed` type to indicate that intention. + retyped to the `Mixed` type to indicate that intention. Note that the + `Mixed` type may not be supported by most Database backends. union : (Table | Vector Table) -> Match_Columns -> Boolean | Report_Unmatched -> Boolean -> Problem_Behavior -> Table union self tables match_columns=Match_Columns.By_Name keep_unmatched_columns=Report_Unmatched allow_type_widening=True on_problems=Report_Warning = all_tables = case tables of v : Vector -> [self] + v single_table -> [self, single_table] - ## `is_everything_ok` should actually never be False; it will either be - True or will contain a dataflow error propagating through the if. - is_everything_ok = all_tables.all (check_table "tables") - if is_everything_ok then + all_tables.all (check_table "tables") . if_not_error <| problem_builder = Problem_Builder.new matched_column_sets = Match_Columns_Helpers.match_columns all_tables match_columns keep_unmatched_columns problem_builder result_row_count = all_tables.fold 0 c-> t-> c + t.row_count @@ -1605,8 +1607,8 @@ type Table ## Returns a Table describing this table's contents. - The table lists all columns, counts of non-null items and storage types - of each column. + The table lists all columns, counts of non-null items and value types of + each column. > Example Get information about a table. @@ -1773,9 +1775,7 @@ type Table limit = Math.max (Math.min (end - offset) (length - offset)) 0 Table.Value (self.java_table.slice offset limit) - ## UNSTABLE - - Returns a table containing the rows of `self` table with their order + ## Returns a table containing the rows of `self` table with their order reversed. > Example diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Enso_Types.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Enso_Types.enso index fb29bf76e14c..3a78f8592391 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Enso_Types.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Enso_Types.enso @@ -11,7 +11,7 @@ import project.Data.Type.Value_Type.Bits for integers it will return 64-bit integers even if the value could fit in a smaller one; and for Text values variable-length text will be preferred over fixed-length. -most_specific_value_type : Any -> Value_Type +most_specific_value_type : Any -> Boolean -> Value_Type most_specific_value_type value use_smallest=False = ## TODO implement the `use_smallest` logic _ = use_smallest diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Value_Type.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Value_Type.enso index d73e824a077d..8dce875f3574 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Value_Type.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Value_Type.enso @@ -29,12 +29,19 @@ type Bits 64 -> Bits.Bits_64 _ : Integer -> Error.throw (Illegal_Argument.Error "Invalid number of bits for a float or integer type.") - ## Provides the text representation of the bit-size. + ## PRIVATE + Provides the text representation of the bit-size. to_text : Text to_text self = self.to_bits.to_text + " bits" +## PRIVATE type Bits_Comparator + ## PRIVATE + compare : Bits -> Bits -> Ordering compare x y = Comparable.from x.to_bits . compare x.to_bits y.to_bits + + ## PRIVATE + hash : Bits -> Integer hash x = Comparable.from x.to_bits . hash x.to_bits Comparable.from (_:Bits) = Bits_Comparator @@ -80,7 +87,7 @@ type Value_Type Arguments: - precision: the total number of digits in the number. - scale: the number of digits after the decimal point. - Decimal precision:(Integer|Nothing)=Nothing scale:(Integer|Nothing)=0 + Decimal precision:(Integer|Nothing)=Nothing scale:(Integer|Nothing)=Nothing ## Character string. @@ -179,8 +186,8 @@ type Value_Type Value_Type.Date_Time _ -> True _ -> False - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED Checks if the provided value type is a textual type (with any settings) and runs the following action or reports a type error. expect_text : Value_Type -> Any -> Text -> Any ! Invalid_Value_Type @@ -188,8 +195,8 @@ type Value_Type if Value_Type.is_text value_type then action else Error.throw (Invalid_Value_Type.Error Value_Type.Char value_type related_column) - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED Checks if the provided value type is a boolean type and runs the following action or reports a type error. expect_boolean : Value_Type -> Any -> Any ! Invalid_Value_Type @@ -197,15 +204,16 @@ type Value_Type Value_Type.Boolean -> action _ -> Error.throw (Invalid_Value_Type.Error Value_Type.Boolean value_type) - ## ADVANCED - UNSTABLE + ## PRIVATE + ADVANCED Checks if the provided value type is a `Date` or `Date_Time`. expect_has_date : Value_Type -> Any -> Text -> Any ! Invalid_Value_Type expect_has_date value_type ~action related_column=Nothing = case value_type.has_date of True -> action False -> Error.throw (Invalid_Value_Type.Error "Date or Date_Time" value_type related_column) - ## Provides a text representation of the `Value_Type` meant for + ## PRIVATE + Provides a text representation of the `Value_Type` meant for displaying to the user. to_display_text : Text to_display_text self = case self of diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Value_Type_Helpers.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Value_Type_Helpers.enso index b7f104c99090..f75d4338bc3e 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Value_Type_Helpers.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Type/Value_Type_Helpers.enso @@ -40,20 +40,26 @@ reconcile_types current new = case current of Value_Type.Char current_size current_variable -> case new of Value_Type.Char new_size new_variable -> result_variable = current_variable || new_variable || current_size != new_size - case result_variable of - True -> Value_Type.Char Nothing True - False -> Value_Type.Char current_size False + result_size = max_size current_size new_size + Value_Type.Char result_size result_variable _ -> Value_Type.Mixed Value_Type.Binary current_size current_variable -> case new of Value_Type.Binary new_size new_variable -> result_variable = current_variable || new_variable || current_size != new_size - case result_variable of - True -> Value_Type.Binary Nothing True - False -> Value_Type.Binary current_size False + result_size = max_size current_size new_size + Value_Type.Binary result_size result_variable _ -> Value_Type.Mixed _ -> if current == new then current else Value_Type.Mixed +## PRIVATE + Reconciles two size parameters. If either of them is `Nothing` (meaning + unbounded), returns `Nothing`. If both are bounded, the larger one is + returned. +max_size a b = + if a.is_nothing || b.is_nothing then Nothing else + Math.max a b + ## PRIVATE Finds the most specific value type that will fit all the provided types. @@ -69,11 +75,3 @@ find_common_type types strict = # Double check if Mixed was really allowed to come out. if types.contains Value_Type.Mixed then Value_Type.Mixed else Nothing - -## PRIVATE - Checks if the given type is a valid target type for parsing. - - This will be replaced once we change parse to rely on `Value_Type` instead. -ensure_valid_parse_target type ~action = - expected_types = [Auto, Integer, Decimal, Date, Date_Time, Time_Of_Day, Boolean] - if expected_types.contains type . not then Error.throw (Illegal_Argument.Error "Unsupported target type "+type.to_text+".") else action diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Delimited/Delimited_Format.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Delimited/Delimited_Format.enso index c5849e004272..d2b8799dc71e 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Delimited/Delimited_Format.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Delimited/Delimited_Format.enso @@ -51,7 +51,9 @@ type Delimited_Format defaults to `Nothing` which means that comments are disabled. Delimited (delimiter:Text) (encoding:Encoding=Encoding.utf_8) (skip_rows:Integer=0) (row_limit:Integer|Nothing=Nothing) (quote_style:Quote_Style=Quote_Style.With_Quotes) (headers:Boolean|Infer=Infer) (value_formatter:Data_Formatter|Nothing=Data_Formatter.Value) (keep_invalid_rows:Boolean=True) (line_endings:Line_Ending_Style=Infer) (comment_character:Text|Nothing=Nothing) - ## If the File_Format supports reading from the file, return a configured instance. + ## PRIVATE + ADVANCED + If the File_Format supports reading from the file, return a configured instance. for_file : File -> Delimited_Format | Nothing for_file file = case file.extension of @@ -60,7 +62,9 @@ type Delimited_Format ".tsv" -> Delimited_Format.Delimited '\t' _ -> Nothing - ## If the File_Format supports reading from the web response, return a configured instance. + ## PRIVATE + ADVANCED + If the File_Format supports reading from the web response, return a configured instance. for_web : Text -> URI -> Delimited_Format | Nothing for_web content_type _ = parts = content_type.split ";" . map .trim @@ -75,7 +79,9 @@ type Delimited_Format "text/tab-separated-values" -> Delimited_Format.Delimited '\t' encoding _ -> Nothing - ## Implements the `File.read` for this `File_Format` + ## PRIVATE + ADVANCED + Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any read self file on_problems = Delimited_Reader.read_file self file on_problems @@ -87,7 +93,9 @@ type Delimited_Format text = Text.from_bytes response.body.bytes self.encoding Delimited_Reader.read_text text self Report_Warning - ## Implements the `Table.write` for this `File_Format`. + ## PRIVATE + ADVANCED + Implements the `Table.write` for this `File_Format`. write_table : File -> Table -> Existing_File_Behavior -> Match_Columns -> Problem_Behavior -> File write_table self file table on_existing_file match_columns on_problems = r = Delimited_Writer.write_file table self file on_existing_file match_columns on_problems diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso index b5263dfc71e1..d447ccc9fdc8 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso @@ -1,6 +1,7 @@ from Standard.Base import all import Standard.Table.Data.Expression.Expression_Error +import Standard.Table.Data.Type.Value_Type.Value_Type polyglot java import org.enso.table.error.ColumnCountMismatchException polyglot java import org.enso.table.error.ColumnNameMismatchException @@ -83,6 +84,7 @@ type No_Output_Columns ## Indicates that one column has been matched by multiple selectors, resulting in ambiguous new names. type Ambiguous_Column_Rename + ## PRIVATE Error (column_name : Text) (new_names : Vector Text) ## PRIVATE @@ -95,14 +97,21 @@ type Ambiguous_Column_Rename ## Indicates that no input columns were selected for the operation, so the operation will cause no effect. type No_Input_Columns_Selected + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "No input columns have been selected for the operation." ## Indicates that an aggregation calculation could not be completed. type Invalid_Aggregation + ## PRIVATE Error (column:Text) (rows:[Integer]) (message:Text) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The "+self.column+" could not be calculated at "+self.rows.short_display_text+": "+self.message @@ -110,8 +119,12 @@ type Invalid_Aggregation ## Indicates that some operation relies on equality on floating-point values, which is not recommended. type Floating_Point_Equality + ## PRIVATE Error (location:Text) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "Relying on equality of floating-point numbers is not recommended (within "+self.location+")." @@ -119,16 +132,24 @@ type Floating_Point_Equality ## Indicates that a text value with a delimiter was included in a concatenation without any quote character type Unquoted_Delimiter + ## PRIVATE Error (column:Text) (rows:[Integer]) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The "+self.column+" at rows "+self.rows.short_display_text+" contains the delimiter and there is no specified quote character." ## Warning when additional warnings occurred. type Additional_Warnings + ## PRIVATE Error (count:Integer) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "There were "+self.count.to_text+" additional issues." @@ -139,8 +160,12 @@ type Additional_Warnings Only the first 10 rows are reported, any additional ones are aggregated into a single instance of `Additional_Invalid_Rows`. type Invalid_Row + ## PRIVATE Error (source_file_line_number : Integer) (index : Integer | Nothing) (row : [Text]) (expected_columns : Integer) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = table_loc = case self.index of @@ -154,8 +179,12 @@ type Invalid_Row ## Indicates how many additional `Invalid_Row` warnings have been suppressed. type Additional_Invalid_Rows + ## PRIVATE Error (count : Integer) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "There were "+self.count.to_text+" additional invalid rows." @@ -163,8 +192,12 @@ type Additional_Invalid_Rows ## Indicates that a quote inside of a delimited file cell has been opened but never closed. type Mismatched_Quote + ## PRIVATE Error (cell_text : Text) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = max_length = 50 @@ -173,27 +206,35 @@ type Mismatched_Quote ## Indicates an unexpected parser error. type Parser_Error + ## PRIVATE Error cause ## Indicates that quoting was disabled for a `Delimited` file, but some cells contained characters that do need quoting which may cause the output file to be corrupted. type Unquoted_Characters_In_Output + ## PRIVATE Warning (column : Text) (rows : [Integer]) + ## PRIVATE + Pretty print the unquoted characters error. to_display_text : Text to_display_text self = altered_rows = self.rows.map ix-> if ix == -1 then "the header" else ix "The "+self.column+" at rows "+altered_rows.short_display_text+" contains characters that need quoting, but quoting is disabled. The generated file may be corrupted." + ## PRIVATE to_text : Text to_text self = "Unquoted_Characters_In_Output.Warning "+self.column.pretty+" "+self.rows.to_text ## Indicates that a specified location was not valid. type Invalid_Location + ## PRIVATE Error (location:Text) + ## PRIVATE + Pretty print the invalid location error. to_display_text : Text to_display_text self = "The location '"+self.location+"' is not valid." @@ -203,15 +244,18 @@ type Invalid_Location Arguments: - column: the column in which the problematic cells appeared, if applicable. It may be empty if the value is parsed outside of a context of a column. - - datatype: The expected datatype. + - value_type: The expected value type. - cells: Contents of the cells that did not match the expected datatype format. type Invalid_Format - Error column:(Text|Nothing) (datatype:(Integer|Number|Date|Time|Time_Of_Day|Boolean)) (cells:[Text]) + ## PRIVATE + Error column:(Text|Nothing) (value_type:Value_Type|Integer|Number|Date|Time|Time_Of_Day|Boolean) (cells:[Text]) + ## PRIVATE + Pretty print the invalid format error. to_display_text : Text to_display_text self = - self.cells.length+" cells in column "+self.column+" had invalid format for datatype "+self.datatype.to_text+"." + self.cells.length+" cells in column "+self.column+" had invalid format for type "+self.value_type.to_text+"." ## Indicates that some values contained leading zeros even though these were not allowed. @@ -221,14 +265,18 @@ type Invalid_Format - datatype: The expected datatype. - cells: Contents of the cells that contained leading zeros. type Leading_Zeros + ## PRIVATE Error column:(Text|Nothing) (datatype:(Integer|Number|Date|Time|Time_Of_Day|Boolean)) (cells:[Text]) + ## PRIVATE + Pretty print the leading zeros error. to_display_text : Text - to_display_text self = "Leading zeros in column "+self.column+" with datatype "+self.datatype.to_text+"." + to_display_text self = "Leading zeros in column "+self.column+" with datatype "+self.value_type.to_text+"." ## Indicates that an empty file was encountered, so no data could be loaded. type Empty_File_Error - + ## PRIVATE + Pretty print the empty file error. to_display_text : Text to_display_text = "It is not allowed to create a Table with no columns, so an empty file could not have been loaded." @@ -238,7 +286,8 @@ type Empty_File_Error ## Indicates that an empty sheet was encountered, so no data could be loaded. type Empty_Sheet_Error - + ## PRIVATE + Pretty print the empty sheet error. to_display_text : Text to_display_text = "It is not allowed to create a Table with no columns, so an empty sheet could not have been loaded." @@ -246,51 +295,60 @@ type Empty_Sheet_Error handle_java_exception = Panic.catch EmptySheetException handler=(_ -> Error.throw Empty_Sheet_Error) -## Indicates that multiple `Column_Type_Selector` match the same column. - - If all matching selectors indicate the same type, the warning is reported but - a parse is attempted anyway. If mixed types are requested, the column is not - parsed due to ambiguity. -type Duplicate_Type_Selector - Error column:Text ambiguous:Boolean - - to_display_text : Text - to_display_text self = "Duplicate type selector for column " + self.column + "." - ## Indicates that the column was already present in the table. type Existing_Column + ## PRIVATE Error column_name + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The column '" + self.column_name + "' already exists, but `Set_Mode.Add` was selected." ## Indicates that the column was not present in the table. type Missing_Column + ## PRIVATE Error column_name + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The column '" + self.column_name + "' was not found, but `Set_Mode.Update` was selected." ## Indicates that the target range contains existing data and the user did not specify to overwrite. type Existing_Data + ## PRIVATE Error message + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "Existing data found: " + self.message ## Indicates that the specified range is not large enough to fit the data. type Range_Exceeded + ## PRIVATE Error message + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "Range exceeded: " + self.message ## Indicates that the existing table has a different number of columns to the new table. type Column_Count_Mismatch + ## PRIVATE Error expected actual + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "Expected " + self.expected.to_text + " columns, got " + self.actual.to_text + "." @@ -305,8 +363,12 @@ type Column_Count_Mismatch ## Indicates that the existing table has a different set of column names to the new table. type Column_Name_Mismatch + ## PRIVATE Error missing extras message + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = self.message @@ -324,6 +386,7 @@ type Column_Name_Mismatch Arguments: - column_name: The name of the column that doesn't exist. type No_Such_Column + ## PRIVATE Error column_name ## PRIVATE @@ -344,6 +407,7 @@ type No_Index_Set_Error to_display_text self = "The table does not have an index set." type Invalid_Value_Type + ## PRIVATE Error expected actual related_column=Nothing ## PRIVATE @@ -363,6 +427,7 @@ type Invalid_Value_Type An error representing an invalid JSON format for conversion. type Invalid_JSON_Format + ## PRIVATE Error input message ## PRIVATE @@ -373,8 +438,7 @@ type Invalid_JSON_Format "The input " + self.input.to_text + " had an invalid format due to: " + self.message.to_text + "." type Column_Type_Mismatch - ## UNSTABLE - + ## PRIVATE An error indicating a mismatch of column types of merged columns. Error (column_name : Text) (expected_type : Text) (got_type : Text) @@ -386,8 +450,7 @@ type Column_Type_Mismatch "The column ["+self.column_name+"] expects type "+self.expected_type+" but one of the provided tables had type "+self.got_type+" which is not compatible with it." type No_Common_Type - ## UNSTABLE - + ## PRIVATE An error indicating that no common type could be found for the merged columns. Error (column_name : Text) @@ -400,60 +463,92 @@ type No_Common_Type "No common type could have been found for the columns corresponding to ["+self.column_name+"]. If you want to allow mixed types, please retype the columns to the `Mixed` before the concatenation (note however that most Database backends do not support `Mixed` types, so it may work only for the in-memory backend)." type Unmatched_Columns - ## UNSTABLE + ## PRIVATE An error indicating that some columns were not present in all of the merged tables. Error (column_names : Vector Text) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The following columns were not present in some of the provided tables: " + (self.column_names.map (n -> "["+n+"]") . join ", ") + ". The missing values have been filled with `Nothing`." type Cross_Join_Row_Limit_Exceeded - ## Indicates that a `cross_join` has been attempted where the right table + ## PRIVATE + Indicates that a `cross_join` has been attempted where the right table has more rows than allowed by the limit. Error (limit : Integer) (existing_rows : Integer) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The cross join operation exceeded the maximum number of rows allowed. The limit is "+self.limit.to_text+" and the number of rows in the right table was "+self.existing_rows.to_text+". The limit may be turned off by setting the `right_row_limit` option to `Nothing`." type Row_Count_Mismatch - ## Indicates that the row counts of zipped tables do not match. + ## PRIVATE + Indicates that the row counts of zipped tables do not match. Error (left_rows : Integer) (right_rows : Integer) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The number of rows in the left table ("+self.left_rows.to_text+") does not match the number of rows in the right table ("+self.right_rows.to_text+")." type Invalid_Aggregate_Column - ## Indicates that a provided name is not found within available columns nor + ## PRIVATE + Indicates that a provided name is not found within available columns nor represents a valid expression. Error (name : Text) (expression_error : Expression_Error | No_Such_Column | Nothing) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The name ["+self.name+"] is not a valid column name nor expression." type Inexact_Type_Coercion - ## Indicates that the requested `Value_Type` is not available in the given + ## PRIVATE + Indicates that the requested `Value_Type` is not available in the given backend, so it was replaced by its closest available type. Warning (requested_type : Value_Type) (actual_type : Value_Type) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The requested type ["+self.requested_type.to_text+"] is not available in the given backend, so it was replaced by its closest available type ["+self.actual_type.to_text+"]." + ## PRIVATE + + Create a human-readable version of the error. to_text : Text to_text self = "Inexact_Type_Coercion.Warning (requested_type = " + self.requested_type.to_text + ") (actual_type = " + self.actual_type.to_text + ")" +## TODO figure out this error in #6112 +type Lossy_Conversion + ## Indicates that some likely not-insignificant information was lost during + a conversion. + Error + type Invalid_Value_For_Type - ## Indicates that a column construction/transformation failed because the + ## PRIVATE + Indicates that a column construction/transformation failed because the provided value is not valid for the requested column type. Error (value : Any) (value_type : Value_Type) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The value ["+self.value.to_text+"] is not valid for the column type ["+self.value_type.to_text+"]." diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Format.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Format.enso index 22abe76d1700..84c75121f3bf 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Format.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Format.enso @@ -45,20 +45,26 @@ type Excel_Format `Excel_Section.Workbook`. Excel (section:Excel_Section=Excel_Section.Workbook) (headers:(Boolean|Infer)=Infer) (xls_format:(Boolean|Infer)=Infer) (default_sheet:Text="EnsoSheet") - ## If the File_Format supports reading from the file, return a configured instance. + ## PRIVATE + ADVANCED + If the File_Format supports reading from the file, return a configured instance. for_file : File -> Excel_Format | Nothing for_file file = is_xls = should_treat_as_xls_format Infer file if is_xls.is_error then Nothing else Excel_Format.Excel xls_format=is_xls - ## If the File_Format supports reading from the web response, return a configured instance. + ## PRIVATE + ADVANCED + If the File_Format supports reading from the web response, return a configured instance. for_web : Text -> URI -> Excel_Format | Nothing for_web _ _ = ## Currently not loading Excel files automatically as these need to be loaded as a connection. Nothing - ## Implements the `File.read` for this `File_Format` + ## PRIVATE + ADVANCED + Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any read self file on_problems = format = should_treat_as_xls_format self.xls_format file @@ -66,7 +72,9 @@ type Excel_Format Excel_Section.Workbook -> Excel_Workbook.new file format self.headers _ -> Excel_Reader.read_file file self.section self.headers on_problems format - ## Implements the `Table.write` for this `File_Format`. + ## PRIVATE + ADVANCED + Implements the `Table.write` for this `File_Format`. Depending on the `section` will control where to write. - If `Excel_Section.Workbook` (the default), the `table` will be written diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Range.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Range.enso index 6a051c81c0a4..c2d550da504f 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Range.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Range.enso @@ -16,7 +16,8 @@ excel_2007_column_limit = 16384 excel_2007_row_limit = 1048576 type Excel_Range - ## Specifies a range within an Excel Workbook. + ## PRIVATE + Specifies a range within an Excel Workbook. Value java_range:Java_Range ## Gets the name of the sheet. @@ -55,11 +56,14 @@ type Excel_Range address : Text address self = self.java_range.getAddress - ## Displays the Excel_Range. + ## PRIVATE + Displays the Excel_Range. to_text : Text to_text self = "Excel_Range " + self.address - ## Validates if a column index (1-based) is within the valid range for + ## PRIVATE + ADVANCED + Validates if a column index (1-based) is within the valid range for Excel. Arguments: @@ -68,7 +72,9 @@ type Excel_Range is_valid_column column = (column > 0) && (column <= excel_2007_column_limit) - ## Validates if a row index (1-based) is within the valid range for Excel. + ## PRIVATE + ADVANCED + Validates if a row index (1-based) is within the valid range for Excel. Arguments: - row: 1-based index to check. @@ -76,7 +82,9 @@ type Excel_Range is_valid_row row = (row > 0) && (row <= excel_2007_row_limit) - ## Given a column name, parses to the index (1-based) or return index + ## PRIVATE + ADVANCED + Given a column name, parses to the index (1-based) or return index unchanged. column_index : (Text|Integer) -> Integer column_index column = case column of diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Parse_Values_Helper.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Parse_Values_Helper.enso index 20d957000b6d..fef6b288d2d7 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Parse_Values_Helper.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Parse_Values_Helper.enso @@ -9,10 +9,10 @@ polyglot java import org.enso.table.parsing.problems.LeadingZeros ## PRIVATE Translates a parse related problem additionally enriching it with expected datatype information that is not originally present on the Java side. -translate_parsing_problem expected_datatype problem = case problem of +translate_parsing_problem expected_value_type problem = case problem of java_problem : InvalidFormat -> - Invalid_Format.Error java_problem.column expected_datatype (Vector.from_polyglot_array java_problem.cells) + Invalid_Format.Error java_problem.column expected_value_type (Vector.from_polyglot_array java_problem.cells) java_problem : LeadingZeros -> - Leading_Zeros.Error java_problem.column expected_datatype (Vector.from_polyglot_array java_problem.cells) + Leading_Zeros.Error java_problem.column expected_value_type (Vector.from_polyglot_array java_problem.cells) _ -> Panic.throw (Illegal_State.Error "Reported an unknown problem type: "+problem.to_text) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Problem_Builder.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Problem_Builder.enso index 6442b736e604..c90d924b2779 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Problem_Builder.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Problem_Builder.enso @@ -5,25 +5,32 @@ import project.Internal.Vector_Builder.Vector_Builder from project.Errors import Missing_Input_Columns, Column_Indexes_Out_Of_Range, Duplicate_Output_Column_Names, Invalid_Output_Column_Names, Invalid_Aggregate_Column +## PRIVATE type Problem_Builder + ## PRIVATE Value types_to_always_throw oob_indices missing_input_columns other + ## PRIVATE report_oob_indices self indices = append_to_ref self.oob_indices indices + ## PRIVATE report_missing_input_columns self columns = append_to_ref self.missing_input_columns columns + ## PRIVATE report_unique_name_strategy self unique_name_strategy = if unique_name_strategy.invalid_names.not_empty then self.report_other_warning (Invalid_Output_Column_Names.Error unique_name_strategy.invalid_names) if unique_name_strategy.renames.not_empty then self.report_other_warning (Duplicate_Output_Column_Names.Error unique_name_strategy.renames) + ## PRIVATE report_other_warning self warning = self.other.append warning - ## Returns a vector containing all reported problems, aggregated. + ## PRIVATE + Returns a vector containing all reported problems, aggregated. build_problemset : Vector build_problemset self = problems = Vector.new_builder @@ -38,7 +45,8 @@ type Problem_Builder problems.to_vector - ## Attaches gathered warnings to the result. + ## PRIVATE + Attaches gathered warnings to the result. Any errors from the `result` take precedence over the ones owned by this builder. attach_problems_after : Problem_Behavior -> Any -> Any @@ -48,7 +56,8 @@ type Problem_Builder problems -> problem_behavior.attach_problems_after result problems - ## Attaches gathered warnings to the result of the provided computation. + ## PRIVATE + Attaches gathered warnings to the result of the provided computation. If in `Report_Error` mode and there are any problems gathered, the first one will be returned as error without even running the computation. attach_problems_before : Problem_Behavior -> Any -> Any diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Rows_View.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Rows_View.enso index 9f6f096b88a3..fbddddc82a0f 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Rows_View.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Rows_View.enso @@ -3,6 +3,7 @@ from Standard.Base import all from project.Data.Table import Table from project.Data.Row import Row +## PRIVATE type Rows_View ## PRIVATE Value (table:Table) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Unique_Name_Strategy.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Unique_Name_Strategy.enso index 29cfdf9c71d0..867c80966e09 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Unique_Name_Strategy.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Unique_Name_Strategy.enso @@ -10,7 +10,9 @@ type Unique_Name_Strategy - deduplicator: Name deduplicator Value deduplicator - ## Creates a new Unique_Name_Strategy instance. + ## PRIVATE + ADVANCED + Creates a new Unique_Name_Strategy instance. This is a mutable data structure, that allows for creating a collection of columns names and making them unique. It will track any duplicates or @@ -26,7 +28,9 @@ type Unique_Name_Strategy new : Unique_Name_Strategy new = Unique_Name_Strategy.Value NameDeduplicator.new - ## Changes names from the second list so that they do not clash with names + ## PRIVATE + ADVANCED + Changes names from the second list so that they do not clash with names from the first list and with each other. It returns a new list where each new name corresponds to a name from the @@ -61,7 +65,9 @@ type Unique_Name_Strategy Vector.from_polyglot_array <| self.deduplicator.combineWithPrefix first second second_prefix - ## Vector of any duplicates renamed. + ## PRIVATE + ADVANCED + Vector of any duplicates renamed. Note that this vector will not contain renames where just the second_prefix was added. @@ -69,12 +75,15 @@ type Unique_Name_Strategy renames self = Vector.from_polyglot_array self.deduplicator.getDuplicatedNames - ## Vector of any invalid names. + ## PRIVATE + ADVANCED + Vector of any invalid names. invalid_names : Vector invalid_names self = Vector.from_polyglot_array self.deduplicator.getInvalidNames - - ## Takes a value and converts to a valid (but not necessarily unique) name. + ## PRIVATE + ADVANCED + Takes a value and converts to a valid (but not necessarily unique) name. Arguments: - name: The column name to make valid. @@ -91,8 +100,9 @@ type Unique_Name_Strategy Nothing -> self.make_valid_name "" _ -> self.make_valid_name input.to_text - - ## Takes a name and gets a unique version. + ## PRIVATE + ADVANCED + Takes a name and gets a unique version. Arguments: - name: The column name to make unique. @@ -104,21 +114,27 @@ type Unique_Name_Strategy make_unique : Text -> Text make_unique self name = self.deduplicator.makeUnique name - ## Tells if the given name has not yet been encountered. + ## PRIVATE + ADVANCED + Tells if the given name has not yet been encountered. It does not use up the name - it needs to be marked with `mark_used` if needed. is_unique : Text -> Boolean is_unique self name = self.deduplicator.isUnique name - ## Takes a list of names and gets a list of unique versions. + ## PRIVATE + ADVANCED + Takes a list of names and gets a list of unique versions. Arguments: - names: The column names to make unique. make_all_unique : Vector Text -> Vector Text make_all_unique self names = names.map self.make_unique - ## Takes a list of names and marks them as used, so that any further names + ## PRIVATE + ADVANCED + Takes a list of names and marks them as used, so that any further names clashing with those will have a prefix added. mark_used : Vector Text -> Nothing mark_used self names = names.each (self.deduplicator.markUsed _) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Widget_Helpers.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Widget_Helpers.enso index 1ed7844e213f..c6274fca3cb7 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Widget_Helpers.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Widget_Helpers.enso @@ -16,8 +16,10 @@ make_column_name_selector table display=Display.Always = Selector for type argument on `Column.parse`. parse_type_selector : Single_Choice parse_type_selector = - choice = ['Auto', 'Integer', 'Decimal', 'Date', 'Date_Time', 'Time_Of_Day', 'Boolean'] - Single_Choice display=Display.Always values=(choice.map n->(Option n)) + choice = ['Auto', 'Value_Type.Integer', 'Value_Type.Float', 'Value_Type.Date', 'Value_Type.Date_Time', 'Value_Type.Time', 'Value_Type.Boolean'] + names = ['Auto', 'Integer', 'Float', 'Date', 'Date_Time', 'Time', 'Boolean'] + options = names.zip choice . map pair-> Option pair.first pair.second + Single_Choice display=Display.Always values=options ## PRIVATE Selector for type argument on `Column.parse`. diff --git a/distribution/lib/Standard/Test/0.0.0-dev/src/Problems.enso b/distribution/lib/Standard/Test/0.0.0-dev/src/Problems.enso index a5aff8e873d8..c5d90fa81b43 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/src/Problems.enso +++ b/distribution/lib/Standard/Test/0.0.0-dev/src/Problems.enso @@ -97,11 +97,14 @@ expect_warning expected_warning result = ## UNSTABLE Checks if the provided value has a specific warning attached and if there are no other warnings. + + As a utility, it also returns the found warning. + Arguments: - expected_warning: The expected warning. It can either by a warning type or a concrete value. - result: The value to check. -expect_only_warning : Any -> Any -> Nothing +expect_only_warning : Any -> Any -> Any expect_only_warning expected_warning result = warnings = get_attached_warnings result is_expected x = @@ -114,6 +117,7 @@ expect_only_warning expected_warning result = if invalid.not_empty then loc = Meta.get_source_location 3 Test.fail "Expected the result to contain only the warning: "+found.to_text+", but it also contained: "+invalid.to_text+' (at '+loc+').' + found ## UNSTABLE diff --git a/distribution/lib/Standard/Visualization/0.0.0-dev/src/Helpers.enso b/distribution/lib/Standard/Visualization/0.0.0-dev/src/Helpers.enso index 198f26cb3186..46635555c6cb 100644 --- a/distribution/lib/Standard/Visualization/0.0.0-dev/src/Helpers.enso +++ b/distribution/lib/Standard/Visualization/0.0.0-dev/src/Helpers.enso @@ -9,16 +9,12 @@ import project.Id.Id from project.Text import get_lazy_visualisation_text_window import project.Table as Table_Visualisation -## UNSTABLE - ADVANCED - +## PRIVATE Specifies that the builtin JSON visualization should be used for any type, unless specified otherwise. Any.default_visualization self = Id.json -## UNSTABLE - ADVANCED - +## PRIVATE Returns a Text used to display this value in the IDE. The particular representation is left unspecified and subject to change in @@ -71,8 +67,7 @@ Any.map_valid self f = f self Any.catch_ : Any -> Any Any.catch_ self ~val = self.catch Any (_-> val) -## UNSTABLE - +## PRIVATE Returns a display representation of the dataflow error on which it is called. > Example @@ -127,15 +122,14 @@ recover_errors ~body = result.catch Any err-> JS_Object.from_pairs [["error", err.to_display_text]] . to_text -## UNSTABLE - ADVANCED +## PRIVATE Guides the visualization system to display the most suitable graphical representation for this table. Vector.default_visualization : Id Vector.default_visualization self = Id.table -## UNSTABLE +## PRIVATE Transform the vector into text for displaying as part of its default visualization. @@ -155,16 +149,14 @@ render_vector object depth=0 max_depth=5 max_length=100 = _ : JS_Object -> render object depth max_depth max_length _ -> object.to_default_visualization_data -## UNSTABLE - ADVANCED +## PRIVATE Guides the visualization system to display the most suitable graphical representation for this table. Array.default_visualization : Id Array.default_visualization self = Id.table -## UNSTABLE - ADVANCED +## PRIVATE Returns a Text used to display this value in the IDE. @@ -193,8 +185,7 @@ Table.lookup_ignore_case self name = self.columns.find if_missing=(Error.throw Nothing) <| col-> col.name.equals_ignore_case name -## UNSTABLE - ADVANCED +## PRIVATE Guides the visualization system to display the most suitable graphical representation for this table. @@ -205,8 +196,7 @@ Table.default_visualization self = if cols.contains "x" && cols.contains "y" then Id.scatter_plot else Id.table -## UNSTABLE - ADVANCED +## PRIVATE Returns a Text used to display this table in the IDE by default. @@ -222,8 +212,7 @@ Table.to_default_visualization_data self = JS_Object.from_pairs [['name', name], ['data', items]] JS_Object.from_pairs [row_count, ['columns', cols]] . to_text -## UNSTABLE - ADVANCED +## PRIVATE Guides the visualization system to display the most suitable graphical representation for this Column. @@ -246,8 +235,7 @@ make_lazy_visualisation_data text text_window_position text_window_size chunk_si if text.length < min_length_for_laziness then text else get_lazy_visualisation_text_window text text_window_position text_window_size chunk_size -## UNSTABLE - ADVANCED +## PRIVATE Returns the data requested to render a lazy view of the default visualisation. Any.to_lazy_visualization_data : Vector Integer -> Vector Integer -> Integer -> Text Any.to_lazy_visualization_data self text_window_position text_window_size chunk_size = @@ -255,15 +243,13 @@ Any.to_lazy_visualization_data self text_window_position text_window_size chunk_ https://www.pivotaltracker.com/story/show/184061302 "" + make_lazy_visualisation_data self.to_default_visualization_data text_window_position text_window_size chunk_size -## UNSTABLE - ADVANCED +## PRIVATE Returns the data requested to render a lazy view of the default visualisation. Text.to_default_visualization_data : Text Text.to_default_visualization_data self = self.to_lazy_visualization_data [0,0] [10,10] 20 -## UNSTABLE - ADVANCED +## PRIVATE Returns the data requested to render a lazy view of the default visualisation. Text.to_lazy_visualization_data : Vector Integer -> Vector Integer -> Integer -> Text Text.to_lazy_visualization_data self text_window_position text_window_size chunk_size = @@ -273,9 +259,7 @@ Text.to_lazy_visualization_data self text_window_position text_window_size chunk https://www.pivotaltracker.com/story/show/184061302 "" + get_lazy_visualisation_text_window self text_window_position text_window_size chunk_size -## UNSTABLE - ADVANCED - +## PRIVATE Shows a JSON serialization of a truncated version of this column, for the benefit of visualization in the IDE. Column.to_default_visualization_data : Text @@ -286,17 +270,13 @@ Column.to_default_visualization_data self = data = ['data', self.to_vector.take (First max_data)] JS_Object.from_pairs [size, name, data] . to_text -## UNSTABLE - ADVANCED - +## PRIVATE Guides the visualization system to display the most suitable graphical representation for this Row. Row.default_visualization : Id Row.default_visualization self = Id.table -## UNSTABLE - ADVANCED - +## PRIVATE Returns a Text used to display this table in the IDE by default. Returns a JSON object containing useful metadata and previews of column @@ -305,8 +285,7 @@ Row.to_default_visualization_data : Text Row.to_default_visualization_data self = self.to_vector.to_default_visualization_data -## UNSTABLE - ADVANCED +## PRIVATE Returns the data requested to render a lazy view of the default visualisation. Table.to_lazy_visualization_data : Vector Integer -> Vector Integer -> Vector Integer -> Integer -> Text Table.to_lazy_visualization_data self table_cell_position text_window_position text_window_size chunk_size = diff --git a/distribution/lib/Standard/Visualization/0.0.0-dev/src/Histogram.enso b/distribution/lib/Standard/Visualization/0.0.0-dev/src/Histogram.enso index 7e1aab83dafe..78ea1ef66742 100644 --- a/distribution/lib/Standard/Visualization/0.0.0-dev/src/Histogram.enso +++ b/distribution/lib/Standard/Visualization/0.0.0-dev/src/Histogram.enso @@ -26,7 +26,6 @@ type Update Value values label ## PRIVATE - Generate JSON that can be consumed by the visualization. to_js_object : JS_Object to_js_object self = diff --git a/distribution/lib/Standard/Visualization/0.0.0-dev/src/Id.enso b/distribution/lib/Standard/Visualization/0.0.0-dev/src/Id.enso index 9052383ff9f8..77f75989cd36 100644 --- a/distribution/lib/Standard/Visualization/0.0.0-dev/src/Id.enso +++ b/distribution/lib/Standard/Visualization/0.0.0-dev/src/Id.enso @@ -1,17 +1,22 @@ from Standard.Base import all -## An ID used by the visualization system to identify different ways of +## PRIVATE + An ID used by the visualization system to identify different ways of displaying data. type Id - ## A builtin visualization, implemented in the graphical interface and not + ## PRIVATE + A builtin visualization, implemented in the graphical interface and not imported from any library. Builtin name - ## A visualization implemented in a library. + ## PRIVATE + A visualization implemented in a library. Library project name - ## Serializes this ID to a JSON format understandable by the graphical + ## PRIVATE + Serializes this ID to a JSON format understandable by the graphical interface. + to_js_object : JS_Object to_js_object self = project = case self of Id.Builtin _ -> Nothing @@ -20,65 +25,47 @@ type Id JS_Object.from_pairs [["name", full_name]] JS_Object.from_pairs [["library", project], ["name", self.name]] - ## UNSTABLE - ADVANCED - + ## PRIVATE An identifier for the builtin JSON visualization json : Id json = Id.Builtin "JSON" - ## UNSTABLE - ADVANCED - + ## PRIVATE An identifier for the builtin Scatter Plot visualization scatter_plot : Id scatter_plot = Id.Builtin "Scatter Plot" - ## UNSTABLE - ADVANCED - + ## PRIVATE An identifier for the builtin Histogram visualization histogram : Id histogram = Id.Builtin "Histogram" - ## UNSTABLE - ADVANCED - + ## PRIVATE An identifier for the builtin Heatmap visualization heatmap : Id heatmap = Id.Builtin "Heatmap" - ## UNSTABLE - ADVANCED - + ## PRIVATE An identifier for the builtin Table visualization table : Id table = Id.Builtin "Table" - ## UNSTABLE - ADVANCED - + ## PRIVATE An identifier for the builtin SQL Query visualization sql_query : Id sql_query = Id.Builtin "SQL Query" - ## UNSTABLE - ADVANCED - + ## PRIVATE An identifier for the builtin Geo Map visualization geo_map : Id geo_map = Id.Builtin "Geo Map" - ## UNSTABLE - ADVANCED - + ## PRIVATE An identifier for the builtin Image visualization image : Id image = Id.Builtin "Image" - ## UNSTABLE - ADVANCED - + ## PRIVATE Creates an ID for a library-defined visualization Arguments: diff --git a/docs/language-server/protocol-language-server.md b/docs/language-server/protocol-language-server.md index 7aeb7c7f545b..8228d60c1caa 100644 --- a/docs/language-server/protocol-language-server.md +++ b/docs/language-server/protocol-language-server.md @@ -4212,6 +4212,14 @@ main = Sent from client to the server to receive the full suggestions database. +#### Deprecated + +The request always returns empty `entries` field with the correct +`currentVersion`. The suggestions are sent during the initial project +compilation as a part of +[`search/suggestionsDatabaseUpdate`](#searchsuggestionsdatabaseupdate) +notification. + - **Type:** Request - **Direction:** Client -> Server - **Connection:** Protocol diff --git a/engine/language-server/src/main/resources/application.conf b/engine/language-server/src/main/resources/application.conf index 509a892c0f13..cc2f531d0ec4 100644 --- a/engine/language-server/src/main/resources/application.conf +++ b/engine/language-server/src/main/resources/application.conf @@ -8,6 +8,7 @@ logging-service.logger { akka.event = error akka.io = error akka.stream = error - slick = error + slick.jdbc.JdbcBackend.statement = error # log SQL queries on debug level + slick."*" = error org.eclipse.jgit = error } diff --git a/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala b/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala index 1e20da908337..bca20712a879 100644 --- a/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala +++ b/engine/language-server/src/main/scala/org/enso/languageserver/search/SuggestionsHandler.scala @@ -342,13 +342,8 @@ final class SuggestionsHandler( .pipeTo(sender()) case GetSuggestionsDatabase => - suggestionsRepo.getAll - .map { case (version, entries) => - GetSuggestionsDatabaseResult( - version, - entries.map { entry => SuggestionDatabaseEntry(entry) } - ) - } + suggestionsRepo.currentVersion + .map(GetSuggestionsDatabaseResult(_, Seq())) .pipeTo(sender()) case Completion(path, pos, selfType, returnType, tags, isStatic) => @@ -490,8 +485,8 @@ final class SuggestionsHandler( /** Handle the suggestions of the loaded library. * - * Adds the new suggestions to the suggestions database and sends the - * appropriate notification to the client. + * Adds the new suggestions to the suggestions database and returns the + * appropriate notification message. * * @param suggestions the loaded suggestions * @return the API suggestions database update notification @@ -500,34 +495,12 @@ final class SuggestionsHandler( suggestions: Vector[Suggestion] ): Future[SuggestionsDatabaseUpdateNotification] = { for { - treeResults <- suggestionsRepo.applyTree( - suggestions.map(Api.SuggestionUpdate(_, Api.SuggestionAction.Add())) - ) - version <- suggestionsRepo.currentVersion + (version, ids) <- suggestionsRepo.insertAll(suggestions) } yield { - val treeUpdates = treeResults.flatMap { - case QueryResult(ids, Api.SuggestionUpdate(suggestion, action)) => - action match { - case Api.SuggestionAction.Add() => - if (ids.isEmpty) { - val verb = action.getClass.getSimpleName - logger.error("Cannot {} [{}].", verb, suggestion) - } - ids.map( - SuggestionsDatabaseUpdate.Add( - _, - suggestion - ) - ) - case action => - logger.error( - "Invalid action during suggestions loading [{}].", - action - ) - Seq() - } - } - SuggestionsDatabaseUpdateNotification(version, treeUpdates) + val updates = ids + .zip(suggestions) + .map(SuggestionsDatabaseUpdate.Add.tupled) + SuggestionsDatabaseUpdateNotification(version, updates) } } diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala index e24469aa8026..6ba29a69f007 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/search/SuggestionsHandlerSpec.scala @@ -11,7 +11,6 @@ import org.enso.languageserver.capability.CapabilityProtocol.{ import org.enso.languageserver.data._ import org.enso.languageserver.event.InitializedEvent import org.enso.languageserver.filemanager._ -import org.enso.languageserver.search.SearchProtocol.SuggestionDatabaseEntry import org.enso.languageserver.session.JsonSession import org.enso.languageserver.session.SessionRouter.DeliverToJsonController import org.enso.polyglot.data.{Tree, TypeGraph} @@ -55,7 +54,7 @@ class SuggestionsHandlerSpec "SuggestionsHandler" should { - "subscribe to notification updates" /*taggedAs Retry*/ in withDb { + "subscribe to notification updates" taggedAs Retry in withDb { (_, _, _, _, handler) => val clientId = UUID.randomUUID() @@ -341,7 +340,7 @@ class SuggestionsHandlerSpec DeliverToJsonController( clientId, SearchProtocol.SuggestionsDatabaseUpdateNotification( - (updates1.size + updates2.size).toLong, + (updates1.size + updates2.size).toLong - 1, updates2 ) ) @@ -716,16 +715,13 @@ class SuggestionsHandlerSpec expectMsg(SearchProtocol.GetSuggestionsDatabaseResult(0, Seq())) } - "get suggestions database" taggedAs Retry in withDb { + "get suggestions database return empty result" taggedAs Retry in withDb { (_, repo, _, _, handler) => Await.ready(repo.insert(Suggestions.constructor), Timeout) handler ! SearchProtocol.GetSuggestionsDatabase expectMsg( - SearchProtocol.GetSuggestionsDatabaseResult( - 1, - Seq(SuggestionDatabaseEntry(1L, Suggestions.constructor)) - ) + SearchProtocol.GetSuggestionsDatabaseResult(1, Seq()) ) } @@ -759,15 +755,15 @@ class SuggestionsHandlerSpec expectMsg( SearchProtocol.CompletionResult( - Suggestions.all.length.toLong, + 1L, Seq( - inserted(0).get, - inserted(1).get, - inserted(2).get, - inserted(6).get, - inserted(7).get, - inserted(8).get, - inserted(3).get + inserted(0), + inserted(1), + inserted(2), + inserted(6), + inserted(7), + inserted(8), + inserted(3) ) ) ) @@ -788,8 +784,8 @@ class SuggestionsHandlerSpec expectMsg( SearchProtocol.CompletionResult( - Suggestions.all.length.toLong, - Seq(methodId, methodOnAnyId).flatten + 1L, + Seq(methodId, methodOnAnyId) ) ) } @@ -813,8 +809,8 @@ class SuggestionsHandlerSpec expectMsg( SearchProtocol.CompletionResult( - Suggestions.all.length.toLong, - Seq(integerMethodId, numberMethodId, anyMethodId).flatten + 1L, + Seq(integerMethodId, numberMethodId, anyMethodId) ) ) } @@ -835,8 +831,8 @@ class SuggestionsHandlerSpec expectMsg( SearchProtocol.CompletionResult( - Suggestions.all.length.toLong, - Seq(anyMethodId).flatten + 1L, + Seq(anyMethodId) ) ) } @@ -856,8 +852,8 @@ class SuggestionsHandlerSpec expectMsg( SearchProtocol.CompletionResult( - Suggestions.all.length.toLong, - Seq(functionId).flatten + 1L, + Seq(functionId) ) ) } @@ -877,8 +873,8 @@ class SuggestionsHandlerSpec expectMsg( SearchProtocol.CompletionResult( - Suggestions.all.length.toLong, - Seq(localId).flatten + 1L, + Seq(localId) ) ) } @@ -910,8 +906,8 @@ class SuggestionsHandlerSpec expectMsg( SearchProtocol.CompletionResult( - all.length.toLong, - Seq(methodId).flatten + 1L, + Seq(methodId) ) ) } diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/SuggestionsHandlerEventsTest.scala b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/SuggestionsHandlerEventsTest.scala index 85056674b3f2..ef60d9895483 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/SuggestionsHandlerEventsTest.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/SuggestionsHandlerEventsTest.scala @@ -371,145 +371,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { { "jsonrpc" : "2.0", "id" : 3, "result" : { - "entries" : [ - { - "id" : 1, - "suggestion" : { - "type" : "type", - "module" : "local.Test.Main", - "name" : "Newtype", - "params" : [ - { - "name" : "a", - "reprType" : "Any", - "isSuspended" : false, - "hasDefault" : false, - "defaultValue" : null, - "tagValues" : null - } - ], - "parentType" : "Any" - } - }, - { - "id" : 2, - "suggestion" : { - "type" : "constructor", - "module" : "local.Test.Main", - "name" : "MyType", - "arguments" : [ - { - "name" : "a", - "reprType" : "Any", - "isSuspended" : false, - "hasDefault" : false, - "defaultValue" : null, - "tagValues" : null - } - ], - "returnType" : "MyAtom", - "documentation" : " PRIVATE\n\n A key-value store. This type assumes all keys are pairwise comparable,\n using the `<`, `>` and `==` operators.\n\n Arguments:\n - one: The first.\n - two_three: The *second*.\n\n ? Info\n Here is a thing." - } - }, - { - "id" : 4, - "suggestion" : { - "type" : "function", - "externalId" : "78d452ce-ed48-48f1-b4f2-b7f45f8dff89", - "module" : "local.Test.Main", - "name" : "print", - "arguments" : [ - { - "name" : "a", - "reprType" : "Any", - "isSuspended" : false, - "hasDefault" : false, - "defaultValue" : null, - "tagValues" : null - }, - { - "name" : "b", - "reprType" : "Any", - "isSuspended" : true, - "hasDefault" : false, - "defaultValue" : null, - "tagValues" : null - }, - { - "name" : "c", - "reprType" : "Any", - "isSuspended" : false, - "hasDefault" : true, - "defaultValue" : "C", - "tagValues" : null - } - ], - "returnType" : "IO", - "scope" : { - "start" : { - "line" : 1, - "character" : 9 - }, - "end" : { - "line" : 1, - "character" : 22 - } - }, - "documentation" : "My Function" - } - }, - { - "id" : 3, - "suggestion" : { - "type" : "method", - "externalId" : "ea9d7734-26a7-4f65-9dd9-c648eaf57d63", - "module" : "local.Test.Main", - "name" : "foo", - "arguments" : [ - { - "name" : "this", - "reprType" : "MyType", - "isSuspended" : false, - "hasDefault" : false, - "defaultValue" : null, - "tagValues" : null - }, - { - "name" : "foo", - "reprType" : "Number", - "isSuspended" : false, - "hasDefault" : true, - "defaultValue" : "42", - "tagValues" : null - } - ], - "selfType" : "MyType", - "returnType" : "Number", - "isStatic" : false, - "documentation" : "Lovely" - } - }, - { - "id" : 5, - "suggestion" : { - "type" : "local", - "externalId" : "dc077227-d9b6-4620-9b51-792c2a69419d", - "module" : "local.Test.Main", - "name" : "x", - "returnType" : "Number", - "scope" : { - "start" : { - "line" : 21, - "character" : 0 - }, - "end" : { - "line" : 89, - "character" : 0 - } - } - } - } - ], + "entries" : [], "currentVersion" : 5 } }""") @@ -629,7 +491,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { } } ], - "currentVersion" : 8 + "currentVersion" : 7 } } """) @@ -673,7 +535,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { } } ], - "currentVersion" : 9 + "currentVersion" : 8 } } """) @@ -717,7 +579,7 @@ class SuggestionsHandlerEventsTest extends BaseServerTest with FlakySpec { "id" : 5 } ], - "currentVersion" : 9 + "currentVersion" : 8 } } """) diff --git a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala index aa47d780c0df..ef5720e9a732 100644 --- a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala +++ b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala @@ -52,6 +52,19 @@ object Suggestion { type ExternalId = UUID + /** @return `true` if the suggestion id defined in the global (module) scope. + */ + def isGlobal(suggestion: Suggestion): Boolean = + suggestion match { + case _: Suggestion.Module => true + case _: Suggestion.Type => true + case _: Suggestion.Constructor => true + case _: Suggestion.Method => true + case _: Suggestion.Conversion => true + case _: Suggestion.Function => false + case _: Suggestion.Local => false + } + /** The type of a suggestion. */ sealed trait Kind object Kind { @@ -82,7 +95,6 @@ object Suggestion { /** The conversion suggestion. */ case object Conversion extends Kind { val From = "from" - val To = "to" } /** The function suggestion. */ @@ -112,13 +124,13 @@ object Suggestion { def apply(suggestion: Suggestion): Option[String] = suggestion match { - case _: Module => None - case _: Type => None - case _: Constructor => None - case method: Method => Some(method.selfType) - case _: Conversion => None - case _: Function => None - case _: Local => None + case _: Module => None + case _: Type => None + case constructor: Constructor => Some(constructor.returnType) + case method: Method => Some(method.selfType) + case conversion: Conversion => Some(conversion.sourceType) + case _: Function => None + case _: Local => None } } @@ -167,6 +179,25 @@ object Suggestion { */ case class Scope(start: Position, end: Position) + object Scope { + + /** Creates a scope from the provided suggestion. + * + * @param suggestion the provided suggestion + * @return the scope of the suggestion + */ + def apply(suggestion: Suggestion): Option[Scope] = + suggestion match { + case _: Module => None + case _: Type => None + case _: Constructor => None + case _: Method => None + case _: Conversion => None + case function: Function => Some(function.scope) + case local: Local => Some(local.scope) + } + } + /** A module. * * @param module the fully qualified module name diff --git a/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala b/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala index 0b2b4cf55539..765137213dd5 100644 --- a/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala +++ b/engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala @@ -27,6 +27,7 @@ class ContextFactory { * @param useGlobalIrCacheLocation whether or not to use the global IR cache * location * @param options additional options for the Context + * @param executionEnvironment optional name of the execution environment to use during execution * @return configured Context instance */ def create( @@ -40,8 +41,12 @@ class ContextFactory { strictErrors: Boolean = false, useGlobalIrCacheLocation: Boolean = true, enableAutoParallelism: Boolean = false, + executionEnvironment: Option[String] = None, options: java.util.Map[String, String] = java.util.Collections.emptyMap ): PolyglotContext = { + executionEnvironment.foreach { name => + options.put("enso.ExecutionEnvironment", name) + } val context = Context .newBuilder() .allowExperimentalOptions(true) diff --git a/engine/runner/src/main/scala/org/enso/runner/Main.scala b/engine/runner/src/main/scala/org/enso/runner/Main.scala index 27d6cb853ac9..f9e5494a651f 100644 --- a/engine/runner/src/main/scala/org/enso/runner/Main.scala +++ b/engine/runner/src/main/scala/org/enso/runner/Main.scala @@ -74,6 +74,7 @@ object Main { private val AUTH_TOKEN = "auth-token" private val AUTO_PARALLELISM_OPTION = "with-auto-parallelism" private val SKIP_GRAALVM_UPDATER = "skip-graalvm-updater" + private val EXECUTION_ENVIRONMENT_OPTION = "execution-environment" private lazy val logger = Logger[Main.type] @@ -355,6 +356,16 @@ object Main { .desc("Skips GraalVM and its components setup during bootstrapping.") .build + val executionEnvironmentOption = CliOption.builder + .longOpt(EXECUTION_ENVIRONMENT_OPTION) + .hasArg(true) + .numberOfArgs(1) + .argName("name") + .desc( + "Execution environment to use during execution (`live`/`design`). Defaults to `design`." + ) + .build() + val options = new Options options .addOption(help) @@ -396,6 +407,7 @@ object Main { .addOptionGroup(cacheOptionsGroup) .addOption(autoParallelism) .addOption(skipGraalVMUpdater) + .addOption(executionEnvironmentOption) options } @@ -530,6 +542,7 @@ object Main { * @param enableIrCaches are IR caches enabled * @param inspect shall inspect option be enabled * @param dump shall graphs be sent to the IGV + * @apram executionEnvironment optional name of the execution environment to use during execution */ private def run( path: String, @@ -540,7 +553,8 @@ object Main { enableIrCaches: Boolean, enableAutoParallelism: Boolean, inspect: Boolean, - dump: Boolean + dump: Boolean, + executionEnvironment: Option[String] ): Unit = { val file = new File(path) if (!file.exists) { @@ -580,6 +594,7 @@ object Main { enableIrCaches, strictErrors = true, enableAutoParallelism = enableAutoParallelism, + executionEnvironment = executionEnvironment, options = options ) if (projectMode) { @@ -1092,7 +1107,8 @@ object Main { shouldEnableIrCaches(line), line.hasOption(AUTO_PARALLELISM_OPTION), line.hasOption(INSPECT_OPTION), - line.hasOption(DUMP_GRAPHS_OPTION) + line.hasOption(DUMP_GRAPHS_OPTION), + Option(line.getOptionValue(EXECUTION_ENVIRONMENT_OPTION)) ) } if (line.hasOption(REPL_OPTION)) { diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/BuiltinTypesTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/BuiltinTypesTest.scala index bfeaa54b04bc..939b2469ec65 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/BuiltinTypesTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/BuiltinTypesTest.scala @@ -258,7 +258,7 @@ class BuiltinTypesTest 4 ) should contain theSameElementsAs Seq( Api.Response(requestId, Api.PushContextResponse(contextId)), - TestMessages.update(contextId, idY, ConstantsGen.FUNCTION_BUILTIN), + TestMessages.update(contextId, idY, ConstantsGen.FUNCTION), TestMessages.update(contextId, idMain, ConstantsGen.INTEGER), context.executionComplete(contextId) ) @@ -319,7 +319,7 @@ class BuiltinTypesTest 3 ) should contain theSameElementsAs Seq( Api.Response(requestId, Api.PushContextResponse(contextId)), - TestMessages.update(contextId, idMain, ConstantsGen.FUNCTION_BUILTIN), + TestMessages.update(contextId, idMain, ConstantsGen.FUNCTION), context.executionComplete(contextId) ) } diff --git a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala index 35f51582af55..ce40026459cc 100644 --- a/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala +++ b/engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala @@ -1952,7 +1952,7 @@ class RuntimeServerTest context.receiveNIgnoreStdLib(5) should contain theSameElementsAs Seq( Api.Response(Api.BackgroundJobsStartedNotification()), Api.Response(requestId, Api.PushContextResponse(contextId)), - TestMessages.update(contextId, xId, ConstantsGen.FUNCTION_BUILTIN), + TestMessages.update(contextId, xId, ConstantsGen.FUNCTION), TestMessages.update(contextId, mainRes, ConstantsGen.NOTHING), context.executionComplete(contextId) ) diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/semantic/ListBenchmarks.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/semantic/ListBenchmarks.java index 0ac40ed24f1f..ef8b748ab812 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/semantic/ListBenchmarks.java +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/semantic/ListBenchmarks.java @@ -51,8 +51,25 @@ public void initializeBenchmark(BenchmarkParams params) throws Exception { from Standard.Base.Data.List.List import Cons, Nil import Standard.Base.IO + type Lenivy + Nic + Hlava ~x ~xs + + map self fn = case self of + Lenivy.Nic -> Lenivy.Nic + Lenivy.Hlava x xs -> Lenivy.Hlava (fn x) (xs.map fn) + plus_one list = list.map (x -> x + 1) + leniva_suma list acc = case list of + Lenivy.Nic -> acc + Lenivy.Hlava x xs -> @Tail_Call leniva_suma xs acc+x + + lenivy_generator n = + go x v l = if x > n then l else + @Tail_Call go x+1 v+1 (Lenivy.Hlava v l) + go 1 1 Lenivy.Nic + sum list acc = case list of Nil -> acc @@ -69,15 +86,22 @@ public void initializeBenchmark(BenchmarkParams params) throws Exception { this.self = module.invokeMember("get_associated_type"); Function getMethod = (name) -> module.invokeMember("get_method", self, name); - Value longList = getMethod.apply("generator").execute(self, LENGTH_OF_EXPERIMENT); - this.plusOne = getMethod.apply("plus_one"); - this.sum = getMethod.apply("sum"); switch (benchmarkName) { case "mapOverList": { - this.list = longList; - this.oldSum = sum.execute(self, longList, 0); + this.list = getMethod.apply("generator").execute(self, LENGTH_OF_EXPERIMENT); + this.sum = getMethod.apply("sum"); + this.oldSum = sum.execute(self, this.list, 0); + if (!this.oldSum.fitsInLong()) { + throw new AssertionError("Expecting a number " + this.oldSum); + } + break; + } + case "mapOverLazyList": { + this.list = getMethod.apply("lenivy_generator").execute(self, LENGTH_OF_EXPERIMENT); + this.sum = getMethod.apply("leniva_suma"); + this.oldSum = sum.execute(self, this.list, 0); if (!this.oldSum.fitsInLong()) { throw new AssertionError("Expecting a number " + this.oldSum); } @@ -93,6 +117,11 @@ public void mapOverList(Blackhole matter) { performBenchmark(matter); } + @Benchmark + public void mapOverLazyList(Blackhole matter) { + performBenchmark(matter); + } + private void performBenchmark(Blackhole hole) throws AssertionError { var newList = plusOne.execute(self, list); var newSum = sum.execute(self, newList, 0); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/EnsoLanguage.java b/engine/runtime/src/main/java/org/enso/interpreter/EnsoLanguage.java index ec0165004327..f299fb948785 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/EnsoLanguage.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/EnsoLanguage.java @@ -36,7 +36,7 @@ import org.enso.interpreter.node.ExpressionNode; import org.enso.interpreter.node.ProgramRootNode; import org.enso.interpreter.runtime.EnsoContext; -import org.enso.interpreter.runtime.state.IOPermissions; +import org.enso.interpreter.runtime.state.ExecutionEnvironment; import org.enso.interpreter.runtime.tag.AvoidIdInstrumentationTag; import org.enso.interpreter.runtime.tag.IdentifiedTag; import org.enso.interpreter.runtime.tag.Patchable; @@ -320,12 +320,13 @@ private boolean astContainsExprTypes(Tree ast, List> exprT } @Option( - name = "IOEnvironment", - category = OptionCategory.USER, - help = "The IO environment for program execution.") - public static final OptionKey IO_ENVIRONMENT = - new OptionKey<>( - IOPermissions.PRODUCTION, new OptionType<>("IOEnvironment", IOPermissions::forName)); + name = "ExecutionEnvironment", + category = OptionCategory.USER, + help = "The environment for program execution. Defaults to `design`.") + public static final OptionKey EXECUTION_ENVIRONMENT = + new OptionKey<>( + ExecutionEnvironment.DESIGN, new OptionType<>("ExecutionEnvironment", ExecutionEnvironment::forName)); + private static final OptionDescriptors OPTIONS = OptionDescriptors.createUnion( diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/controlflow/permission/PermissionGuardNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/controlflow/permission/PermissionGuardNode.java deleted file mode 100644 index 4a895703e497..000000000000 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/controlflow/permission/PermissionGuardNode.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.enso.interpreter.node.controlflow.permission; - -import com.oracle.truffle.api.frame.VirtualFrame; -import com.oracle.truffle.api.profiles.BranchProfile; -import org.enso.interpreter.node.ExpressionNode; -import org.enso.interpreter.runtime.EnsoContext; -import org.enso.interpreter.runtime.callable.function.Function; -import org.enso.interpreter.runtime.data.text.Text; -import org.enso.interpreter.runtime.error.PanicException; -import org.enso.interpreter.runtime.state.State; - -public class PermissionGuardNode extends ExpressionNode { - private @Child ExpressionNode body; - private final boolean checkInput; - private final boolean checkOutput; - private final BranchProfile inputDisallowed = BranchProfile.create(); - private final BranchProfile outputDisallowed = BranchProfile.create(); - - public PermissionGuardNode(ExpressionNode body, boolean checkInput, boolean checkOutput) { - this.body = body; - this.checkInput = checkInput; - this.checkOutput = checkOutput; - } - - @Override - public Object executeGeneric(VirtualFrame frame) { - State state = Function.ArgumentsHelper.getState(frame.getArguments()); - - if (checkInput && !state.getIoPermissions().isInputAllowed()) { - inputDisallowed.enter(); - throw new PanicException( - EnsoContext.get(this) - .getBuiltins() - .error() - .getForbiddenOperation() - .newInstance(Text.create("Input")), - this); - } - - if (checkOutput && !state.getIoPermissions().isOutputAllowed()) { - outputDisallowed.enter(); - throw new PanicException( - EnsoContext.get(this) - .getBuiltins() - .error() - .getForbiddenOperation() - .newInstance(Text.create("Output")), - this); - } - - return body.executeGeneric(frame); - } -} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/error/ForbiddenOperation.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/error/ForbiddenOperation.java index 8387405d8e30..90e63b31bd84 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/error/ForbiddenOperation.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/error/ForbiddenOperation.java @@ -7,6 +7,12 @@ @BuiltinType public class ForbiddenOperation extends UniquelyConstructibleBuiltin { + + @Override + protected String getConstructorName() { + return "Error"; + } + @Override protected List getConstructorParamNames() { return List.of("operation"); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/error/Unimplemented.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/error/Unimplemented.java new file mode 100644 index 000000000000..d363192b9c92 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/error/Unimplemented.java @@ -0,0 +1,20 @@ +package org.enso.interpreter.node.expression.builtin.error; + +import org.enso.interpreter.dsl.BuiltinType; +import org.enso.interpreter.node.expression.builtin.UniquelyConstructibleBuiltin; + +import java.util.List; + +@BuiltinType +public class Unimplemented extends UniquelyConstructibleBuiltin { + + @Override + protected String getConstructorName() { + return "Error"; + } + + @Override + protected List getConstructorParamNames() { + return List.of("message"); + } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/Context.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/Context.java new file mode 100644 index 000000000000..9245214dc505 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/Context.java @@ -0,0 +1,27 @@ +package org.enso.interpreter.node.expression.builtin.runtime; + +import org.enso.interpreter.dsl.BuiltinType; +import org.enso.interpreter.node.expression.builtin.Builtin; +import org.enso.interpreter.runtime.callable.atom.AtomConstructor; + +import java.util.List; + +@BuiltinType +public class Context extends Builtin { + @Override + protected List getDeclaredConstructors() { + return List.of(new Cons(INPUT_NAME), new Cons(OUTPUT_NAME)); + } + + public static final String INPUT_NAME = "Input"; + + public static final String OUTPUT_NAME = "Output"; + + public AtomConstructor getInput() { + return getConstructors()[0]; + } + + public AtomConstructor getOutput() { + return getConstructors()[1]; + } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/ContextIsEnabledNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/ContextIsEnabledNode.java new file mode 100644 index 000000000000..b5d50efa8e3a --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/ContextIsEnabledNode.java @@ -0,0 +1,32 @@ +package org.enso.interpreter.node.expression.builtin.runtime; + +import com.oracle.truffle.api.nodes.Node; +import org.enso.interpreter.dsl.BuiltinMethod; +import org.enso.interpreter.node.expression.builtin.text.util.ExpectStringNode; +import org.enso.interpreter.runtime.EnsoContext; +import org.enso.interpreter.runtime.callable.atom.Atom; +import org.enso.interpreter.runtime.error.PanicException; +import org.enso.interpreter.runtime.state.ExecutionEnvironment; +import org.enso.interpreter.runtime.state.State; + +@BuiltinMethod( + type = "Context", + name = "is_enabled", + description = "Check if the context is enabled in the provided execution environment.") +public class ContextIsEnabledNode extends Node { + private @Child ExpectStringNode expectStringNode = ExpectStringNode.build(); + + Object execute(State state, Atom self, Object environmentName) { + String envName = expectStringNode.execute(environmentName); + ExecutionEnvironment currentEnv = state.currentEnvironment(); + if (!currentEnv.getName().equals(envName)) { + Atom error = + EnsoContext.get(this) + .getBuiltins() + .error() + .makeUnimplemented("execution environment mismatch"); + throw new PanicException(error, this); + } + return currentEnv.hasContextEnabled(self.getConstructor().getName()); + } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeCurrentExecutionEnvironmentNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeCurrentExecutionEnvironmentNode.java new file mode 100644 index 000000000000..a881e75af174 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeCurrentExecutionEnvironmentNode.java @@ -0,0 +1,17 @@ +package org.enso.interpreter.node.expression.builtin.runtime; + +import com.oracle.truffle.api.nodes.Node; +import org.enso.interpreter.dsl.BuiltinMethod; +import org.enso.interpreter.runtime.data.text.Text; +import org.enso.interpreter.runtime.state.State; + +@BuiltinMethod( + type = "Runtime", + name = "current_execution_environment", + description = "Returns the name of the current execution environment.", + autoRegister = false) +public class RuntimeCurrentExecutionEnvironmentNode extends Node { + Object execute(State state) { + return Text.create(state.currentEnvironment().getName()); + } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/AllowOutputInNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeWithDisabledContextNode.java similarity index 65% rename from engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/AllowOutputInNode.java rename to engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeWithDisabledContextNode.java index 3d1caaf57822..f0ceddb93d5b 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/AllowOutputInNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeWithDisabledContextNode.java @@ -6,20 +6,21 @@ import org.enso.interpreter.node.BaseNode; import org.enso.interpreter.node.callable.thunk.ThunkExecutorNode; import org.enso.interpreter.node.expression.builtin.text.util.ExpectStringNode; +import org.enso.interpreter.runtime.callable.atom.Atom; import org.enso.interpreter.runtime.state.State; @BuiltinMethod( type = "Runtime", - name = "allow_output_in", - description = "Allows output in the specified scope.", + name = "with_disabled_context_builtin", + description = "Disallows context in the specified scope.", autoRegister = false) -public class AllowOutputInNode extends Node { +public class RuntimeWithDisabledContextNode extends Node { private @Child ThunkExecutorNode thunkExecutorNode = ThunkExecutorNode.build(); private @Child ExpectStringNode expectStringNode = ExpectStringNode.build(); - Object execute(State state, Object env_name, @Suspend Object action) { + Object execute(State state, Atom context, @Suspend Object action, Object env_name) { String envName = expectStringNode.execute(env_name); return thunkExecutorNode.executeThunk( - action, state.withOutputAllowedIn(envName), BaseNode.TailStatus.NOT_TAIL); + action, state.withContextDisabledIn(context, envName), BaseNode.TailStatus.NOT_TAIL); } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/AllowInputInNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeWithEnabledContextNode.java similarity index 65% rename from engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/AllowInputInNode.java rename to engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeWithEnabledContextNode.java index 5d64fa3baae4..240329569a99 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/AllowInputInNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/runtime/RuntimeWithEnabledContextNode.java @@ -6,20 +6,21 @@ import org.enso.interpreter.node.BaseNode; import org.enso.interpreter.node.callable.thunk.ThunkExecutorNode; import org.enso.interpreter.node.expression.builtin.text.util.ExpectStringNode; +import org.enso.interpreter.runtime.callable.atom.Atom; import org.enso.interpreter.runtime.state.State; @BuiltinMethod( type = "Runtime", - name = "allow_input_in", - description = "Allows input in the specified scope.", + name = "with_enabled_context_builtin", + description = "Allows context in the specified scope.", autoRegister = false) -public class AllowInputInNode extends Node { +public class RuntimeWithEnabledContextNode extends Node { private @Child ThunkExecutorNode thunkExecutorNode = ThunkExecutorNode.build(); private @Child ExpectStringNode expectStringNode = ExpectStringNode.build(); - Object execute(State state, Object env_name, @Suspend Object action) { + Object execute(State state, Atom context, @Suspend Object action, Object env_name) { String envName = expectStringNode.execute(env_name); return thunkExecutorNode.executeThunk( - action, state.withInputAllowedIn(envName), BaseNode.TailStatus.NOT_TAIL); + action, state.withContextEnabledIn(context, envName), BaseNode.TailStatus.NOT_TAIL); } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/text/AnyToDisplayTextNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/text/AnyToDisplayTextNode.java index e0bed63431ab..d2391a655b63 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/text/AnyToDisplayTextNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/text/AnyToDisplayTextNode.java @@ -1,5 +1,7 @@ package org.enso.interpreter.node.expression.builtin.text; +import com.ibm.icu.text.BreakIterator; +import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.Fallback; import com.oracle.truffle.api.dsl.Specialization; @@ -11,6 +13,7 @@ import org.enso.interpreter.dsl.BuiltinMethod; import org.enso.interpreter.node.expression.builtin.text.util.TypeToDisplayTextNode; import org.enso.interpreter.runtime.data.text.Text; +import org.enso.polyglot.common_utils.Core_Text_Utils; @BuiltinMethod(type = "Any", name = "to_display_text") public abstract class AnyToDisplayTextNode extends Node { @@ -32,6 +35,22 @@ Text showExceptions( } } + @Specialization + Text convertText(Text self) { + final var limit = 80; + if (self.length() < limit) { + return self; + } else { + return takePrefix(self, limit); + } + } + + @CompilerDirectives.TruffleBoundary + private static Text takePrefix(Text self, final int limit) { + var prefix = Core_Text_Utils.take_prefix(self.toString(), limit); + return Text.create(prefix); + } + @Fallback Text doShowType(Object self, @Cached TypeToDisplayTextNode typeToDisplayTextNode) { return Text.create(typeToDisplayTextNode.execute(self)); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 992b757d4685..b4495eda1faa 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -21,7 +21,7 @@ import org.enso.interpreter.runtime.builtin.Builtins; import org.enso.interpreter.runtime.data.Type; import org.enso.interpreter.runtime.scope.TopLevelScope; -import org.enso.interpreter.runtime.state.IOPermissions; +import org.enso.interpreter.runtime.state.ExecutionEnvironment; import org.enso.interpreter.runtime.state.State; import org.enso.interpreter.runtime.util.TruffleFileSystem; import org.enso.interpreter.util.ScalaConversions; @@ -80,7 +80,7 @@ public class EnsoContext { private final AtomicLong clock = new AtomicLong(); private final Shape rootStateShape = Shape.newBuilder().layout(State.Container.class).build(); - private final IOPermissions rootIOPermissions; + private final ExecutionEnvironment executionEnvironment; /** * Creates a new Enso context. @@ -112,7 +112,7 @@ public EnsoContext( var isParallelismEnabled = getOption(RuntimeOptions.ENABLE_AUTO_PARALLELISM_KEY); this.isIrCachingDisabled = getOption(RuntimeOptions.DISABLE_IR_CACHES_KEY) || isParallelismEnabled; - this.rootIOPermissions = getOption(EnsoLanguage.IO_ENVIRONMENT); + this.executionEnvironment = getOption(EnsoLanguage.EXECUTION_ENVIRONMENT); this.shouldWaitForPendingSerializationJobs = getOption(RuntimeOptions.WAIT_FOR_PENDING_SERIALIZATION_JOBS_KEY); @@ -508,8 +508,8 @@ public long clockTick() { return clock.getAndIncrement(); } - public IOPermissions getRootIOPermissions() { - return rootIOPermissions; + public ExecutionEnvironment getExecutionEnvironment() { + return executionEnvironment; } public Shape getRootStateShape() { diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/builtin/Builtins.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/builtin/Builtins.java index 24fb08d09a6f..594c499bb642 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/builtin/Builtins.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/builtin/Builtins.java @@ -33,6 +33,7 @@ import org.enso.interpreter.node.expression.builtin.ordering.DefaultComparator; import org.enso.interpreter.node.expression.builtin.ordering.Ordering; import org.enso.interpreter.node.expression.builtin.resource.ManagedResource; +import org.enso.interpreter.node.expression.builtin.runtime.Context; import org.enso.interpreter.node.expression.builtin.text.Text; import org.enso.interpreter.runtime.EnsoContext; import org.enso.interpreter.runtime.Module; @@ -83,6 +84,8 @@ public static class Debug { private final ModuleScope scope; private final Number number; private final Boolean bool; + + private final Context contexts; private final Ordering ordering; private final Comparable comparable; private final DefaultComparator defaultComparator; @@ -137,6 +140,7 @@ public Builtins(EnsoContext context) { system = new System(this); number = new Number(this); bool = this.getBuiltinType(Boolean.class); + contexts = this.getBuiltinType(Context.class); any = builtins.get(Any.class); nothing = builtins.get(Nothing.class); @@ -446,6 +450,12 @@ public Number number() { return number; } + + /** @return the builtin Context type */ + public Context context() { + return contexts; + } + /** @return the container for boolean constructors. */ public Boolean bool() { return bool; diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/builtin/Error.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/builtin/Error.java index c897a32df26a..109a2758e5e6 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/builtin/Error.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/builtin/Error.java @@ -40,6 +40,8 @@ public class Error { private final CaughtPanic caughtPanic; private final ForbiddenOperation forbiddenOperation; + private final Unimplemented unimplemented; + @CompilerDirectives.CompilationFinal private Atom arithmeticErrorShiftTooBig; @CompilerDirectives.CompilationFinal private Atom arithmeticErrorDivideByZero; @@ -72,6 +74,7 @@ public Error(Builtins builtins, EnsoContext context) { panic = builtins.getBuiltinType(Panic.class); caughtPanic = builtins.getBuiltinType(CaughtPanic.class); forbiddenOperation = builtins.getBuiltinType(ForbiddenOperation.class); + unimplemented = builtins.getBuiltinType(Unimplemented.class); } public Atom makeSyntaxError(Object message) { @@ -226,6 +229,10 @@ public ForbiddenOperation getForbiddenOperation() { return forbiddenOperation; } + public Atom makeUnimplemented(String operation) { + return unimplemented.newInstance(operation); + } + public Atom makeNumberParseError(String message) { return numberParseError.newInstance(Text.create(message)); } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/AtomConstructor.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/AtomConstructor.java index 61565bc35055..13886479f87c 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/AtomConstructor.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/AtomConstructor.java @@ -123,7 +123,7 @@ public AtomConstructor initializeFields( cachedInstance = null; } if (Layout.isAritySupported(args.length)) { - boxedLayout = Layout.create(args.length, 0); + boxedLayout = Layout.create(args.length, 0, args); } this.constructorFunction = buildConstructorFunction(language, localScope, assignments, varReads, annotations, args); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/unboxing/Layout.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/unboxing/Layout.java index fe9ecd89dd57..f3ee0397e0c0 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/unboxing/Layout.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/unboxing/Layout.java @@ -6,6 +6,7 @@ import org.enso.interpreter.dsl.atom.LayoutSpec; import org.enso.interpreter.node.expression.atom.InstantiateNode; import org.enso.interpreter.runtime.EnsoContext; +import org.enso.interpreter.runtime.callable.argument.ArgumentDefinition; import org.enso.interpreter.runtime.callable.atom.AtomConstructor; /** @@ -54,6 +55,7 @@ public static int countLongs(long flags) { private final @CompilerDirectives.CompilationFinal(dimensions = 1) UnboxingAtom.FieldGetterNode[] uncachedFieldGetters; + private final @CompilerDirectives.CompilationFinal(dimensions = 1) ArgumentDefinition[] args; private final @CompilerDirectives.CompilationFinal(dimensions = 1) NodeFactory< ? extends UnboxingAtom.FieldSetterNode>[] fieldSetterFactories; @@ -69,16 +71,14 @@ public Layout( int[] fieldToStorage, NodeFactory[] fieldGetterFactories, NodeFactory[] fieldSetterFactories, - NodeFactory instantiatorFactory) { + NodeFactory instantiatorFactory, + ArgumentDefinition[] args) { + this.args = args; this.inputFlags = inputFlags; this.fieldToStorage = fieldToStorage; this.instantiatorFactory = instantiatorFactory; this.fieldGetterFactories = fieldGetterFactories; this.uncachedFieldGetters = new UnboxingAtom.FieldGetterNode[fieldGetterFactories.length]; - for (int i = 0; i < fieldGetterFactories.length; i++) { - this.uncachedFieldGetters[i] = fieldGetterFactories[i].getUncachedInstance(); - assert this.uncachedFieldGetters[i] != null; - } this.fieldSetterFactories = fieldSetterFactories; this.uncachedFieldSetters = new UnboxingAtom.FieldSetterNode[fieldSetterFactories.length]; for (int i = 0; i < fieldSetterFactories.length; i++) { @@ -86,6 +86,15 @@ public Layout( this.uncachedFieldSetters[i] = fieldSetterFactories[i].getUncachedInstance(); } } + for (int i = 0; i < fieldGetterFactories.length; i++) { + this.uncachedFieldGetters[i] = fieldGetterFactories[i].getUncachedInstance(); + assert this.uncachedFieldGetters[i] != null; + if (args[i].isSuspended()) { + this.uncachedFieldGetters[i] = + SuspendedFieldGetterNode.build( + this.uncachedFieldGetters[i], this.uncachedFieldSetters[i]); + } + } } public static boolean isAritySupported(int arity) { @@ -98,7 +107,7 @@ public static boolean isAritySupported(int arity) { * factories for getters, setters and instantiators. */ @SuppressWarnings("unchecked") - public static Layout create(int arity, long typeFlags) { + public static Layout create(int arity, long typeFlags, ArgumentDefinition[] args) { if (arity > 32) { throw new IllegalArgumentException("Too many fields in unboxed atom"); } @@ -137,7 +146,7 @@ public static Layout create(int arity, long typeFlags) { var instantiatorFactory = LayoutFactory.getInstantiatorNodeFactory(numUnboxed, numBoxed); return new Layout( - typeFlags, fieldToStorage, getterFactories, setterFactories, instantiatorFactory); + typeFlags, fieldToStorage, getterFactories, setterFactories, instantiatorFactory, args); } public UnboxingAtom.FieldGetterNode[] getUncachedFieldGetters() { @@ -148,6 +157,10 @@ public UnboxingAtom.FieldGetterNode[] buildGetters() { var getters = new UnboxingAtom.FieldGetterNode[fieldGetterFactories.length]; for (int i = 0; i < fieldGetterFactories.length; i++) { getters[i] = fieldGetterFactories[i].createNode(); + if (args[i].isSuspended()) { + var setterOrNull = buildSetter(i); + getters[i] = SuspendedFieldGetterNode.build(getters[i], setterOrNull); + } } return getters; } @@ -157,7 +170,11 @@ public UnboxingAtom.FieldGetterNode getUncachedFieldGetter(int index) { } public UnboxingAtom.FieldGetterNode buildGetter(int index) { - return fieldGetterFactories[index].createNode(); + var node = fieldGetterFactories[index].createNode(); + if (args[index].isSuspended()) { + node = SuspendedFieldGetterNode.build(node, buildSetter(index)); + } + return node; } public UnboxingAtom.FieldSetterNode getUncachedFieldSetter(int index) { @@ -165,7 +182,8 @@ public UnboxingAtom.FieldSetterNode getUncachedFieldSetter(int index) { } public UnboxingAtom.FieldSetterNode buildSetter(int index) { - return fieldSetterFactories[index].createNode(); + var fieldSetterFactory = fieldSetterFactories[index]; + return fieldSetterFactory == null ? null : fieldSetterFactory.createNode(); } public boolean isDoubleAt(int fieldIndex) { @@ -233,7 +251,7 @@ public Object execute(Object[] arguments) { if (layouts.length == this.unboxedLayouts.length) { // Layouts stored in this node are probably up-to-date; create a new one and try to // register it. - var newLayout = Layout.create(arity, flags); + var newLayout = Layout.create(arity, flags, boxedLayout.layout.args); constructor.atomicallyAddLayout(newLayout, this.unboxedLayouts.length); } updateFromConstructor(); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/unboxing/SuspendedFieldGetterNode.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/unboxing/SuspendedFieldGetterNode.java new file mode 100644 index 000000000000..5760f8515892 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/callable/atom/unboxing/SuspendedFieldGetterNode.java @@ -0,0 +1,65 @@ +package org.enso.interpreter.runtime.callable.atom.unboxing; + +import com.oracle.truffle.api.exception.AbstractTruffleException; +import com.oracle.truffle.api.interop.TruffleObject; +import com.oracle.truffle.api.nodes.Node; +import org.enso.interpreter.node.callable.InvokeCallableNode; +import org.enso.interpreter.node.callable.dispatch.InvokeFunctionNode; +import org.enso.interpreter.runtime.EnsoContext; +import org.enso.interpreter.runtime.callable.argument.CallArgumentInfo; +import org.enso.interpreter.runtime.callable.atom.Atom; +import org.enso.interpreter.runtime.callable.function.Function; +import org.enso.interpreter.runtime.state.State; + +/** + * Getter node that reads a field value. If the value is a thunk the node + * evaluates it and replaces the original lazy value with the new value. + */ +final class SuspendedFieldGetterNode extends UnboxingAtom.FieldGetterNode { + @Node.Child + private UnboxingAtom.FieldSetterNode set; + @Node.Child + private UnboxingAtom.FieldGetterNode get; + @Node.Child + private InvokeFunctionNode invoke = InvokeFunctionNode.build( + new CallArgumentInfo[0], InvokeCallableNode.DefaultsExecutionMode.EXECUTE, InvokeCallableNode.ArgumentsExecutionMode.EXECUTE + ); + + private SuspendedFieldGetterNode(UnboxingAtom.FieldGetterNode get, UnboxingAtom.FieldSetterNode set) { + this.get = get; + this.set = set; + } + + static UnboxingAtom.FieldGetterNode build(UnboxingAtom.FieldGetterNode get, UnboxingAtom.FieldSetterNode set) { + return new SuspendedFieldGetterNode(get, set); + } + + @Override + public Object execute(Atom atom) { + java.lang.Object value = get.execute(atom); + if (value instanceof Function fn && fn.isThunk()) { + try { + org.enso.interpreter.runtime.EnsoContext ctx = EnsoContext.get(this); + java.lang.Object newValue = invoke.execute(fn, null, State.create(ctx), new Object[0]); + set.execute(atom, newValue); + return newValue; + } catch (AbstractTruffleException ex) { + var rethrow = new SuspendedException(ex); + set.execute(atom, rethrow); + throw ex; + } + } else if (value instanceof SuspendedException suspended) { + throw suspended.ex; + } else { + return value; + } + } + + private static final class SuspendedException implements TruffleObject { + final AbstractTruffleException ex; + + SuspendedException(AbstractTruffleException ex) { + this.ex = ex; + } + } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/ExecutionEnvironment.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/ExecutionEnvironment.java new file mode 100644 index 000000000000..59dd9511dcd9 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/ExecutionEnvironment.java @@ -0,0 +1,110 @@ +package org.enso.interpreter.runtime.state; + +import org.enso.interpreter.node.expression.builtin.runtime.Context; +import org.enso.interpreter.runtime.EnsoContext; +import org.enso.interpreter.runtime.callable.atom.Atom; + +public class ExecutionEnvironment { + + private final String name; + + // Ideally we would "just" use a map here. But that leads + // to native image build problems. This in turn leads to + // TruffleBoundary annotations which in turn leads to slow path. + private final String[] keys; + private final Boolean[] permissions; + + public static final String LIVE_ENVIRONMENT_NAME = "live"; + + public static final String DESIGN_ENVIRONMENT_NAME = "design"; + + public static final ExecutionEnvironment LIVE = initLive(LIVE_ENVIRONMENT_NAME); + public static final ExecutionEnvironment DESIGN = + new ExecutionEnvironment(DESIGN_ENVIRONMENT_NAME); + + private static final ExecutionEnvironment initLive(String name) { + String[] keys = new String[] {Context.INPUT_NAME, Context.OUTPUT_NAME}; + Boolean[] permissions = new Boolean[] {true, true}; + return new ExecutionEnvironment(name, keys, permissions); + } + + public ExecutionEnvironment(String name) { + this.name = name; + this.keys = new String[0]; + this.permissions = new Boolean[0]; + } + + private ExecutionEnvironment(String name, String[] keys, Boolean[] permissions) { + this.name = name; + this.keys = keys; + this.permissions = permissions; + } + + public String getName() { + return this.name; + } + + public ExecutionEnvironment withContextEnabled(Atom context) { + return update(context, true); + } + + public ExecutionEnvironment withContextDisabled(Atom context) { + return update(context, false); + } + + private ExecutionEnvironment update(Atom context, boolean value) { + assert context.getType() == EnsoContext.get(null).getBuiltins().context().getType(); + int keyFound = -1; + for (int i = 0; i < keys.length; i++) { + if (keys[i].equals(context.getConstructor().getName())) { + keyFound = i; + } + } + String[] keys1; + Boolean[] permissions1; + if (keyFound != -1) { + keys1 = cloneArray(keys, new String[keys.length]); + permissions1 = cloneArray(permissions, new Boolean[keys.length]); + permissions1[keyFound] = value; + } else { + keys1 = cloneArray(keys, new String[keys.length + 1]); + permissions1 = cloneArray(permissions, new Boolean[keys.length + 1]); + keyFound = keys.length; + keys1[keyFound] = context.getConstructor().getName(); + permissions1[keyFound] = value; + } + return new ExecutionEnvironment(name, keys1, permissions1); + } + + private T[] cloneArray(T[] fromArray, T[] toArray) { + for (int i = 0; i < fromArray.length; i++) { + toArray[i] = fromArray[i]; + } + return toArray; + } + + public Boolean hasContextEnabled(String context) { + int keyFound = -1; + for (int i = 0; i < keys.length; i++) { + if (keys[i].equals(context)) { + keyFound = i; + } + } + if (keyFound != -1) { + return permissions[keyFound]; + } else { + return false; + } + } + + public static ExecutionEnvironment forName(String name) { + switch (name) { + case LIVE_ENVIRONMENT_NAME: + return LIVE; + case DESIGN_ENVIRONMENT_NAME: + return DESIGN; + default: + throw new RuntimeException("Unsupported Execution Environment `" + name + "`"); + } + } +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/IOPermissions.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/IOPermissions.java deleted file mode 100644 index 9fd1faabf8db..000000000000 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/IOPermissions.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.enso.interpreter.runtime.state; - -public class IOPermissions { - private final String name; - private final boolean isInputAllowed; - private final boolean isOutputAllowed; - - private IOPermissions(String name, boolean isInputAllowed, boolean isOutputAllowed) { - this.name = name; - this.isInputAllowed = isInputAllowed; - this.isOutputAllowed = isOutputAllowed; - } - - public static final IOPermissions PRODUCTION = new IOPermissions("production", true, true); - public static final IOPermissions DEVELOPMENT = new IOPermissions("development", false, false); - - public static IOPermissions forName(String name) { - switch (name) { - case "production": - return PRODUCTION; - case "development": - return DEVELOPMENT; - default: - return new IOPermissions(name, false, false); - } - } - - public IOPermissions allowInputIn(String name) { - if (name.equals(this.name) && !isInputAllowed) { - return new IOPermissions(this.name, true, isOutputAllowed); - } - return this; - } - - public IOPermissions allowOutputIn(String name) { - if (name.equals(this.name) && !isOutputAllowed) { - return new IOPermissions(this.name, isInputAllowed, true); - } - return this; - } - - public boolean isInputAllowed() { - return isInputAllowed; - } - - public boolean isOutputAllowed() { - return isOutputAllowed; - } -} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/State.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/State.java index 105a83dd1e1d..382d9b14ede3 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/State.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/state/State.java @@ -2,35 +2,46 @@ import com.oracle.truffle.api.object.DynamicObject; import com.oracle.truffle.api.object.Shape; +import org.enso.interpreter.node.expression.builtin.runtime.Context; import org.enso.interpreter.runtime.EnsoContext; +import org.enso.interpreter.runtime.callable.atom.Atom; public class State { private final Container container; - private final IOPermissions ioPermissions; - public State(Container container, IOPermissions ioPermissions) { + private final ExecutionEnvironment executionEnvironment; + + public State(Container container, ExecutionEnvironment executionEnvironment) { this.container = container; - this.ioPermissions = ioPermissions; + this.executionEnvironment = executionEnvironment; } public Container getContainer() { return container; } - public IOPermissions getIoPermissions() { - return ioPermissions; + public ExecutionEnvironment currentEnvironment() { + return executionEnvironment; } public static State create(EnsoContext context) { - return new State(Container.create(context), context.getRootIOPermissions()); + return new State(Container.create(context), context.getExecutionEnvironment()); } - public State withInputAllowedIn(String name) { - return new State(container, ioPermissions.allowInputIn(name)); + public State withContextEnabledIn(Atom context, String environmentName) { + if (executionEnvironment.getName().equals(environmentName)) { + return new State(container, executionEnvironment.withContextEnabled(context)); + } else { + return this; + } } - public State withOutputAllowedIn(String name) { - return new State(container, ioPermissions.allowOutputIn(name)); + public State withContextDisabledIn(Atom context, String environmentName) { + if (executionEnvironment.getName().equals(environmentName)) { + return new State(container, executionEnvironment.withContextDisabled(context)); + } else { + return this; + } } public static class Container extends DynamicObject { diff --git a/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala b/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala index 291cec18d59d..4dcd2a3f631d 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala @@ -241,6 +241,7 @@ final class SerializationManager( SuggestionBuilder(module) .build(module.getName, module.getIr) .toVector + .filter(Suggestion.isGlobal) } .foreach(suggestions.add) val cachedSuggestions = diff --git a/engine/runtime/src/main/scala/org/enso/compiler/codegen/IrToTruffle.scala b/engine/runtime/src/main/scala/org/enso/compiler/codegen/IrToTruffle.scala index f211a1416524..96157a939bb4 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/codegen/IrToTruffle.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/codegen/IrToTruffle.scala @@ -42,7 +42,6 @@ import org.enso.interpreter.node.callable.{ SequenceLiteralNode } import org.enso.interpreter.node.controlflow.caseexpr._ -import org.enso.interpreter.node.controlflow.permission.PermissionGuardNode import org.enso.interpreter.node.expression.atom.{ ConstantNode, QualifiedAccessorNode @@ -1657,12 +1656,7 @@ class IrToTruffle( case _ => ExpressionProcessor.this.run(body, false, subjectToInstrumentation) } - val block = BlockNode.build(argExpressions.toArray, bodyExpr) - effectContext match { - case Some("Input") => new PermissionGuardNode(block, true, false); - case Some("Output") => new PermissionGuardNode(block, false, true); - case _ => block - } + BlockNode.build(argExpressions.toArray, bodyExpr) } private def computeSlots(): ( diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/AnalyzeModuleInScopeJob.scala b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/AnalyzeModuleInScopeJob.scala index 744c012c0636..232d49031359 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/AnalyzeModuleInScopeJob.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/instrument/job/AnalyzeModuleInScopeJob.scala @@ -46,7 +46,7 @@ final class AnalyzeModuleInScopeJob( val moduleName = module.getName val newSuggestions = SuggestionBuilder(module.getSource.getCharacters) .build(moduleName, module.getIr) - .filter(isSuggestionGlobal) + .filter(Suggestion.isGlobal) val version = ctx.versioning.evalVersion(module.getSource.getCharacters) val prevExports = ModuleExports(moduleName.toString, Set()) val newExports = exportsBuilder.build(module.getName, module.getIr) @@ -63,17 +63,6 @@ final class AnalyzeModuleInScopeJob( } } - private def isSuggestionGlobal(suggestion: Suggestion): Boolean = - suggestion match { - case _: Suggestion.Module => true - case _: Suggestion.Type => true - case _: Suggestion.Constructor => true - case _: Suggestion.Method => true - case _: Suggestion.Conversion => true - case _: Suggestion.Function => false - case _: Suggestion.Local => false - } - /** Send notification about module updates. * * @param payload the module update diff --git a/engine/runtime/src/test/java/org/enso/interpreter/test/LazyAtomFieldTest.java b/engine/runtime/src/test/java/org/enso/interpreter/test/LazyAtomFieldTest.java new file mode 100644 index 000000000000..bc28cd100313 --- /dev/null +++ b/engine/runtime/src/test/java/org/enso/interpreter/test/LazyAtomFieldTest.java @@ -0,0 +1,135 @@ +package org.enso.interpreter.test; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.Map; +import java.util.stream.Collectors; +import org.enso.polyglot.MethodNames; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Language; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.Value; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import org.junit.Before; +import org.junit.Test; + +public class LazyAtomFieldTest { + private static final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private Context ctx; + + @Before + public void prepareCtx() { + this.ctx = Context.newBuilder() + .allowExperimentalOptions(true) + .allowIO(true) + .allowAllAccess(true) + .logHandler(new ByteArrayOutputStream()) + .out(out) + .option( + RuntimeOptions.LANGUAGE_HOME_OVERRIDE, + Paths.get("../../distribution/component").toFile().getAbsolutePath() + ).build(); + final Map langs = ctx.getEngine().getLanguages(); + assertNotNull("Enso found: " + langs, langs.get("enso")); + out.reset(); + } + + @Test + public void evaluation() throws Exception { + final String code = """ + from Standard.Base import IO + + type Lazy + LazyValue ~x ~y + + say self w = "Hello " + w.to_text + + meaning self = + IO.println "Computing meaning" + v = self.x * self.y + IO.println "Computed meaning" + v + + meanings = + compute_x = + IO.println "Computing x" + v = 6 + IO.println "Computing x done" + v + + compute_y = + IO.println "Computing y" + v = 7 + IO.println "Computing y done" + v + + IO.println "Start" + l = Lazy.LazyValue compute_x compute_y + IO.println "Lazy value ready" + IO.println <| l.say "World!" + IO.println l.meaning + IO.println <| l.say "Again!" + IO.println l.meaning + l.meaning + """; + var meanings = evalCode(code, "meanings"); + assertEquals(42, meanings.asInt()); + + String log = out.toString(StandardCharsets.UTF_8); + var lazyReadyAndThen = log.lines().dropWhile(l -> l.contains("Lazy value ready")).collect(Collectors.toList()); + var computingX = lazyReadyAndThen.stream().filter(l -> l.contains("Computing x done")).count(); + assertEquals(log, 1, computingX); + var computingY = lazyReadyAndThen.stream().filter(l -> l.contains("Computing y done")).count(); + assertEquals(log, 1, computingY); + var hellos = lazyReadyAndThen.stream().filter(l -> l.startsWith("Hello")).count(); + assertEquals(log, 2, hellos); + } + + @Test + public void testInfiniteListGenerator() throws Exception { + final String code = """ + import Standard.Base.IO + + type Lazy + Nil + Cons ~x ~xs + + take self n = if n == 0 then Lazy.Nil else case self of + Lazy.Nil -> Lazy.Nil + Lazy.Cons x xs -> Lazy.Cons x (xs.take n-1) + + sum self acc = case self of + Lazy.Nil -> acc + Lazy.Cons x xs -> @Tail_Call xs.sum acc+x + + generator n = Lazy.Cons n (Lazy.generator n+1) + + both n = + g = Lazy.generator 1 + // IO.println "Generator is computed" + t = g.take n + // IO.println "Generator is taken" + t . sum 0 + """; + + var both = evalCode(code, "both"); + var sum = both.execute(100); + String log = out.toString(StandardCharsets.UTF_8); + assertEquals(log, 5050, sum.asLong()); + } + + private Value evalCode(final String code, final String methodName) throws URISyntaxException { + final var testName = "test.enso"; + final URI testUri = new URI("memory://" + testName); + final Source src = Source.newBuilder("enso", code, testName) + .uri(testUri) + .buildLiteral(); + var module = ctx.eval(src); + return module.invokeMember(MethodNames.Module.EVAL_EXPRESSION, methodName); + } +} diff --git a/engine/runtime/src/test/java/org/enso/interpreter/test/ValuesGenerator.java b/engine/runtime/src/test/java/org/enso/interpreter/test/ValuesGenerator.java index 1dc3f5fcdc2d..b4bde2934528 100644 --- a/engine/runtime/src/test/java/org/enso/interpreter/test/ValuesGenerator.java +++ b/engine/runtime/src/test/java/org/enso/interpreter/test/ValuesGenerator.java @@ -721,6 +721,7 @@ public List allValues() throws Exception { public List allTypes() throws Exception { var collect = new ArrayList(); for (var m : getClass().getMethods()) { + if (m.getName().startsWith("type")) { if (m.getReturnType() == Value.class) { var r = (Value) m.invoke(this); diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/DesignExecutionEnvironmentTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/DesignExecutionEnvironmentTest.scala new file mode 100644 index 000000000000..7688a7457e5f --- /dev/null +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/DesignExecutionEnvironmentTest.scala @@ -0,0 +1,117 @@ +package org.enso.interpreter.test.semantic + +import org.enso.interpreter.test.{InterpreterContext, InterpreterTest} +import org.graalvm.polyglot.Context + +class DesignExecutionEnvironmentTest extends InterpreterTest { + + override def contextModifiers: Option[Context#Builder => Context#Builder] = + Some(_.option("enso.ExecutionEnvironment", "design")) + + override def subject: String = "design execution environment" + + override def specify(implicit + interpreterContext: InterpreterContext + ): Unit = { + + "error on Input actions" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled i environment="design" + | + |main = Panic.catch Any (input_action 2) p-> p.payload.to_text + |""".stripMargin + eval(code) shouldEqual "(Forbidden_Operation.Error 'Input')" + } + + "error on invalid context actions" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input,Output + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled i environment="design" + | + |main = Panic.catch Any (Runtime.with_enabled_context Output (input_action 2)) p-> p.payload.to_text + |""".stripMargin + eval(code) shouldEqual "(Forbidden_Operation.Error 'Input')" + } + + "error on invalid environment actions" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled i environment="live" + | + |main = Panic.catch Any (input_action 2) p-> p.payload.to_text + |""".stripMargin + eval( + code + ) shouldEqual "(Unimplemented.Error execution environment mismatch)" + } + + "pass on Input actions" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled i environment="design" + | + |main = Runtime.with_enabled_context Input (input_action 2) + |""".stripMargin + eval(code) shouldEqual 2 + } + + "error on Output actions" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Output + | + |output_action : Integer -> Integer + |output_action i = Output.if_enabled i environment="design" + | + |main = Panic.catch Any (output_action 2) p-> p.payload.to_text + |""".stripMargin + eval(code) shouldEqual "(Forbidden_Operation.Error 'Output')" + } + + "scope of context is limited" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled i + | + |main = + | res = Runtime.with_enabled_context Input (input_action 2) + | Panic.catch Any (input_action 2) p-> res.to_text+" and "+p.payload.to_text + |""".stripMargin + eval(code) shouldEqual "2 and (Forbidden_Operation.Error 'Input')" + } + + "allow locally running IO" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input, Output + | + |output_action : Integer -> Integer + |output_action i = Output.if_enabled (i+1) environment="design" + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled (i*2) environment="design" + | + |main = + | r = Runtime.with_enabled_context Input environment="design" <| Runtime.with_enabled_context Output environment="design" <| output_action <| input_action 123 + | [r].to_text + |""".stripMargin + eval(code) shouldEqual "[247]" + } + } +} diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/IOContextTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/IOContextTest.scala deleted file mode 100644 index 0193eb252917..000000000000 --- a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/IOContextTest.scala +++ /dev/null @@ -1,92 +0,0 @@ -package org.enso.interpreter.test.semantic - -import org.enso.interpreter.test.{InterpreterContext, InterpreterTest} -import org.graalvm.polyglot.Context - -// This test conceivably be written in pure Enso, but it requires a special way -// of spawning the engine, hence it's easier to keep here. -class DevelopmentContextTest extends InterpreterTest { - - override def contextModifiers: Option[Context#Builder => Context#Builder] = - Some(_.option("enso.IOEnvironment", "development")) - - override def subject: String = "development IO Context" - - override def specify(implicit - interpreterContext: InterpreterContext - ): Unit = { - "error on Input actions" in { - val code = - """from Standard.Base import all - |from Standard.Base.Runtime.IO_Permissions import Input - | - |input_action : Integer -> Integer in Input - |input_action i = i - | - |main = Panic.catch Any (input_action 1) p-> p.payload.to_text - |""".stripMargin - eval(code) shouldEqual "(Forbidden_Operation_Data 'Input')" - } - - "error on Output actions" in { - val code = - """from Standard.Base import all - |from Standard.Base.Runtime.IO_Permissions import Output - | - |output_action : Integer -> Nothing in Output - |output_action i = i - | - |main = Panic.catch Any (output_action 1) p-> p.payload.to_text - |""".stripMargin - eval(code) shouldEqual "(Forbidden_Operation_Data 'Output')" - } - - "allow locally running IO" in { - val code = - """from Standard.Base import all - |from Standard.Base.Runtime.IO_Permissions import Input, Output - | - |output_action : Integer -> Integer in Output - |output_action i = i + 1 - | - |input_action : Integer -> Integer in Input - |input_action i = i * 2 - | - |main = - | r_1 = Runtime.allow_input_in "development" <| input_action 123 - | r_2 = Runtime.allow_output_in "development" <| output_action 123 - | r_3 = Runtime.allow_input_in "development" <| Runtime.allow_output_in "development" <| output_action <| input_action 123 - | [r_1, r_2, r_3].to_text - |""".stripMargin - eval(code) shouldEqual "[246, 124, 247]" - } - } -} - -class ProductionContextTest extends InterpreterTest { - override def subject: String = "production IO Context" - - override def specify(implicit - interpreterContext: InterpreterContext - ): Unit = { - "allow all IO" in { - val code = - """from Standard.Base import all - |from Standard.Base.Runtime.IO_Permissions import Input, Output - | - |output_action : Integer -> Integer in Output - |output_action i = i + 1 - | - |input_action : Integer -> Integer in Input - |input_action i = i * 2 - | - |main = - | r_1 = input_action 123 - | r_2 = output_action 123 - | r_3 = output_action <| input_action 123 - | [r_1, r_2, r_3].to_text - |""".stripMargin - eval(code) shouldEqual "[246, 124, 247]" - } - } -} diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/LiveExecutionEnvironmentTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/LiveExecutionEnvironmentTest.scala new file mode 100644 index 000000000000..024568701a53 --- /dev/null +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/semantic/LiveExecutionEnvironmentTest.scala @@ -0,0 +1,102 @@ +package org.enso.interpreter.test.semantic + +import org.enso.interpreter.test.{InterpreterContext, InterpreterTest} +import org.graalvm.polyglot.Context + +class LiveExecutionEnvironmentTest extends InterpreterTest { + + override def contextModifiers: Option[Context#Builder => Context#Builder] = + Some(_.option("enso.ExecutionEnvironment", "live")) + + override def subject: String = "live execution environment" + + override def specify(implicit + interpreterContext: InterpreterContext + ): Unit = { + + "pass on Input actions for live environment" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled i environment="live" + | + |main = input_action 2 + |""".stripMargin + eval(code) shouldEqual 2 + } + + "error on invalid context actions" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input,Output + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled i environment="live" + | + |main = Panic.catch Any (Runtime.with_disabled_context Input (input_action 2)) p-> p.payload.to_text + |""".stripMargin + eval(code) shouldEqual "(Forbidden_Operation.Error 'Input')" + } + + "error on invalid environment actions" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled i environment="design" + | + |main = Panic.catch Any (input_action 2) p-> p.payload.to_text + |""".stripMargin + eval( + code + ) shouldEqual "(Unimplemented.Error execution environment mismatch)" + } + + "pass on Input actions with Context enabled explicitly" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled i environment="live" + | + |main = Runtime.with_enabled_context Input (input_action 2) + |""".stripMargin + eval(code) shouldEqual 2 + } + + "pass on Output actions" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Output + | + |output_action : Integer -> Integer + |output_action i = Output.if_enabled i environment="live" + | + |main = output_action 2 + |""".stripMargin + eval(code) shouldEqual 2 + } + + "allow locally running IO" in { + val code = + """from Standard.Base import all + |from Standard.Base.Runtime.Context import Input, Output + | + |output_action : Integer -> Integer + |output_action i = Output.if_enabled (i+1) environment="live" + | + |input_action : Integer -> Integer + |input_action i = Input.if_enabled (i*2) environment="live" + | + |main = + | r = Runtime.with_enabled_context Input environment="live" <| Runtime.with_enabled_context Output environment="design" <| output_action <| input_action 123 + | [r].to_text + |""".stripMargin + eval(code) shouldEqual "[247]" + } + } +} diff --git a/lib/rust/bitmap/src/lib.rs b/lib/rust/bitmap/src/lib.rs index a179a04e5e9e..5d8a341172d7 100644 --- a/lib/rust/bitmap/src/lib.rs +++ b/lib/rust/bitmap/src/lib.rs @@ -79,9 +79,9 @@ impl Image { const DIMENSIONS: usize = 2; let dimensions = lines.next().ok_or(Error::Truncated)?; let dimensions = std::str::from_utf8(dimensions).map_err(|_| Error::Invalid)?; - let (height, width) = dimensions.split_once(' ').ok_or(Error::Truncated)?; - let height = height.parse().map_err(|_| Error::Invalid)?; + let (width, height) = dimensions.split_once(' ').ok_or(Error::Truncated)?; let width = width.parse().map_err(|_| Error::Invalid)?; + let height = height.parse().map_err(|_| Error::Invalid)?; let num_shades = lines.next().ok_or(Error::Truncated)?; if num_shades != b"255" { return Err(Error::Invalid); diff --git a/lib/rust/ensogl/component/text/src/font/msdf/src/texture.rs b/lib/rust/ensogl/component/text/src/font/msdf/src/texture.rs index 180280f22bac..da0cd7ea52ff 100644 --- a/lib/rust/ensogl/component/text/src/font/msdf/src/texture.rs +++ b/lib/rust/ensogl/component/text/src/font/msdf/src/texture.rs @@ -84,7 +84,13 @@ impl Texture { /// Set the raw pixel data. #[profile(Debug)] pub fn set_data(&self, image: enso_bitmap::Image) { - debug_assert_eq!(image.width, Self::WIDTH); + if image.width != Self::WIDTH { + let expected = Self::WIDTH; + let actual = image.width; + error!( + "Corrupted MSDF texture data. Expected width: {expected}; actual width: {actual}." + ); + } *self.data.borrow_mut() = image.data; } } diff --git a/lib/rust/ensogl/core/src/display/shape/compound/rectangle.rs b/lib/rust/ensogl/core/src/display/shape/compound/rectangle.rs index 38c67ed93f6b..4e3461067b25 100644 --- a/lib/rust/ensogl/core/src/display/shape/compound/rectangle.rs +++ b/lib/rust/ensogl/core/src/display/shape/compound/rectangle.rs @@ -9,10 +9,9 @@ use crate::data::color; use crate::display; - -// =============== -// === Exports === -// =============== +// ============== +// === Export === +// ============== pub use shape::Shape; diff --git a/lib/rust/ensogl/core/src/gui/component.rs b/lib/rust/ensogl/core/src/gui/component.rs index ff546fcb2f18..e952430fd22c 100644 --- a/lib/rust/ensogl/core/src/gui/component.rs +++ b/lib/rust/ensogl/core/src/gui/component.rs @@ -18,7 +18,6 @@ use crate::display::Sprite; use crate::frp; - // ============== // === Export === // ============== diff --git a/lib/rust/ensogl/examples/instance-ordering/src/lib.rs b/lib/rust/ensogl/examples/instance-ordering/src/lib.rs index efd7e75df43c..271b271c0394 100644 --- a/lib/rust/ensogl/examples/instance-ordering/src/lib.rs +++ b/lib/rust/ensogl/examples/instance-ordering/src/lib.rs @@ -6,11 +6,11 @@ #![allow(clippy::bool_to_int_with_if)] #![allow(clippy::let_and_return)] -use ensogl_core::display; use ensogl_core::display::world::*; use ensogl_core::prelude::*; use ensogl_core::data::color; +use ensogl_core::display; use ensogl_core::display::navigation::navigator::Navigator; use ensogl_core::display::object::ObjectOps; use ensogl_core::display::shape::compound::rectangle; diff --git a/lib/scala/common-polyglot-core-utils/src/main/java/org/enso/polyglot/common_utils/Core_Text_Utils.java b/lib/scala/common-polyglot-core-utils/src/main/java/org/enso/polyglot/common_utils/Core_Text_Utils.java index f2a692d84f6e..f1befe6264ca 100644 --- a/lib/scala/common-polyglot-core-utils/src/main/java/org/enso/polyglot/common_utils/Core_Text_Utils.java +++ b/lib/scala/common-polyglot-core-utils/src/main/java/org/enso/polyglot/common_utils/Core_Text_Utils.java @@ -3,6 +3,9 @@ import com.ibm.icu.text.BreakIterator; public class Core_Text_Utils { + private Core_Text_Utils() { + } + /** Computes the length of the string as the number of grapheme clusters it contains. */ public static int computeGraphemeLength(String text) { BreakIterator iter = BreakIterator.getCharacterInstance(); @@ -14,6 +17,17 @@ public static int computeGraphemeLength(String text) { return len; } + /** Returns a prefix of the string not exceeding the provided grapheme length. */ + public static String take_prefix(String str, long grapheme_length) { + BreakIterator iter = BreakIterator.getCharacterInstance(); + iter.setText(str); + if (iter.next(Math.toIntExact(grapheme_length)) == BreakIterator.DONE) { + return str; + } else { + return str.substring(0, iter.current()); + } + } + /** Pretty prints the string, escaping special characters. */ public static String prettyPrint(String str) { int len = str.length(); diff --git a/lib/scala/searcher/src/bench/java/org/enso/searcher/sql/SuggestionsRepoBenchmark.java b/lib/scala/searcher/src/bench/java/org/enso/searcher/sql/SuggestionsRepoBenchmark.java index e347171fbeb0..ecb4b0a50852 100644 --- a/lib/scala/searcher/src/bench/java/org/enso/searcher/sql/SuggestionsRepoBenchmark.java +++ b/lib/scala/searcher/src/bench/java/org/enso/searcher/sql/SuggestionsRepoBenchmark.java @@ -34,8 +34,6 @@ public class SuggestionsRepoBenchmark { final Path dbfile = Path.of(System.getProperty("java.io.tmpdir"), "bench-suggestions.db"); final Seq kinds = SuggestionRandom.nextKinds(); final Seq> updateInput = SuggestionRandom.nextUpdateAllInput(); - final Seq> getAllMethodsInput = - SuggestionRandom.nextGetAllMethodsInput(); SqlSuggestionsRepo repo; @@ -64,7 +62,7 @@ public void tearDown() { int insertBatch(int size) throws TimeoutException, InterruptedException { Suggestion[] stubs = Stream.generate(SuggestionRandom::nextSuggestion).limit(size).toArray(Suggestion[]::new); - return (int) Await.result(repo.insertBatch(stubs), TIMEOUT); + return (int) Await.result(repo.insertBatchJava(stubs), TIMEOUT); } static scala.Option none() { @@ -142,11 +140,6 @@ public Object searchByAll() throws TimeoutException, InterruptedException { TIMEOUT); } - @Benchmark - public Object getAllMethods() throws TimeoutException, InterruptedException { - return Await.result(repo.getAllMethods(getAllMethodsInput), TIMEOUT); - } - @Benchmark public Object updateByExternalId() throws TimeoutException, InterruptedException { return Await.result(repo.updateAll(updateInput), TIMEOUT); diff --git a/lib/scala/searcher/src/bench/scala/org/enso/searcher/sql/SuggestionRandom.scala b/lib/scala/searcher/src/bench/scala/org/enso/searcher/sql/SuggestionRandom.scala index 4a6afaee2317..64834002aa4f 100644 --- a/lib/scala/searcher/src/bench/scala/org/enso/searcher/sql/SuggestionRandom.scala +++ b/lib/scala/searcher/src/bench/scala/org/enso/searcher/sql/SuggestionRandom.scala @@ -11,12 +11,6 @@ object SuggestionRandom { def nextUpdateAllInput(): Seq[(UUID, String)] = Seq(UUID.randomUUID() -> nextString()) - def nextGetAllMethodsInput(): Seq[(String, String, String)] = - Seq( - (nextString(), nextString(), nextString()), - (nextString(), nextString(), nextString()) - ) - def nextKinds(): Seq[Suggestion.Kind] = Set.fill(1)(nextKind()).toSeq diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/SuggestionsRepo.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/SuggestionsRepo.scala index f1c1129e358a..7192b80a67b6 100644 --- a/lib/scala/searcher/src/main/scala/org/enso/searcher/SuggestionsRepo.scala +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/SuggestionsRepo.scala @@ -3,7 +3,6 @@ package org.enso.searcher import org.enso.polyglot.Suggestion import org.enso.polyglot.runtime.Runtime.Api.{ ExportsUpdate, - SuggestionArgumentAction, SuggestionUpdate, SuggestionsDatabaseAction } @@ -24,19 +23,6 @@ trait SuggestionsRepo[F[_]] { */ def getAll: F[(Long, Seq[SuggestionEntry])] - /** Get suggestions by the method call info. - * - * @param calls the list of triples: module, self type and method name - * @return the list of found suggestion ids - */ - def getAllMethods(calls: Seq[(String, String, String)]): F[Seq[Option[Long]]] - - /** Get all available modules. - * - * @return the list of distinct module names. - */ - def getAllModules: F[Seq[String]] - /** Search suggestion by various parameters. * * @param module the module name search parameter @@ -65,19 +51,19 @@ trait SuggestionsRepo[F[_]] { */ def select(id: Long): F[Option[Suggestion]] - /** Insert the suggestion + /** Insert the suggestion. * * @param suggestion the suggestion to insert * @return the id of an inserted suggestion */ def insert(suggestion: Suggestion): F[Option[Long]] - /** Insert a list of suggestions + /** Insert a list of suggestions. * * @param suggestions the suggestions to insert * @return the current database version and a list of inserted suggestion ids */ - def insertAll(suggestions: Seq[Suggestion]): F[(Long, Seq[Option[Long]])] + def insertAll(suggestions: Seq[Suggestion]): F[(Long, Seq[Long])] /** Apply suggestion updates. * @@ -120,18 +106,10 @@ trait SuggestionsRepo[F[_]] { */ def removeModules(modules: Seq[String]): F[(Long, Seq[Long])] - /** Remove a list of suggestions. - * - * @param suggestions the suggestions to remove - * @return the current database version and a list of removed suggestion ids - */ - def removeAll(suggestions: Seq[Suggestion]): F[(Long, Seq[Option[Long]])] - /** Update the suggestion. * * @param suggestion the key suggestion * @param externalId the external id to update - * @param arguments the arguments to update * @param returnType the return type to update * @param documentation the documentation string to update * @param scope the scope to update @@ -139,7 +117,6 @@ trait SuggestionsRepo[F[_]] { def update( suggestion: Suggestion, externalId: Option[Option[Suggestion.ExternalId]], - arguments: Option[Seq[SuggestionArgumentAction]], returnType: Option[String], documentation: Option[Option[String]], scope: Option[Suggestion.Scope], diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala index 4b09b91087cc..28dc737d7c7a 100644 --- a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/SqlSuggestionsRepo.scala @@ -1,13 +1,11 @@ package org.enso.searcher.sql import java.util.UUID - import org.enso.polyglot.{ExportedSymbol, Suggestion} import org.enso.polyglot.runtime.Runtime.Api.{ ExportsAction, ExportsUpdate, SuggestionAction, - SuggestionArgumentAction, SuggestionUpdate, SuggestionsDatabaseAction } @@ -17,7 +15,7 @@ import slick.jdbc.SQLiteProfile.api._ import slick.jdbc.meta.MTable import slick.relational.RelationalProfile -import scala.collection.immutable.HashMap +import scala.collection.immutable.{HashMap, ListMap} import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} @@ -26,18 +24,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit ec: ExecutionContext ) extends SuggestionsRepo[Future] { - /** The query returning the arguments joined with the corresponding - * suggestions. - */ - private val joined: Query[ - (Rep[Option[ArgumentsTable]], SuggestionsTable), - (Option[ArgumentRow], SuggestionRow), - Seq - ] = - Arguments - .joinRight(Suggestions) - .on(_.suggestionId === _.id) - /** Initialize the repo. */ override def init: Future[Unit] = db.run(initQuery) @@ -50,15 +36,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit override def getAll: Future[(Long, Seq[SuggestionEntry])] = db.run(getAllQuery) - /** @inheritdoc */ - override def getAllMethods( - calls: Seq[(String, String, String)] - ): Future[Seq[Option[Long]]] = - db.run(getAllMethodsQuery(calls)) - - override def getAllModules: Future[Seq[String]] = - db.run(getAllModulesQuery.transactionally) - /** @inheritdoc */ override def search( module: Option[String], @@ -81,8 +58,8 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit /** @inheritdoc */ override def insertAll( suggestions: Seq[Suggestion] - ): Future[(Long, Seq[Option[Long]])] = - db.run(insertAllQuery(suggestions).transactionally) + ): Future[(Long, Seq[Long])] = + db.run(insertAllWithVersionQuery(suggestions).transactionally) /** @inheritdoc */ override def applyTree( @@ -110,17 +87,10 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit override def removeModules(modules: Seq[String]): Future[(Long, Seq[Long])] = db.run(removeByModuleQuery(modules)) - /** @inheritdoc */ - override def removeAll( - suggestions: Seq[Suggestion] - ): Future[(Long, Seq[Option[Long]])] = - db.run(removeAllQuery(suggestions).transactionally) - /** @inheritdoc */ override def update( suggestion: Suggestion, externalId: Option[Option[Suggestion.ExternalId]], - arguments: Option[Seq[SuggestionArgumentAction]], returnType: Option[String], documentation: Option[Option[String]], scope: Option[Suggestion.Scope], @@ -130,7 +100,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit updateQuery( suggestion, externalId, - arguments, returnType, documentation, scope, @@ -171,13 +140,11 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit def clearSchemaVersion: Future[Unit] = db.run(clearSchemaVersionQuery) - /** Insert suggestions in a batch. - * - * @param suggestions the list of suggestions to insert - * @return the current database size - */ - private[sql] def insertBatch(suggestions: Array[Suggestion]): Future[Int] = - db.run(insertBatchQuery(suggestions).transactionally) + def insertBatchJava(suggestions: Array[Suggestion]): Future[Int] = + db.run(insertBatchJavaQuery(suggestions).transactionally) + + def selectAllSuggestions: Future[Seq[SuggestionEntry]] = + db.run(selectAllSuggestionsQuery.transactionally) /** The query to initialize the repo. */ private def initQuery: DBIO[Unit] = { @@ -195,7 +162,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit } yield () val tables: Seq[TableQuery[RelationalTable[_]]] = - Seq(Suggestions, Arguments, SuggestionsVersion, SchemaVersion) + Seq(Suggestions, SuggestionsVersion, SchemaVersion) .asInstanceOf[Seq[TableQuery[RelationalTable[_]]]] val initSchemas = for { @@ -214,7 +181,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit private def cleanQuery: DBIO[Unit] = { for { _ <- Suggestions.delete - _ <- Arguments.delete _ <- SuggestionsVersion.delete } yield () } @@ -224,47 +190,10 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit * @return the current database version with the list of suggestion entries */ private def getAllQuery: DBIO[(Long, Seq[SuggestionEntry])] = { - val query = for { - suggestions <- joined.result.map(joinedToSuggestionEntries) - version <- currentVersionQuery - } yield (version, suggestions) - query - } - - /** The query to get the suggestions by the method call info. - * - * @param calls the triples containing module name, self type, method name - * @return the list of found suggestion ids - */ - def getAllMethodsQuery( - calls: Seq[(String, String, String)] - ): DBIO[Seq[Option[Long]]] = - if (calls.isEmpty) { - DBIO.successful(Seq()) - } else { - val query = Suggestions - .filter { row => - calls - .map { case (module, selfType, name) => - row.module === module && row.selfType === selfType && row.name === name - } - .reduce(_ || _) - } - .map(row => (row.id, row.module, row.selfType, row.name)) - query.result.map { tuples => - val result = tuples.map { case (id, module, selfType, name) => - (module, selfType, name) -> id - }.toMap - calls.map(result.get) - } - } - - /** The query to get all module names. - * - * @return the list of distinct module names. - */ - def getAllModulesQuery: DBIO[Seq[String]] = { - Suggestions.map(_.module).distinct.result + for { + rows <- Suggestions.result + version <- currentVersionQuery + } yield (version, rows.map(toSuggestionEntry)) } /** The query to search suggestion by various parameters. @@ -330,11 +259,9 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit * @return return the suggestion */ private def selectQuery(id: Long): DBIO[Option[Suggestion]] = { - val query = for { - (argument, suggestion) <- joined - if suggestion.id === id - } yield (argument, suggestion) - query.result.map(coll => joinedToSuggestions(coll).headOption) + for { + rows <- Suggestions.filter(_.id === id).result + } yield rows.headOption.map(toSuggestion) } /** The query to insert the suggestion @@ -343,13 +270,10 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit * @return the id of an inserted suggestion */ private def insertQuery(suggestion: Suggestion): DBIO[Option[Long]] = { - val (suggestionRow, args) = toSuggestionRow(suggestion) + val suggestionRow = toSuggestionRow(suggestion) val query = for { id <- Suggestions.returning(Suggestions.map(_.id)) += suggestionRow - _ <- Arguments ++= args.zipWithIndex.map { case (argument, ix) => - toArgumentRow(id, ix, argument) - } - _ <- incrementVersionQuery + _ <- incrementVersionQuery } yield id query.asTry.map { case Failure(_) => None @@ -357,21 +281,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit } } - /** The query to insert a list of suggestions - * - * @param suggestions the suggestions to insert - * @return the list of inserted suggestion ids - */ - private def insertAllQuery( - suggestions: Seq[Suggestion] - ): DBIO[(Long, Seq[Option[Long]])] = { - val query = for { - ids <- DBIO.sequence(suggestions.map(insertQuery)) - version <- currentVersionQuery - } yield (version, ids) - query - } - /** The query to apply the suggestion updates. * * @param tree the sequence of updates @@ -395,15 +304,25 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit scope, reexport ) => - updateSuggestionQuery( - suggestion, - extId, - args, - returnType, - doc, - scope, - reexport - ) + if ( + extId.isDefined || + args.isDefined || + returnType.isDefined || + doc.isDefined || + scope.isDefined || + reexport.isDefined + ) { + updateSuggestionQuery( + suggestion, + extId, + returnType, + doc, + scope, + reexport + ) + } else { + DBIO.successful(None) + } } query.map(rs => QueryResult(rs.toSeq, update)) } @@ -516,7 +435,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit * @return the id of removed suggestion */ private def removeQuery(suggestion: Suggestion): DBIO[Option[Long]] = { - val (raw, _) = toSuggestionRow(suggestion) + val raw = toSuggestionRow(suggestion) val selectQuery = selectSuggestionQuery(raw) val deleteQuery = for { rows <- selectQuery.result @@ -557,21 +476,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit } yield rows } - /** The query to remove a list of suggestions. - * - * @param suggestions the suggestions to remove - * @return the list of removed suggestion ids - */ - private def removeAllQuery( - suggestions: Seq[Suggestion] - ): DBIO[(Long, Seq[Option[Long]])] = { - val query = for { - ids <- DBIO.sequence(suggestions.map(removeQuery)) - version <- currentVersionQuery - } yield (version, ids) - query - } - /** The query to update a suggestion. * * @param externalId the external id of a suggestion @@ -597,7 +501,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit * * @param suggestion the key suggestion * @param externalId the external id to update - * @param arguments the arguments to update * @param returnType the return type to update * @param documentation the documentation string to update * @param scope the scope to update @@ -605,7 +508,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit private def updateQuery( suggestion: Suggestion, externalId: Option[Option[Suggestion.ExternalId]], - arguments: Option[Seq[SuggestionArgumentAction]], returnType: Option[String], documentation: Option[Option[String]], scope: Option[Suggestion.Scope], @@ -615,7 +517,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit idOpt <- updateSuggestionQuery( suggestion, externalId, - arguments, returnType, documentation, scope, @@ -628,7 +529,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit * * @param suggestion the key suggestion * @param externalId the external id to update - * @param arguments the arguments to update * @param returnType the return type to update * @param documentation the documentation string to update * @param scope the scope to update @@ -636,14 +536,13 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit private def updateSuggestionQuery( suggestion: Suggestion, externalId: Option[Option[Suggestion.ExternalId]], - arguments: Option[Seq[SuggestionArgumentAction]], returnType: Option[String], documentation: Option[Option[String]], scope: Option[Suggestion.Scope], reexport: Option[Option[String]] ): DBIO[Option[Long]] = { - val (raw, _) = toSuggestionRow(suggestion) - val query = selectSuggestionQuery(raw) + val raw = toSuggestionRow(suggestion) + val query = selectSuggestionQuery(raw) val updateQ = for { r1 <- DBIO.sequenceOption { @@ -681,76 +580,16 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit } } r5 <- DBIO.sequenceOption { - arguments.map { args => - def updateArgs(suggestionId: Long): DBIO[Seq[Int]] = - DBIO.sequence( - args.map(updateSuggestionArgumentQuery(suggestionId, _)) - ) - for { - idOpt <- query.map(_.id).result.headOption - r <- DBIO.sequenceOption(idOpt.map(updateArgs)) - } yield r.map(_.sum) - } - } - r6 <- DBIO.sequenceOption { reexport.map { reexportOpt => query.map(_.reexport).update(reexportOpt) } } - } yield (r1 ++ r2 ++ r3 ++ r4 ++ r5.flatten ++ r6).sum + } yield (r1 ++ r2 ++ r3 ++ r4 ++ r5).sum for { id <- query.map(_.id).result.headOption n <- updateQ _ <- if (n > 0) incrementVersionQuery else DBIO.successful(()) - } yield if (n > 0) id else None - } - - private def updateSuggestionArgumentQuery( - suggestionId: Long, - action: SuggestionArgumentAction - ): DBIO[Int] = { - val argsQuery = Arguments.filter(_.suggestionId === suggestionId) - action match { - case SuggestionArgumentAction.Add(index, argument) => - for { - _ <- argsQuery.filter(_.index === index).delete - n <- Arguments += toArgumentRow(suggestionId, index, argument) - } yield n - case SuggestionArgumentAction.Remove(index) => - for { - n <- argsQuery.filter(_.index === index).delete - } yield n - case SuggestionArgumentAction.Modify( - index, - nameOpt, - tpeOpt, - suspendedOpt, - defaultOpt, - valueOpt - ) => - val argQuery = argsQuery.filter(_.index === index) - for { - r1 <- DBIO.sequenceOption { - nameOpt.map(name => argQuery.map(_.name).update(name)) - } - r2 <- DBIO.sequenceOption { - tpeOpt.map(tpe => argQuery.map(_.tpe).update(tpe)) - } - r3 <- DBIO.sequenceOption { - suspendedOpt.map(suspended => - argQuery.map(_.isSuspended).update(suspended) - ) - } - r4 <- DBIO.sequenceOption { - defaultOpt.map(default => - argQuery.map(_.hasDefault).update(default) - ) - } - r5 <- DBIO.sequenceOption { - valueOpt.map(value => argQuery.map(_.defaultValue).update(value)) - } - } yield Seq(r1, r2, r3, r4, r5).flatten.sum - } + } yield id } /** The query to update a list of suggestions by external id. @@ -781,13 +620,12 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit /** The query to increment the current version of the repo. */ private def incrementVersionQuery: DBIO[Long] = { - val incrementQuery = for { + for { version <- SuggestionsVersion.returning( SuggestionsVersion.map(_.id) ) += SuggestionsVersionRow(None) _ <- SuggestionsVersion.filterNot(_.id === version).delete } yield version - incrementQuery } /** The query to get current version of the repo. */ @@ -826,16 +664,52 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit * @param suggestions the list of suggestions to insert * @return the current size of the database */ - private def insertBatchQuery( - suggestions: Array[Suggestion] + private def insertBatchJavaQuery( + suggestions: Iterable[Suggestion] ): DBIO[Int] = { val rows = suggestions.map(toSuggestionRow) for { - _ <- (Suggestions ++= rows.map(_._1)).asTry + _ <- (Suggestions ++= rows).asTry size <- Suggestions.length.result } yield size } + /** The query to insert suggestions in a batch. + * + * @param suggestions the list of suggestions to insert + * @return the current size of the database + */ + private def insertAllQuery( + suggestions: Iterable[Suggestion] + ): DBIO[Seq[Long]] = { + val suggestionsMap = + suggestions.map(s => SuggestionRowUniqueIndex(s) -> s).to(ListMap) + val rows = suggestions.map(toSuggestionRow) + for { + _ <- Suggestions ++= rows + rows <- Suggestions.result + } yield { + val rowsMap = rows.map(r => SuggestionRowUniqueIndex(r) -> r.id.get).toMap + suggestionsMap.keys.map(rowsMap(_)).toSeq + } + } + + private def insertAllWithVersionQuery( + suggestions: Iterable[Suggestion] + ): DBIO[(Long, Seq[Long])] = { + for { + ids <- insertAllQuery(suggestions) + version <- incrementVersionQuery + } yield (version, ids) + } + + private def selectAllSuggestionsQuery: DBIO[Seq[SuggestionEntry]] = + for { + rows <- Suggestions.result + } yield { + rows.map(row => SuggestionEntry(row.id.get, toSuggestion(row))) + } + /** Create a search query by the provided parameters. * * Even if the module is specified, the response includes all available @@ -882,41 +756,11 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit } } - /** Convert the rows of suggestions joined with arguments to a list of - * suggestions. - */ - private def joinedToSuggestions( - coll: Seq[(Option[ArgumentRow], SuggestionRow)] - ): Seq[Suggestion] = { - coll - .groupBy(_._2) - .view - .mapValues(_.flatMap(_._1)) - .map(Function.tupled(toSuggestion)) - .toSeq - } - - /** Convert the rows of suggestions joined with arguments to a list of - * suggestion entries. - */ - private def joinedToSuggestionEntries( - coll: Seq[(Option[ArgumentRow], SuggestionRow)] - ): Seq[SuggestionEntry] = { - coll - .groupBy(_._2) - .view - .mapValues(_.flatMap(_._1)) - .map(Function.tupled(toSuggestionEntry)) - .toSeq - } - /** Convert the suggestion to a row in the suggestions table. */ - private def toSuggestionRow( - suggestion: Suggestion - ): (SuggestionRow, Seq[Suggestion.Argument]) = + private def toSuggestionRow(suggestion: Suggestion): SuggestionRow = suggestion match { case Suggestion.Module(module, doc, reexport) => - val row = SuggestionRow( + SuggestionRow( id = None, externalIdLeast = None, externalIdMost = None, @@ -934,18 +778,17 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit documentation = doc, reexport = reexport ) - row -> Seq() case Suggestion.Type( expr, module, name, - params, + _, returnType, parentType, doc, reexport ) => - val row = SuggestionRow( + SuggestionRow( id = None, externalIdLeast = expr.map(_.getLeastSignificantBits), externalIdMost = expr.map(_.getMostSignificantBits), @@ -963,17 +806,16 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit scopeEndOffset = ScopeColumn.EMPTY, reexport = reexport ) - row -> params case Suggestion.Constructor( expr, module, name, - args, + _, returnType, doc, reexport ) => - val row = SuggestionRow( + SuggestionRow( id = None, externalIdLeast = expr.map(_.getLeastSignificantBits), externalIdMost = expr.map(_.getMostSignificantBits), @@ -991,19 +833,18 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit scopeEndOffset = ScopeColumn.EMPTY, reexport = reexport ) - row -> args case Suggestion.Method( expr, module, name, - args, + _, selfType, returnType, isStatic, doc, reexport ) => - val row = SuggestionRow( + SuggestionRow( id = None, externalIdLeast = expr.map(_.getLeastSignificantBits), externalIdMost = expr.map(_.getMostSignificantBits), @@ -1021,31 +862,23 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit scopeEndOffset = ScopeColumn.EMPTY, reexport = reexport ) - row -> args case Suggestion.Conversion( expr, module, - args, + _, sourceType, returnType, doc, reexport ) => - val firstArg = Suggestion.Argument( - Suggestion.Kind.Conversion.From, - sourceType, - false, - false, - None - ) - val row = SuggestionRow( + SuggestionRow( id = None, externalIdLeast = expr.map(_.getLeastSignificantBits), externalIdMost = expr.map(_.getMostSignificantBits), kind = SuggestionKind.CONVERSION, module = module, - name = toConversionMethodName(sourceType, returnType), - selfType = SelfTypeColumn.EMPTY, + name = NameColumn.conversionMethodName(sourceType, returnType), + selfType = sourceType, returnType = returnType, parentType = None, isStatic = false, @@ -1056,17 +889,16 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit scopeEndOffset = ScopeColumn.EMPTY, reexport = reexport ) - row -> (firstArg +: args) case Suggestion.Function( expr, module, name, - args, + _, returnType, scope, doc ) => - val row = SuggestionRow( + SuggestionRow( id = None, externalIdLeast = expr.map(_.getLeastSignificantBits), externalIdMost = expr.map(_.getMostSignificantBits), @@ -1084,9 +916,8 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit scopeEndOffset = scope.end.character, reexport = None ) - row -> args case Suggestion.Local(expr, module, name, returnType, scope, doc) => - val row = SuggestionRow( + SuggestionRow( id = None, externalIdLeast = expr.map(_.getLeastSignificantBits), externalIdMost = expr.map(_.getMostSignificantBits), @@ -1104,45 +935,14 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit scopeEndOffset = scope.end.character, reexport = None ) - row -> Seq() } - /** Create the method name for conversion */ - private def toConversionMethodName( - sourceType: String, - returnType: String - ): String = - s"${Suggestion.Kind.Conversion.From}_${sourceType}_${returnType}" - - /** Convert the argument to a row in the arguments table. */ - private def toArgumentRow( - suggestionId: Long, - index: Int, - argument: Suggestion.Argument - ): ArgumentRow = - ArgumentRow( - id = None, - suggestionId = suggestionId, - index = index, - name = argument.name, - tpe = argument.reprType, - isSuspended = argument.isSuspended, - hasDefault = argument.hasDefault, - defaultValue = argument.defaultValue - ) - /** Convert the database rows to a suggestion entry. */ - private def toSuggestionEntry( - suggestion: SuggestionRow, - arguments: Seq[ArgumentRow] - ): SuggestionEntry = - SuggestionEntry(suggestion.id.get, toSuggestion(suggestion, arguments)) - - /** Convert the databaes rows to a suggestion. */ - private def toSuggestion( - suggestion: SuggestionRow, - arguments: Seq[ArgumentRow] - ): Suggestion = + private def toSuggestionEntry(suggestion: SuggestionRow): SuggestionEntry = + SuggestionEntry(suggestion.id.get, toSuggestion(suggestion)) + + /** Convert the database rows to a suggestion. */ + private def toSuggestion(suggestion: SuggestionRow): Suggestion = suggestion.kind match { case SuggestionKind.MODULE => Suggestion.Module( @@ -1156,7 +956,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit toUUID(suggestion.externalIdLeast, suggestion.externalIdMost), module = suggestion.module, name = suggestion.name, - params = arguments.sortBy(_.index).map(toArgument), + params = Seq(), returnType = suggestion.returnType, parentType = suggestion.parentType, documentation = suggestion.documentation, @@ -1168,7 +968,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit toUUID(suggestion.externalIdLeast, suggestion.externalIdMost), module = suggestion.module, name = suggestion.name, - arguments = arguments.sortBy(_.index).map(toArgument), + arguments = Seq(), returnType = suggestion.returnType, documentation = suggestion.documentation, reexport = suggestion.reexport @@ -1179,7 +979,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit toUUID(suggestion.externalIdLeast, suggestion.externalIdMost), module = suggestion.module, name = suggestion.name, - arguments = arguments.sortBy(_.index).map(toArgument), + arguments = Seq(), selfType = suggestion.selfType, returnType = suggestion.returnType, isStatic = suggestion.isStatic, @@ -1191,8 +991,8 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit externalId = toUUID(suggestion.externalIdLeast, suggestion.externalIdMost), module = suggestion.module, - arguments = arguments.sortBy(_.index).tail.map(toArgument), - sourceType = arguments.minBy(_.index).tpe, + arguments = Seq(), + sourceType = suggestion.selfType, returnType = suggestion.returnType, documentation = suggestion.documentation, reexport = suggestion.reexport @@ -1203,7 +1003,7 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit toUUID(suggestion.externalIdLeast, suggestion.externalIdMost), module = suggestion.module, name = suggestion.name, - arguments = arguments.sortBy(_.index).map(toArgument), + arguments = Seq(), returnType = suggestion.returnType, scope = Suggestion.Scope( Suggestion.Position( @@ -1240,16 +1040,6 @@ final class SqlSuggestionsRepo(val db: SqlDatabase)(implicit throw new NoSuchElementException(s"Unknown suggestion kind: $k") } - /** Convert the database row to the suggestion argument. */ - private def toArgument(row: ArgumentRow): Suggestion.Argument = - Suggestion.Argument( - name = row.name, - reprType = row.tpe, - isSuspended = row.isSuspended, - hasDefault = row.hasDefault, - defaultValue = row.defaultValue - ) - /** Convert bits to the UUID. * * @param least the least significant bits of the UUID diff --git a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/Tables.scala b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/Tables.scala index ed1c6b362242..f7af57726e33 100644 --- a/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/Tables.scala +++ b/lib/scala/searcher/src/main/scala/org/enso/searcher/sql/Tables.scala @@ -5,28 +5,6 @@ import slick.jdbc.SQLiteProfile.api._ import scala.annotation.nowarn -/** A row in the arguments table. - * - * @param id the id of an argument - * @param suggestionId the id of the suggestion - * @param index the argument position in the arguments list - * @param name the argument name - * @param tpe the argument type - * @param isSuspended is the argument lazy - * @param hasDefault does the argument have the default value - * @param defaultValue optional default value - */ -case class ArgumentRow( - id: Option[Long], - suggestionId: Long, - index: Int, - name: String, - tpe: String, - isSuspended: Boolean, - hasDefault: Boolean, - defaultValue: Option[String] -) - /** A row in the suggestions table. * * @param id the id of a suggestion @@ -109,6 +87,17 @@ object SuggestionKind { case Suggestion.Kind.Function => FUNCTION case Suggestion.Kind.Local => LOCAL } + + def toSuggestion(kind: Byte): Suggestion.Kind = + kind match { + case MODULE => Suggestion.Kind.Module + case TYPE => Suggestion.Kind.Type + case CONSTRUCTOR => Suggestion.Kind.Constructor + case METHOD => Suggestion.Kind.Method + case FUNCTION => Suggestion.Kind.Function + case LOCAL => Suggestion.Kind.Local + case CONVERSION => Suggestion.Kind.Conversion + } } object ScopeColumn { @@ -123,38 +112,15 @@ object SelfTypeColumn { val EMPTY: String = "\u0500" } -/** The schema of the arguments table. */ -@nowarn("msg=multiarg infix syntax") -final class ArgumentsTable(tag: Tag) - extends Table[ArgumentRow](tag, "arguments") { - - def id = column[Long]("id", O.PrimaryKey, O.AutoInc) - def suggestionId = column[Long]("suggestion_id") - def index = column[Int]("index") - def name = column[String]("name") - def tpe = column[String]("type") - def isSuspended = column[Boolean]("is_suspended", O.Default(false)) - def hasDefault = column[Boolean]("has_default", O.Default(false)) - def defaultValue = column[Option[String]]("default_value") - def * = - ( - id.?, - suggestionId, - index, - name, - tpe, - isSuspended, - hasDefault, - defaultValue - ) <> - (ArgumentRow.tupled, ArgumentRow.unapply) +object NameColumn { + + /** Create the method name for conversion */ + def conversionMethodName( + sourceType: String, + returnType: String + ): String = + s"${Suggestion.Kind.Conversion.From}_${sourceType}_${returnType}" - def suggestion = - foreignKey("suggestion_fk", suggestionId, Suggestions)( - _.id, - onUpdate = ForeignKeyAction.Restrict, - onDelete = ForeignKeyAction.Cascade - ) } /** The schema of the suggestions table. */ @@ -230,6 +196,75 @@ final class SuggestionsTable(tag: Tag) ) } +/** An element of unique suggestion index. + * + * @param kind the type of a suggestion + * @param module the module name + * @param name the suggestion name + * @param selfType the self type of a suggestion + * @param scopeStartLine the line of the start position of the scope + * @param scopeStartOffset the offset of the start position of the scope + * @param scopeEndLine the line of the end position of the scope + * @param scopeEndOffset the offset of the end position of the scope + */ +final case class SuggestionRowUniqueIndex( + kind: Suggestion.Kind, + module: String, + name: String, + selfType: String, + scopeStartLine: Int, + scopeStartOffset: Int, + scopeEndLine: Int, + scopeEndOffset: Int +) + +object SuggestionRowUniqueIndex { + + /** Create an index element from the provided suggestion. + * + * @param suggestion the suggestion + * @return an index element representing the provided suggestion + */ + def apply(suggestion: Suggestion): SuggestionRowUniqueIndex = { + val scope = Suggestion.Scope(suggestion) + val suggestionName = suggestion match { + case conversion: Suggestion.Conversion => + NameColumn.conversionMethodName( + conversion.sourceType, + conversion.returnType + ) + case _ => suggestion.name + } + new SuggestionRowUniqueIndex( + Suggestion.Kind(suggestion), + suggestion.module, + suggestionName, + Suggestion.SelfType(suggestion).getOrElse(SelfTypeColumn.EMPTY), + scope.map(_.start.line).getOrElse(ScopeColumn.EMPTY), + scope.map(_.start.character).getOrElse(ScopeColumn.EMPTY), + scope.map(_.end.line).getOrElse(ScopeColumn.EMPTY), + scope.map(_.end.character).getOrElse(ScopeColumn.EMPTY) + ) + } + + /** Create an index element from the provided suggestion row. + * + * @param row the suggestion row + * @return an index element representing the provided suggestion row + */ + def apply(row: SuggestionRow): SuggestionRowUniqueIndex = + new SuggestionRowUniqueIndex( + SuggestionKind.toSuggestion(row.kind), + row.module, + row.name, + row.selfType, + row.scopeStartLine, + row.scopeStartOffset, + row.scopeEndLine, + row.scopeEndOffset + ) +} + /** The schema of the module_versions table. */ @nowarn("msg=multiarg infix syntax") final class ModuleVersionsTable(tag: Tag) @@ -262,8 +297,6 @@ final class SchemaVersionTable(tag: Tag) def * = id.? <> (SchemaVersionRow.apply, SchemaVersionRow.unapply) } -object Arguments extends TableQuery(new ArgumentsTable(_)) - object Suggestions extends TableQuery(new SuggestionsTable(_)) object ModuleVersions extends TableQuery(new ModuleVersionsTable(_)) diff --git a/lib/scala/searcher/src/test/resources/logback-test.xml b/lib/scala/searcher/src/test/resources/logback-test.xml index 6774939257b9..5921dbc12a31 100644 --- a/lib/scala/searcher/src/test/resources/logback-test.xml +++ b/lib/scala/searcher/src/test/resources/logback-test.xml @@ -8,10 +8,11 @@ - - + + + - + diff --git a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala index 80e531fdc2e7..ff3ed4cd1fe9 100644 --- a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala +++ b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/SuggestionsRepoTest.scala @@ -2,19 +2,28 @@ package org.enso.searcher.sql import java.nio.file.{Files, Path} import java.util.UUID - import org.enso.polyglot.{ExportedSymbol, ModuleExports, Suggestion} import org.enso.polyglot.runtime.Runtime.Api +import org.enso.searcher.SuggestionEntry import org.enso.searcher.data.QueryResult +import org.enso.searcher.sql.equality.SuggestionsEquality import org.enso.testkit.RetrySpec +import org.scalactic.TripleEqualsSupport import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import java.sql.SQLException + import scala.concurrent.Await import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ -class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { +class SuggestionsRepoTest + extends AnyWordSpec + with Matchers + with RetrySpec + with TripleEqualsSupport + with SuggestionsEquality { val Timeout: FiniteDuration = 20.seconds @@ -61,26 +70,8 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { thrown.version shouldEqual wrongSchemaVersion } - "get all suggestions" taggedAs Retry in withRepo { repo => - val action = - for { - _ <- repo.insertAll( - Seq( - suggestion.module, - suggestion.tpe, - suggestion.constructor, - suggestion.method, - suggestion.instanceMethod, - suggestion.conversion, - suggestion.function, - suggestion.local - ) - ) - all <- repo.getAll - } yield all._2 - - val suggestions = Await.result(action, Timeout).map(_.suggestion) - suggestions should contain theSameElementsAs Seq( + "insert all suggestions" taggedAs Retry in withRepo { repo => + val suggestions = Seq( suggestion.module, suggestion.tpe, suggestion.constructor, @@ -90,115 +81,44 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.function, suggestion.local ) + val action = + for { + v1 <- repo.currentVersion + (v2, ids) <- repo.insertAll(suggestions) + all <- repo.selectAllSuggestions + } yield (ids, all, v1, v2) + + val (ids, entries, v1, v2) = Await.result(action, Timeout) + val expectedEntries = ids.zip(suggestions).map(SuggestionEntry.tupled) + entries should contain theSameElementsAs expectedEntries + v1 should not equal v2 } - "get suggestions by method call info" taggedAs Retry in withRepo { repo => - val action = for { - (_, ids) <- repo.insertAll( - Seq( - suggestion.module, - suggestion.tpe, - suggestion.constructor, - suggestion.method, - suggestion.instanceMethod, - suggestion.conversion, - suggestion.function, - suggestion.local - ) - ) - results <- repo.getAllMethods( - Seq( - ("local.Test.Main", "local.Test.Main", "main"), - ("local.Test.Main", "local.Test.Main", "foo") - ) - ) - } yield (ids, results) - - val (ids, results) = Await.result(action, Timeout) - results should contain theSameElementsInOrderAs Seq(ids(3), None) - } - - "get suggestions by empty method call info" taggedAs Retry in withRepo { - repo => - val action = for { - _ <- repo.insertAll( - Seq( - suggestion.module, - suggestion.tpe, - suggestion.constructor, - suggestion.method, - suggestion.conversion, - suggestion.function, - suggestion.local - ) - ) - results <- repo.getAllMethods(Seq()) - } yield results - - val results = Await.result(action, Timeout) - results.isEmpty shouldEqual true - } - - "get all module names" taggedAs Retry in withRepo { repo => - val action = for { - _ <- repo.insertAll( - Seq( - suggestion.module, - suggestion.tpe, - suggestion.constructor, - suggestion.method, - suggestion.conversion, - suggestion.function, - suggestion.local - ) - ) - results <- repo.getAllModules - } yield results - - val results = Await.result(action, Timeout) - results shouldEqual Seq(suggestion.constructor.module) - } - - "fail to insert duplicate suggestion" taggedAs Retry in withRepo { repo => + "get all suggestions" taggedAs Retry in withRepo { repo => val action = for { - (_, ids) <- repo.insertAll( + _ <- repo.insertAll( Seq( suggestion.module, - suggestion.module, - suggestion.tpe, suggestion.tpe, suggestion.constructor, - suggestion.constructor, - suggestion.method, suggestion.method, - suggestion.conversion, + suggestion.instanceMethod, suggestion.conversion, suggestion.function, - suggestion.function, - suggestion.local, suggestion.local ) ) all <- repo.getAll - } yield (ids, all._2) - - val (ids, all) = Await.result(action, Timeout) - ids(0) shouldBe a[Some[_]] - ids(1) shouldBe a[None.type] - ids(2) shouldBe a[Some[_]] - ids(3) shouldBe a[None.type] - ids(4) shouldBe a[Some[_]] - ids(5) shouldBe a[None.type] - ids(6) shouldBe a[Some[_]] - ids(7) shouldBe a[None.type] - ids(8) shouldBe a[Some[_]] - ids(9) shouldBe a[None.type] - all.map(_.suggestion) should contain theSameElementsAs Seq( + } yield all._2 + + val suggestions = Await.result(action, Timeout).map(_.suggestion) + suggestions should contain theSameElementsAs Seq( suggestion.module, suggestion.tpe, suggestion.constructor, suggestion.method, + suggestion.instanceMethod, suggestion.conversion, suggestion.function, suggestion.local @@ -209,16 +129,10 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { repo => val action = for { - (v1, ids) <- repo.insertAll(Seq(suggestion.local, suggestion.local)) - (v2, all) <- repo.getAll - } yield (v1, v2, ids, all) + _ <- repo.insertAll(Seq(suggestion.local, suggestion.local)) + } yield () - val (v1, v2, ids, all) = Await.result(action, Timeout) - v1 shouldEqual v2 - ids.flatten.length shouldEqual 1 - all.map(_.suggestion) should contain theSameElementsAs Seq( - suggestion.local - ) + an[SQLException] should be thrownBy Await.result(action, Timeout) } "select suggestion by id" taggedAs Retry in withRepo { repo => @@ -256,7 +170,7 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { ) ) (_, idsRem) <- repo.removeModules(Seq(suggestion.constructor.module)) - } yield (idsIns.flatten, idsRem) + } yield (idsIns, idsRem) val (inserted, removed) = Await.result(action, Timeout) inserted should contain theSameElementsAs removed @@ -284,28 +198,6 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { removed shouldEqual Seq() } - "remove all suggestions" taggedAs Retry in withRepo { repo => - val action = for { - (_, Seq(_, _, id1, _, _, _, id4)) <- repo.insertAll( - Seq( - suggestion.module, - suggestion.tpe, - suggestion.constructor, - suggestion.method, - suggestion.conversion, - suggestion.function, - suggestion.local - ) - ) - (_, ids) <- repo.removeAll( - Seq(suggestion.constructor, suggestion.local) - ) - } yield (Seq(id1, id4), ids) - - val (inserted, removed) = Await.result(action, Timeout) - inserted should contain theSameElementsAs removed - } - "get version" taggedAs Retry in withRepo { repo => val action = repo.currentVersion @@ -401,36 +293,6 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { v3 shouldEqual v4 } - "change version after remove all suggestions" taggedAs Retry in withRepo { - repo => - val action = for { - v1 <- repo.currentVersion - _ <- repo.insert(suggestion.local) - v2 <- repo.currentVersion - (v3, _) <- repo.removeAll(Seq(suggestion.local)) - } yield (v1, v2, v3) - - val (v1, v2, v3) = Await.result(action, Timeout) - v1 should not equal v2 - v2 should not equal v3 - } - - "not change version after failed remove all suggestions" taggedAs Retry in withRepo { - repo => - val action = for { - v1 <- repo.currentVersion - _ <- repo.insert(suggestion.local) - v2 <- repo.currentVersion - (v3, _) <- repo.removeAll(Seq(suggestion.local)) - (v4, _) <- repo.removeAll(Seq(suggestion.local)) - } yield (v1, v2, v3, v4) - - val (v1, v2, v3, v4) = Await.result(action, Timeout) - v1 should not equal v2 - v2 should not equal v3 - v3 shouldEqual v4 - } - "update suggestion by external id" taggedAs Retry in withRepo { repo => val newReturnType = "Quux" val action = for { @@ -465,16 +327,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.method, Some(Some(newUuid)), None, None, None, - None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -496,16 +357,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.function, Some(None), None, None, None, - None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -527,16 +387,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.function, None, - None, Some(newReturnType), None, None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -558,16 +417,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.tpe, None, None, - None, Some(Some(newDoc)), None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -592,16 +450,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.constructor, None, None, - None, Some(Some(newDoc)), None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -626,16 +483,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.module, None, None, - None, Some(Some(newDoc)), None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -658,16 +514,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.conversion, None, None, - None, Some(Some(newDoc)), None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -692,16 +547,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.function, None, None, - None, Some(Some(newDoc)), None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -725,16 +579,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.local, None, None, - None, Some(Some(newDoc)), None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -758,16 +611,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.constructor, None, None, - None, Some(None), None, None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -792,16 +644,15 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { suggestion.local ) ) - (v2, id2) <- repo.update( + (v2, Some(id2)) <- repo.update( suggestion.local, None, None, None, - None, Some(newScope), None ) - s <- repo.select(id1.get) + s <- repo.select(id1) } yield (v1, id1, v2, id2, s) val (v1, id1, v2, id2, s) = Await.result(action, Timeout) v1 should not equal v2 @@ -809,130 +660,9 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { s shouldEqual Some(suggestion.local.copy(scope = newScope)) } - "remove suggestion arguments" taggedAs Retry in withRepo { repo => - val newArgs = Seq( - Api.SuggestionArgumentAction.Remove(1) - ) - val action = for { - (v1, Seq(_, _, id1, _, _, _, _)) <- repo.insertAll( - Seq( - suggestion.module, - suggestion.tpe, - suggestion.constructor, - suggestion.method, - suggestion.conversion, - suggestion.function, - suggestion.local - ) - ) - (v2, id2) <- repo.update( - suggestion.constructor, - None, - Some(newArgs), - None, - None, - None, - None - ) - s <- repo.select(id1.get) - } yield (v1, id1, v2, id2, s) - val (v1, id1, v2, id2, s) = Await.result(action, Timeout) - v1 should not equal v2 - id1 shouldEqual id2 - s shouldEqual Some( - suggestion.constructor.copy(arguments = - suggestion.constructor.arguments.init - ) - ) - } - - "add suggestion arguments" taggedAs Retry in withRepo { repo => - val newArgs = Seq( - Api.SuggestionArgumentAction - .Add(2, suggestion.constructor.arguments(0)), - Api.SuggestionArgumentAction.Add(3, suggestion.constructor.arguments(1)) - ) - val action = for { - (v1, Seq(_, _, id1, _, _, _, _)) <- repo.insertAll( - Seq( - suggestion.module, - suggestion.tpe, - suggestion.constructor, - suggestion.method, - suggestion.conversion, - suggestion.function, - suggestion.local - ) - ) - (v2, id2) <- repo.update( - suggestion.constructor, - None, - Some(newArgs), - None, - None, - None, - None - ) - s <- repo.select(id1.get) - } yield (v1, id1, v2, id2, s) - val (v1, id1, v2, id2, s) = Await.result(action, Timeout) - v1 should not equal v2 - id1 shouldEqual id2 - s shouldEqual Some( - suggestion.constructor.copy(arguments = - suggestion.constructor.arguments ++ suggestion.constructor.arguments - ) - ) - } - - "update suggestion arguments" taggedAs Retry in withRepo { repo => - val newArgs = Seq( - Api.SuggestionArgumentAction.Modify( - 1, - Some("c"), - Some("C"), - Some(true), - Some(true), - Some(Some("C")) - ) - ) - val action = for { - (v1, Seq(_, _, id1, _, _, _, _)) <- repo.insertAll( - Seq( - suggestion.module, - suggestion.tpe, - suggestion.constructor, - suggestion.method, - suggestion.conversion, - suggestion.function, - suggestion.local - ) - ) - (v2, id2) <- repo.update( - suggestion.constructor, - None, - Some(newArgs), - None, - None, - None, - None - ) - s <- repo.select(id1.get) - } yield (v1, id1, v2, id2, s) - val (v1, id1, v2, id2, s) = Await.result(action, Timeout) - v1 should not equal v2 - id1 shouldEqual id2 - s shouldEqual Some( - suggestion.constructor.copy(arguments = - suggestion.constructor.arguments.init :+ - Suggestion.Argument("c", "C", true, true, Some("C")) - ) - ) - } - "update suggestion empty request" taggedAs Retry in withRepo { repo => val action = for { - (v1, _) <- repo.insertAll( + (v1, Seq(_, _, _, id1, _, _, _)) <- repo.insertAll( Seq( suggestion.module, suggestion.tpe, @@ -949,13 +679,12 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { None, None, None, - None, None ) - } yield (v1, v2, id2) - val (v1, v2, id2) = Await.result(action, Timeout) + } yield (v1, v2, id1, id2) + val (v1, v2, id1, id2) = Await.result(action, Timeout) v1 shouldEqual v2 - id2 shouldEqual None + id2 shouldEqual Some(id1) } "change version after updateAll" taggedAs Retry in withRepo { repo => @@ -1034,8 +763,8 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { val (ids, results) = Await.result(action, Timeout) results should contain theSameElementsAs Seq( - QueryResult(ids(0).toSeq, updates(0)), - QueryResult(ids(3).toSeq, updates(1)) + QueryResult(Seq(ids(0)), updates(0)), + QueryResult(Seq(ids(3)), updates(1)) ) } @@ -1076,7 +805,7 @@ class SuggestionsRepoTest extends AnyWordSpec with Matchers with RetrySpec { val (ids, results) = Await.result(action, Timeout) results should contain theSameElementsAs Seq( - QueryResult(ids(0).toSeq, updates(0)), + QueryResult(Seq(ids(0)), updates(0)), QueryResult(Seq(), updates(1)) ) } diff --git a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionEntryEqualityIgnoringArguments.scala b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionEntryEqualityIgnoringArguments.scala new file mode 100644 index 000000000000..0aaa6b353f5b --- /dev/null +++ b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionEntryEqualityIgnoringArguments.scala @@ -0,0 +1,18 @@ +package org.enso.searcher.sql.equality + +import org.enso.searcher.SuggestionEntry +import org.scalactic.Equality + +object SuggestionEntryEqualityIgnoringArguments + extends Equality[SuggestionEntry] { + + /** @inheritdoc */ + override def areEqual(a: SuggestionEntry, o: Any): Boolean = { + o match { + case b: SuggestionEntry => + a.id == b.id && + SuggestionEqualityIgnoringArguments.areEqual(a.suggestion, b.suggestion) + case _ => false + } + } +} diff --git a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionEqualityIgnoringArguments.scala b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionEqualityIgnoringArguments.scala new file mode 100644 index 000000000000..28fe714fc87a --- /dev/null +++ b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionEqualityIgnoringArguments.scala @@ -0,0 +1,20 @@ +package org.enso.searcher.sql.equality + +import org.enso.polyglot.Suggestion +import org.scalactic.Equality + +object SuggestionEqualityIgnoringArguments extends Equality[Suggestion] { + + /** @inheritdoc */ + override def areEqual(a: Suggestion, o: Any): Boolean = { + o match { + case b: Suggestion => + a.module == b.module && + a.name == b.name && + a.externalId == b.externalId && + a.returnType == b.returnType && + a.documentation == b.documentation + case _ => false + } + } +} diff --git a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionOptionEqualityIgnoringArguments.scala b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionOptionEqualityIgnoringArguments.scala new file mode 100644 index 000000000000..6efbcbfa832b --- /dev/null +++ b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionOptionEqualityIgnoringArguments.scala @@ -0,0 +1,19 @@ +package org.enso.searcher.sql.equality + +import org.enso.polyglot.Suggestion +import org.scalactic.Equality + +object SuggestionOptionEqualityIgnoringArguments + extends Equality[Option[Suggestion]] { + + /** @inheritdoc */ + override def areEqual(o1: Option[Suggestion], o2: Any): Boolean = + (o1, o2) match { + case (Some(a), Some(b: Suggestion)) => + SuggestionEqualityIgnoringArguments.areEqual(a, b) + case (None, None) => + true + case _ => + false + } +} diff --git a/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionsEquality.scala b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionsEquality.scala new file mode 100644 index 000000000000..4d0ab6a51389 --- /dev/null +++ b/lib/scala/searcher/src/test/scala/org/enso/searcher/sql/equality/SuggestionsEquality.scala @@ -0,0 +1,19 @@ +package org.enso.searcher.sql.equality + +import org.enso.polyglot.Suggestion +import org.enso.searcher.SuggestionEntry +import org.scalactic.Equality + +trait SuggestionsEquality { + + implicit def suggestionEquality: Equality[Suggestion] = + SuggestionEqualityIgnoringArguments + + implicit def suggestionEntryEquality: Equality[SuggestionEntry] = + SuggestionEntryEqualityIgnoringArguments + + implicit def suggestionOptionEquality: Equality[Option[Suggestion]] = + SuggestionOptionEqualityIgnoringArguments +} + +object SuggestionsEquality extends SuggestionsEquality diff --git a/std-bits/base/src/main/java/org/enso/base/Text_Utils.java b/std-bits/base/src/main/java/org/enso/base/Text_Utils.java index 277b0a06814e..34654e262e07 100644 --- a/std-bits/base/src/main/java/org/enso/base/Text_Utils.java +++ b/std-bits/base/src/main/java/org/enso/base/Text_Utils.java @@ -277,13 +277,7 @@ public static long grapheme_length(String str) { /** Returns a prefix of the string not exceeding the provided grapheme length. */ public static String take_prefix(String str, long grapheme_length) { - BreakIterator iter = BreakIterator.getCharacterInstance(); - iter.setText(str); - if (iter.next(Math.toIntExact(grapheme_length)) == BreakIterator.DONE) { - return str; - } else { - return str.substring(0, iter.current()); - } + return Core_Text_Utils.take_prefix(str, grapheme_length); } /** Returns a suffix of the string not exceeding the provided grapheme length. */ diff --git a/std-bits/table/src/main/java/org/enso/table/parsing/TypeInferringParser.java b/std-bits/table/src/main/java/org/enso/table/parsing/TypeInferringParser.java index 51c6d3d27cb8..6e13caebffa9 100644 --- a/std-bits/table/src/main/java/org/enso/table/parsing/TypeInferringParser.java +++ b/std-bits/table/src/main/java/org/enso/table/parsing/TypeInferringParser.java @@ -40,10 +40,11 @@ public Object parseSingleValue(String text, ProblemAggregator problemAggregator) @Override public WithProblems> parseColumn(String columnName, Storage sourceStorage) { - // If there are now rows, the Auto parser would guess some random type (the first one that is - // checked). Instead, - // we just return the empty column unchanged. - if (sourceStorage.size() == 0) { + // If there are no values, the Auto parser would guess some random type (the first one that is + // checked). Instead, we just return the empty column unchanged. + boolean hasNoValues = + (sourceStorage.size() == 0) || (sourceStorage.countMissing() == sourceStorage.size()); + if (hasNoValues) { return fallbackParser.parseColumn(columnName, sourceStorage); } diff --git a/test/Table_Tests/src/Common_Table_Operations/Cast_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Cast_Spec.enso new file mode 100644 index 000000000000..c33f840e4bc1 --- /dev/null +++ b/test/Table_Tests/src/Common_Table_Operations/Cast_Spec.enso @@ -0,0 +1,93 @@ +from Standard.Base import all + +from Standard.Table import Value_Type + +from Standard.Test import Test, Problems +import Standard.Test.Extensions + +from project.Common_Table_Operations.Util import run_default_backend + +main = run_default_backend spec + +spec setup = + prefix = setup.prefix + table_builder = setup.table_builder + # TODO this spec will be expanded in #6112 + Test.group prefix+"Column.cast" pending=(if setup.is_database.not then "Cast is not implemented in the in-memory backend yet.") <| + Test.specify "should allow to cast an integer column to text" <| + t = table_builder [["X", [1, 2, 3000]]] + c = t.at "X" . cast Value_Type.Char + c.value_type.is_text . should_be_true + c.to_vector . should_equal ["1", "2", "3000"] + + Test.specify "should allow to cast a boolean column to integer" <| + t = table_builder [["X", [True, False, True]]] + c = t.at "X" . cast Value_Type.Integer + c.value_type.is_integer . should_be_true + c.to_vector . should_equal [1, 0, 1] + + Test.specify "should allow to cast a boolean column to text" pending="TODO: sqlite has issue with this, figure out in #6112" <| + t = table_builder [["X", [True, False, True]]] + c = t.at "X" . cast Value_Type.Char + c.value_type.is_text . should_be_true + c.to_vector . should_equal ["true", "false", "true"] + + Test.specify "should allow to cast a text column to fixed-length" pending=(if setup.test_selection.fixed_length_text_columns.not then "Fixed-length Char columns are not supported by this backend.") <| + t = table_builder [["X", ["a", "DEF", "a slightly longer text"]]] + c = t.at "X" . cast (Value_Type.Char size=3 variable_length=False) + c.value_type . should_equal (Value_Type.Char size=3 variable_length=False) + c.to_vector . should_equal ["a ", "DEF", "a s"] + + Test.specify "should work if the first row is NULL" <| + t = table_builder [["X", [Nothing, 1, 2, 3000]], ["Y", [Nothing, True, False, True]]] + + c1 = t.at "X" . cast Value_Type.Char + c1.value_type.is_text . should_be_true + c1.to_vector . should_equal [Nothing, "1", "2", "3000"] + + c2 = t.at "Y" . cast Value_Type.Integer + c2.value_type.is_integer . should_be_true + c2.to_vector . should_equal [Nothing, 1, 0, 1] + + Test.specify "should not lose the type after further operations were performed on the result" <| + t = table_builder [["X", [1, 2, 3000]], ["Y", [True, False, True]]] + c1 = t.at "X" . cast Value_Type.Char + c2 = t.at "Y" . cast Value_Type.Integer + + c3 = c1 + '_suffix' + c3.value_type.is_text . should_be_true + c3.to_vector . should_equal ["1_suffix", "2_suffix", "3000_suffix"] + + c4 = c2 + 1000 + c4.value_type.is_integer . should_be_true + c4.to_vector . should_equal [1001, 1000, 1001] + + pending_sqlite_types = if prefix.contains "SQLite" then "TODO: perform SQLite type inference locally - #6208" + Test.specify "should not lose the type after further operations were performed on the result, even if the first row is NULL" pending=pending_sqlite_types <| + t = table_builder [["X", [Nothing, 1, 2, 3000]], ["Y", [Nothing, True, False, True]]] + c1 = t.at "X" . cast Value_Type.Char + c2 = t.at "Y" . cast Value_Type.Integer + + c3 = c1 + '_suffix' + c3.value_type.is_text . should_be_true + c3.to_vector . should_equal [Nothing, "1_suffix", "2_suffix", "3000_suffix"] + + c4 = c2 + 1000 + c4.value_type.is_integer . should_be_true + c4.to_vector . should_equal [Nothing, 1001, 1000, 1001] + + Test.group prefix+"Table.cast" pending=(if setup.is_database.not then "Cast is not implemented in the in-memory backend yet.") <| + Test.specify 'should cast the columns "in-place" and not reorder them' <| + t = table_builder [["X", [1, 2, 3000]], ["Y", [4, 5, 6]], ["Z", [7, 8, 9]], ["A", [True, False, True]]] + t2 = t.cast ["Z", "Y"] Value_Type.Char + t2.column_names . should_equal ["X", "Y", "Z", "A"] + + t2.at "X" . value_type . is_integer . should_be_true + t2.at "Y" . value_type . is_text . should_be_true + t2.at "Z" . value_type . is_text . should_be_true + t2.at "A" . value_type . is_boolean . should_be_true + + t2.at "X" . to_vector . should_equal [1, 2, 3000] + t2.at "Y" . to_vector . should_equal ["4", "5", "6"] + t2.at "Z" . to_vector . should_equal ["7", "8", "9"] + t2.at "A" . to_vector . should_equal [True, False, True] diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index d82288156d2f..86ec7e2263b2 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -19,7 +19,11 @@ spec setup = t = table_builder [["X", [True, False, Nothing, True]]] t.at "X" . iif 22 33 . to_vector . should_equal [22, 33, Nothing, 22] - Test.specify "iif on Columns" pending="Not implemented yet." Nothing + Test.specify "iif on Columns" <| + t = table_builder [["X", [True, False, Nothing, False]], ["Y", [1, 2, 3, 4]], ["Z", [1.5, 2.5, 3.5, 4.5]]] + c = t.at "X" . iif (t.at "Y") (t.at "Z") + c.value_type . is_floating_point . should_be_true + c.to_vector . should_equal [1, 2.5, Nothing, 4.5] t2 = table_builder [["x", [1, 4, 5, Nothing]], ["y", [2, 3, 5, Nothing]], ["b", [False, False, True, Nothing]]] x = t2.at "x" diff --git a/test/Table_Tests/src/Common_Table_Operations/Join/Join_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Join/Join_Spec.enso index 3a7801fbb3d7..7288eb388836 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Join/Join_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Join/Join_Spec.enso @@ -46,16 +46,11 @@ spec setup = Test.specify "should allow Full join" <| t3 = t1.join t2 join_kind=Join_Kind.Full |> materialize |> _.order_by ["X", "W"] - case setup.test_selection.supports_full_join of - True -> - expect_column_names ["X", "Y", "Z", "W"] t3 - t3.at "X" . to_vector . should_equal [Nothing, 1, 2, 2, 3] - t3.at "Y" . to_vector . should_equal [Nothing, 4, 5, 5, 6] - t3.at "Z" . to_vector . should_equal [4, Nothing, 2, 2, 3] - t3.at "W" . to_vector . should_equal [7, Nothing, 4, 6, 5] - False -> - t3.should_fail_with Unsupported_Database_Operation - + expect_column_names ["X", "Y", "Z", "W"] t3 + t3.at "X" . to_vector . should_equal [Nothing, 1, 2, 2, 3] + t3.at "Y" . to_vector . should_equal [Nothing, 4, 5, 5, 6] + t3.at "Z" . to_vector . should_equal [4, Nothing, 2, 2, 3] + t3.at "W" . to_vector . should_equal [7, Nothing, 4, 6, 5] Test.specify "should allow Left Outer join" <| t4 = t1.join t2 join_kind=Join_Kind.Left_Outer |> materialize |> _.order_by ["X", "W"] @@ -263,17 +258,14 @@ spec setup = t2.at "Right_A" . to_vector . should_equal ["B", "C", "C", "D"] t3 = t1.join t1 join_kind=Join_Kind.Full on=(Join_Condition.Equals left="X" right="Y") |> materialize |> _.order_by ["X", "Y", "Right_X"] - case setup.test_selection.supports_full_join of - True -> - expect_column_names ["X", "Y", "A", "Right_X", "Right_Y", "Right_A"] t3 - t3.at "X" . to_vector . should_equal [Nothing, Nothing, 0, 1, 2, 2, 3] - t3.at "Right_Y" . to_vector . should_equal [100, 4, Nothing, 1, 2, 2, 3] - - t3.at "Y" . to_vector . should_equal [Nothing, Nothing, 1, 2, 3, 100, 4] - t3.at "A" . to_vector . should_equal [Nothing, Nothing, "B", "C", "D", "X", "E"] - t3.at "Right_X" . to_vector . should_equal [2, 3, Nothing, 0, 1, 1, 2] - t3.at "Right_A" . to_vector . should_equal ["X", "E", Nothing, "B", "C", "C", "D"] - False -> Nothing + expect_column_names ["X", "Y", "A", "Right_X", "Right_Y", "Right_A"] t3 + t3.at "X" . to_vector . should_equal [Nothing, Nothing, 0, 1, 2, 2, 3] + t3.at "Right_Y" . to_vector . should_equal [100, 4, Nothing, 1, 2, 2, 3] + + t3.at "Y" . to_vector . should_equal [Nothing, Nothing, 1, 2, 3, 100, 4] + t3.at "A" . to_vector . should_equal [Nothing, Nothing, "B", "C", "D", "X", "E"] + t3.at "Right_X" . to_vector . should_equal [2, 3, Nothing, 0, 1, 1, 2] + t3.at "Right_A" . to_vector . should_equal ["X", "E", Nothing, "B", "C", "C", "D"] t4 = table_builder [["X", [Nothing, "a", "B"]], ["Y", ["ą", "b", Nothing]], ["Z", [1, 2, 3]]] t5 = t4.join t4 on=(Join_Condition.Equals_Ignore_Case left="Y" right="X") |> materialize |> _.order_by ["Y"] @@ -487,21 +479,17 @@ spec setup = r3.at 4 . should_equal [2, 3, 2, 5] t4 = t1.join t2 on=[Join_Condition.Equals "A" "C"] join_kind=Join_Kind.Full - case setup.test_selection.supports_full_join of - True -> - expect_column_names ["A", "B", "C", "D"] t4 - r4 = materialize t4 . order_by ["A", "B", "D", "C"] . rows . map .to_vector - within_table t4 <| - r4.length . should_equal 7 - r4.at 0 . should_equal [Nothing, Nothing, Nothing, Nothing] - r4.at 1 . should_equal [Nothing, Nothing, 4, Nothing] - r4.at 2 . should_equal [Nothing, Nothing, Nothing, 6] - r4.at 3 . should_equal [Nothing, 4, Nothing, Nothing] - r4.at 4 . should_equal [Nothing, 4, Nothing, 6] - r4.at 5 . should_equal [1, 7, Nothing, Nothing] - r4.at 6 . should_equal [2, 3, 2, 5] - False -> - (materialize t4) . should_fail_with Unsupported_Database_Operation + expect_column_names ["A", "B", "C", "D"] t4 + r4 = materialize t4 . order_by ["A", "B", "D", "C"] . rows . map .to_vector + within_table t4 <| + r4.length . should_equal 7 + r4.at 0 . should_equal [Nothing, Nothing, Nothing, Nothing] + r4.at 1 . should_equal [Nothing, Nothing, 4, Nothing] + r4.at 2 . should_equal [Nothing, Nothing, Nothing, 6] + r4.at 3 . should_equal [Nothing, 4, Nothing, Nothing] + r4.at 4 . should_equal [Nothing, 4, Nothing, 6] + r4.at 5 . should_equal [1, 7, Nothing, Nothing] + r4.at 6 . should_equal [2, 3, 2, 5] t4_2 = t1.join t2 on=[Join_Condition.Equals "A" "C"] join_kind=Join_Kind.Left_Outer expect_column_names ["A", "B", "C", "D"] t4_2 @@ -617,7 +605,7 @@ spec setup = r2.at 0 . should_equal [2, 20, 5, 5, 100] r2.at 1 . should_equal [3, 30, 7, 7, 200] - Test.specify "should allow full joins with more complex join conditions" pending=(if setup.test_selection.supports_full_join.not then "Full join workaround is not implemented for this backend yet.") <| + Test.specify "should allow full joins with more complex join conditions" <| t1 = table_builder [["X", ["a", "b", "c"]], ["Y", [10, 20, 30]]] t2 = table_builder [["X", ["Ć", "A", "b"]], ["Z", [100, 200, 300]]] diff --git a/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso index 26d7fc7e15f0..8d421755b4f8 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso @@ -20,22 +20,23 @@ main = run_default_backend spec spec setup = prefix = setup.prefix table_builder = setup.table_builder - db_todo = if prefix.contains "In-Memory" then Nothing else "Table.union is not yet implemented for the DB backend." - Test.group prefix+"Table.union" pending=db_todo <| + Test.group prefix+"Table.union" <| Test.specify "should merge columns from multiple tables" <| - t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] - t2 = table_builder [["A", [4, 5, 6]], ["B", ["d", "e", "f"]]] - t3 = table_builder [["A", [7, 8, 9]], ["B", ["g", "h", "i"]]] + t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]], ["C", [True, False, True]]] + t2 = table_builder [["A", [4, 5, 6]], ["B", ["d", "e", "f"]], ["C", [False, True, False]]] + t3 = table_builder [["A", [7, 8, 9]], ["B", ["g", "h", "i"]], ["C", [True, False, False]]] t4 = t1.union t2 - expect_column_names ["A", "B"] t4 + expect_column_names ["A", "B", "C"] t4 t4.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] t4.at "B" . to_vector . should_equal ["a", "b", "c", "d", "e", "f"] + t4.at "C" . to_vector . should_equal [True, False, True, False, True, False] t5 = t3.union [t1, t2] - expect_column_names ["A", "B"] t5 + expect_column_names ["A", "B", "C"] t5 t5.at "A" . to_vector . should_equal [7, 8, 9, 1, 2, 3, 4, 5, 6] t5.at "B" . to_vector . should_equal ["g", "h", "i", "a", "b", "c", "d", "e", "f"] + t5.at "C" . to_vector . should_equal [True, False, False, True, False, True, False, True, False] Test.specify "should fill unmatched columns (by name matching) with nulls and report a warning by default" <| t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] @@ -160,66 +161,100 @@ spec setup = t8 = t1.union [t2, t5, t6, t7] match_columns=Match_Columns.By_Position expect_column_names ["Y", "A", "Z"] t8 + Test.specify "should allow to merge a table with itself" <| + t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] + t2 = t1.union [t1, t1] + expect_column_names ["A", "B"] t2 + t2.at "A" . to_vector . should_equal [1, 2, 3, 1, 2, 3, 1, 2, 3] + t2.at "B" . to_vector . should_equal ["a", "b", "c", "a", "b", "c", "a", "b", "c"] + + Test.specify "should not de-duplicate rows" <| + t1 = table_builder [["A", [1, 1, 3]], ["B", ["a", "a", "c"]]] + t2 = table_builder [["A", [1, 2, 2]], ["B", ["a", "b", "b"]]] + t3 = t1.union t2 + expect_column_names ["A", "B"] t3 + t3.at "A" . to_vector . should_equal [1, 1, 3, 1, 2, 2] + t3.at "B" . to_vector . should_equal ["a", "a", "c", "a", "b", "b"] + Test.specify "should gracefully handle the case where no tables to union were provided" <| t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] - t1.union [] . should_equal t1 - t1.union [] match_columns=Match_Columns.By_Position . should_equal t1 + check_same table = + expect_column_names ["A", "B"] table + table.at "A" . to_vector . should_equal [1, 2, 3] + table.at "B" . to_vector . should_equal ["a", "b", "c"] - t1.union [] keep_unmatched_columns=False . should_equal t1 - t1.union [] match_columns=Match_Columns.By_Position keep_unmatched_columns=False . should_equal t1 + check_same <| t1.union [] + check_same <| t1.union [] match_columns=Match_Columns.By_Position - t1.union [] keep_unmatched_columns=True . should_equal t1 - t1.union [] match_columns=Match_Columns.By_Position keep_unmatched_columns=True . should_equal t1 + check_same <| t1.union [] keep_unmatched_columns=False + check_same <| t1.union [] match_columns=Match_Columns.By_Position keep_unmatched_columns=False + + check_same <| t1.union [] keep_unmatched_columns=True + check_same <| t1.union [] match_columns=Match_Columns.By_Position keep_unmatched_columns=True + + Test.specify "should correctly unify text columns of various lengths" pending=(if setup.test_selection.fixed_length_text_columns.not then "Fixed-length Char columns are not supported by this backend.") <| + t1 = table_builder [["A", ["a", "b", "c"]]] . cast "A" (Value_Type.Char size=1 variable_length=False) + t2 = table_builder [["A", ["xyz", "abc", "def"]]] . cast "A" (Value_Type.Char size=3 variable_length=False) + + t1.at "A" . value_type . should_equal (Value_Type.Char size=1 variable_length=False) + t2.at "A" . value_type . should_equal (Value_Type.Char size=3 variable_length=False) + + t3 = t1.union t2 + expect_column_names ["A"] t3 + t3.at "A" . to_vector . should_equal ["a", "b", "c", "xyz", "abc", "def"] + t3.at "A" . value_type . is_text . should_be_true + t3.at "A" . value_type . variable_length . should_be_true Test.specify "should find a common type that will fit the merged columns" <| t1 = table_builder [["int+bool", [1, 2, 3]], ["int+float", [0, 1, 2]]] t2 = table_builder [["int+bool", [True, False, Nothing]], ["int+float", [1.0, 2.0, 2.5]]] - t1.at "int+bool" . value_type . should_equal Value_Type.Integer - t1.at "int+float" . value_type . should_equal Value_Type.Integer - t2.at "int+bool" . value_type . should_equal Value_Type.Boolean - t2.at "int+float" . value_type . should_equal Value_Type.Float + t1.at "int+bool" . value_type . is_integer . should_be_true + t1.at "int+float" . value_type . is_integer . should_be_true + t2.at "int+bool" . value_type . is_boolean . should_be_true + t2.at "int+float" . value_type . is_floating_point . should_be_true t3 = t1.union t2 expect_column_names ["int+bool", "int+float"] t3 - t3.at "int+bool" . value_type . should_equal Value_Type.Integer - t3.at "int+float" . value_type . should_equal Value_Type.Float + t3.at "int+bool" . value_type . is_integer . should_be_true + t3.at "int+float" . value_type . is_floating_point . should_be_true t3.at "int+bool" . to_vector . should_equal [1, 2, 3, 1, 0, Nothing] t3.at "int+float" . to_vector . should_equal [0, 1, 2, 1.0, 2.0, 2.5] t4 = table_builder [["float", [1.0, 2.0, 3.3]]] t5 = t1.union [t2, t4] match_columns=Match_Columns.By_Position keep_unmatched_columns=False expect_column_names ["int+bool"] t5 - t5.at "int+bool" . value_type . should_equal Value_Type.Float + t5.at "int+bool" . value_type . is_floating_point . should_be_true t5.at "int+bool" . to_vector . should_equal [1, 2, 3, 1, 0, Nothing, 1.0, 2.0, 3.3] - Test.specify "should resort to Mixed value type only if at least one column is already Mixed" <| - ## TODO currently no way to retype a column to Mixed, so we are - using a custom object; this test won't work in DB so it will need - to be adapted once proper type support is implemented - t1 = table_builder [["A", [1, 2, 3]], ["mixed", ["a", My_Type.Value 1 2, Nothing]]] - t2 = table_builder [["A", [4, 5, 6]], ["mixed", [1, 2, 3]]] - t1.at "mixed" . value_type . should_equal Value_Type.Mixed - t2.at "mixed" . value_type . should_equal Value_Type.Integer - - t3 = t1.union t2 - Problems.assume_no_problems t3 - expect_column_names ["A", "mixed"] t3 - t3.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] - t3.at "mixed" . to_vector . should_equal ["a", My_Type.Value 1 2, Nothing, 1, 2, 3] + # Database backends are not required to support Mixed types. + if setup.is_database.not then + Test.specify "should resort to Mixed value type only if at least one column is already Mixed" <| + ## TODO currently no way to retype a column to Mixed, so we are + using a custom object + t1 = table_builder [["A", [1, 2, 3]], ["mixed", ["a", My_Type.Value 1 2, Nothing]]] + t2 = table_builder [["A", [4, 5, 6]], ["mixed", [1, 2, 3]]] + t1.at "mixed" . value_type . should_equal Value_Type.Mixed + t2.at "mixed" . value_type . should_equal Value_Type.Integer + + t3 = t1.union t2 + Problems.assume_no_problems t3 + expect_column_names ["A", "mixed"] t3 + t3.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] + t3.at "mixed" . to_vector . should_equal ["a", My_Type.Value 1 2, Nothing, 1, 2, 3] - t4 = table_builder [["A", [1, 3]], ["mixed", [True, False]]] - t5 = table_builder [["A", [4, 5]], ["mixed", ["X", "y"]]] - t4.at "mixed" . value_type . should_equal Value_Type.Boolean - t5.at "mixed" . value_type . should_equal Value_Type.Char + t4 = table_builder [["A", [1, 3]], ["mixed", [True, False]]] + t5 = table_builder [["A", [4, 5]], ["mixed", ["X", "y"]]] + t4.at "mixed" . value_type . should_equal Value_Type.Boolean + t5.at "mixed" . value_type . should_equal Value_Type.Char - t6 = t5.union [t1, t2, t4] - Problems.assume_no_problems t6 - expect_column_names ["A", "mixed"] t6 - t6.at "A" . to_vector . should_equal [4, 5, 1, 2, 3, 4, 5, 6, 1, 3] - t6.at "mixed" . to_vector . should_equal ["X", "y", "a", My_Type.Value 1 2, Nothing, 1, 2, 3, True, False] - t6.at "mixed" . value_type . should_equal Value_Type.Mixed + t6 = t5.union [t1, t2, t4] + Problems.assume_no_problems t6 + expect_column_names ["A", "mixed"] t6 + t6.at "A" . to_vector . should_equal [4, 5, 1, 2, 3, 4, 5, 6, 1, 3] + t6.at "mixed" . to_vector . should_equal ["X", "y", "a", My_Type.Value 1 2, Nothing, 1, 2, 3, True, False] + t6.at "mixed" . value_type . should_equal Value_Type.Mixed Test.specify "if no common type can be found, should report error and drop the problematic column" <| t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]], ["C", [True, False, Nothing]]] @@ -236,52 +271,60 @@ spec setup = t1 = table_builder [["A", [1, 2, 3]]] t2 = table_builder [["A", [4, 5, 6]], ["B", [1.2, 2.2, 3.1]]] - t2.at "B" . value_type . should_equal Value_Type.Float - t3 = t1.union t2 allow_type_widening=False keep_unmatched_columns=True within_table t3 <| Problems.assume_no_problems t3 expect_column_names ["A", "B"] t3 t3.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] - t3.at "A" . value_type . should_equal Value_Type.Integer t3.at "B" . to_vector . should_equal [Nothing, Nothing, Nothing, 1.2, 2.2, 3.1] - t3.at "B" . value_type . should_equal Value_Type.Float + t3.at "A" . value_type . is_integer . should_be_true + t2.at "B" . value_type . is_floating_point . should_be_true + t3.at "B" . value_type . is_floating_point . should_be_true Test.specify "if type widening is not allowed and types do not match, should report error and drop the problematic column" <| t1 = table_builder [["A", [1, 2, 3]], ["B", [1, 2, 3]], ["C", [True, False, Nothing]], ["D", [10, 20, 30]], ["E", [1.1, 2.5, 3.2]]] t2 = table_builder [["A", [4, 5, 6]], ["B", [1.5, 2.5, 3.5]], ["C", [1, 2, 3]], ["D", [True, True, True]], ["E", [1, 2, 3]]] - t1.at "B" . value_type . should_equal Value_Type.Integer - t1.at "C" . value_type . should_equal Value_Type.Boolean - t1.at "D" . value_type . should_equal Value_Type.Integer - t1.at "E" . value_type . should_equal Value_Type.Float + t1.at "B" . value_type . is_integer . should_be_true + t1.at "C" . value_type . is_boolean . should_be_true + t1.at "D" . value_type . is_integer . should_be_true + t1.at "E" . value_type . is_floating_point . should_be_true - t2.at "B" . value_type . should_equal Value_Type.Float - t2.at "C" . value_type . should_equal Value_Type.Integer - t2.at "D" . value_type . should_equal Value_Type.Boolean - t2.at "E" . value_type . should_equal Value_Type.Integer + t2.at "B" . value_type . is_floating_point . should_be_true + t2.at "C" . value_type . is_integer . should_be_true + t2.at "D" . value_type . is_boolean . should_be_true + t2.at "E" . value_type . is_integer . should_be_true action = t1.union t2 allow_type_widening=False on_problems=_ tester table = expect_column_names ["A"] table table.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] - problems = [Column_Type_Mismatch.Error "B" Value_Type.Integer Value_Type.Float, Column_Type_Mismatch.Error "C" Value_Type.Boolean Value_Type.Integer, Column_Type_Mismatch.Error "D" Value_Type.Integer Value_Type.Boolean, Column_Type_Mismatch.Error "E" Value_Type.Float Value_Type.Integer] - Problems.test_problem_handling action problems tester - - Test.specify "even if type widening is not allowed, if the first column is mixed, it should accept any column to be concatenated to it" <| - t1 = table_builder [["X", ["a", 1, Nothing]]] - t2 = table_builder [["X", [1]]] - t3 = table_builder [["X", [1.2, 2.3, 3.4]]] - t4 = table_builder [["X", ["a", "b"]]] - t5 = table_builder [["X", [True, False]]] - - t1.at "X" . value_type . should_equal Value_Type.Mixed - t2.at "X" . value_type . should_equal Value_Type.Integer - t6 = t1.union [t2, t3, t4, t5] allow_type_widening=False - Problems.assume_no_problems t6 - t6.at "X" . value_type . should_equal Value_Type.Mixed - t6.at "X" . to_vector . should_equal ["a", 1, Nothing, 1, 1.2, 2.3, 3.4, "a", "b", True, False] + problem_checker problem = + problem.should_be_a Column_Type_Mismatch + True + err_checker err = + problem_checker err.catch + warn_checker warnings = + warnings.all problem_checker + Problems.test_advanced_problem_handling action err_checker warn_checker tester + + # Database backends are not required to support Mixed types. + if setup.is_database.not then + Test.specify "even if type widening is not allowed, if the first column is mixed, it should accept any column to be concatenated to it" <| + t1 = table_builder [["X", ["a", 1, Nothing]]] + t2 = table_builder [["X", [1]]] + t3 = table_builder [["X", [1.2, 2.3, 3.4]]] + t4 = table_builder [["X", ["a", "b"]]] + t5 = table_builder [["X", [True, False]]] + + t1.at "X" . value_type . should_equal Value_Type.Mixed + t2.at "X" . value_type . should_equal Value_Type.Integer + + t6 = t1.union [t2, t3, t4, t5] allow_type_widening=False + Problems.assume_no_problems t6 + t6.at "X" . value_type . should_equal Value_Type.Mixed + t6.at "X" . to_vector . should_equal ["a", 1, Nothing, 1, 1.2, 2.3, 3.4, "a", "b", True, False] Test.specify "if type mismatches cause all columns to be dropped, fail with No_Output_Columns" <| t1 = table_builder [["A", [1, 2, 3]]] diff --git a/test/Table_Tests/src/Common_Table_Operations/Main.enso b/test/Table_Tests/src/Common_Table_Operations/Main.enso index c6249021851a..d6cd891ebc9d 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Main.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Main.enso @@ -4,6 +4,7 @@ import project.Common_Table_Operations.Aggregate_Spec import project.Common_Table_Operations.Column_Operations_Spec import project.Common_Table_Operations.Core_Spec import project.Common_Table_Operations.Cross_Tab_Spec +import project.Common_Table_Operations.Cast_Spec import project.Common_Table_Operations.Date_Time_Spec import project.Common_Table_Operations.Distinct_Spec import project.Common_Table_Operations.Expression_Spec @@ -80,21 +81,21 @@ type Test_Selection - is_nan_and_nothing_distinct: Specifies if the backend is able to distinguish between a decimal NaN value and a missing value (Enso's Nothing, or SQL's NULL). If `False`, NaN is treated as a NULL. - - supports_full_join: Specifies if the backend supports full joins. - SQLite doesn't so we need to disable them until we implement a proper - workaround. - distinct_returns_first_row_from_group_if_ordered: If `order_by` was applied before, the distinct operation will return the first row from each group. Guaranteed in the in-memory backend, but may not be supported by all databases. - date_time: Specifies if the backend supports date/time operations. - Config supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=True order_by_unicode_normalization_by_default=False case_insensitive_ascii_only=False take_drop=True allows_mixed_type_comparisons=True supports_unicode_normalization=False is_nan_and_nothing_distinct=True supports_full_join=True distinct_returns_first_row_from_group_if_ordered=True date_time=True + - fixed_length_text_columns: Specifies if the backend supports fixed + length text columns. + Config supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=True order_by_unicode_normalization_by_default=False case_insensitive_ascii_only=False take_drop=True allows_mixed_type_comparisons=True supports_unicode_normalization=False is_nan_and_nothing_distinct=True distinct_returns_first_row_from_group_if_ordered=True date_time=True fixed_length_text_columns=False spec setup = Core_Spec.spec setup Select_Columns_Spec.spec setup Column_Operations_Spec.spec setup Date_Time_Spec.spec setup + Cast_Spec.spec setup Aggregate_Spec.spec setup Filter_Spec.spec setup Missing_Values_Spec.spec setup diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index d3ca37004db2..7a17c709943b 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -181,7 +181,7 @@ run_tests connection db_name = Common_Spec.spec prefix connection postgres_specific_spec connection db_name - common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True order_by_unicode_normalization_by_default=True take_drop=False allows_mixed_type_comparisons=False + common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True order_by_unicode_normalization_by_default=True take_drop=False allows_mixed_type_comparisons=False fixed_length_text_columns=True aggregate_selection = Common_Table_Operations.Aggregate_Spec.Test_Selection.Config first_last_row_order=False aggregation_problems=False agg_in_memory_table = (enso_project.data / "data.csv") . read agg_table = connection.upload_table (Name_Generator.random_name "Agg1") agg_in_memory_table diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index cc0e7efb942a..fae2e551beb8 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -120,7 +120,7 @@ sqlite_spec connection prefix = Common_Spec.spec prefix connection sqlite_specific_spec prefix connection - common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=False order_by=True natural_ordering=False case_insensitive_ordering=True case_insensitive_ascii_only=True take_drop=False is_nan_and_nothing_distinct=False supports_full_join=False date_time=False + common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=False order_by=True natural_ordering=False case_insensitive_ordering=True case_insensitive_ascii_only=True take_drop=False is_nan_and_nothing_distinct=False date_time=False ## For now `advanced_stats`, `first_last`, `text_shortest_longest` and `multi_distinct` remain disabled, because SQLite does not provide the diff --git a/test/Table_Tests/src/Database/Types/Postgres_Type_Mapping_Spec.enso b/test/Table_Tests/src/Database/Types/Postgres_Type_Mapping_Spec.enso index 537c37409a83..0e03fe4b6d9e 100644 --- a/test/Table_Tests/src/Database/Types/Postgres_Type_Mapping_Spec.enso +++ b/test/Table_Tests/src/Database/Types/Postgres_Type_Mapping_Spec.enso @@ -44,8 +44,8 @@ spec connection db_name = t3.at "b" . value_type . should_equal Value_Type.Decimal t3.at "c" . value_type . should_equal (Value_Type.Decimal precision=10 scale=2) t3.at "d" . value_type . should_equal (Value_Type.Decimal precision=20 scale=4) - t3.at "e" . value_type . should_equal (Value_Type.Decimal precision=10) - t3.at "f" . value_type . should_equal (Value_Type.Decimal precision=20) + t3.at "e" . value_type . should_equal (Value_Type.Decimal precision=10 scale=0) + t3.at "f" . value_type . should_equal (Value_Type.Decimal precision=20 scale=0) Test.specify "text" <| t = make_table "texts" [["a", "char(10)"], ["b", "varchar"], ["c", "varchar(20)"], ["d", "text"]] diff --git a/test/Table_Tests/src/Formatting/Data_Formatter_Spec.enso b/test/Table_Tests/src/Formatting/Data_Formatter_Spec.enso index 6875b735382a..3e58c6d23c32 100644 --- a/test/Table_Tests/src/Formatting/Data_Formatter_Spec.enso +++ b/test/Table_Tests/src/Formatting/Data_Formatter_Spec.enso @@ -127,6 +127,27 @@ spec = And newlines toO! formatter.parse complex_text . should_equal complex_text + Test.specify "should report Invalid_Format errors" <| + formatter = Data_Formatter.Value + expect_warning r = + r.should_equal Nothing + Problems.expect_only_warning Invalid_Format r + + r1 = formatter.parse "Text" datatype=Decimal + w1 = expect_warning r1 + w1.value_type . should_equal Decimal + w1.column . should_equal Nothing + + expect_warning <| formatter.parse "Text" datatype=Integer + expect_warning <| formatter.parse "Text" datatype=Boolean + expect_warning <| formatter.parse "Text" datatype=Date + expect_warning <| formatter.parse "Text" datatype=Date_Time + expect_warning <| formatter.parse "Text" datatype=Time_Of_Day + + Test.specify "should not allow unexpected types" <| + formatter = Data_Formatter.Value + formatter.parse "Text" datatype=List . should_fail_with Illegal_Argument + Test.group "DataFormatter.format" <| Test.specify "should handle Nothing" <| Data_Formatter.Value.format Nothing . should_equal Nothing diff --git a/test/Table_Tests/src/Formatting/Parse_Values_Spec.enso b/test/Table_Tests/src/Formatting/Parse_Values_Spec.enso index 01836f4a46df..ff6ef4a29b67 100644 --- a/test/Table_Tests/src/Formatting/Parse_Values_Spec.enso +++ b/test/Table_Tests/src/Formatting/Parse_Values_Spec.enso @@ -11,24 +11,24 @@ import Standard.Test.Extensions import project.Util spec = - Test.group "Table.parse_values" <| + Test.group "Table.parse" <| Test.specify "should correctly parse integers" <| t1 = Table.new [["ints", ["0", "+0", "-0", "+1", "-1", "1", "000", "0010", "12345", Nothing]]] - t2 = t1.parse_values type=Integer + t2 = t1.parse type=Value_Type.Integer t2.at "ints" . to_vector . should_equal [0, 0, 0, 1, -1, 1, Nothing, Nothing, 12345, Nothing] Test.specify "should correctly parse decimals" <| t1 = Table.new [["ints", ["0", "+0", "-0", "+1", "-1", "1", "12345", Nothing]]] - t2 = t1.parse_values type=Decimal + t2 = t1.parse type=Value_Type.Float t2.at "ints" . to_vector . should_equal [0, 0, 0, 1, -1, 1, 12345, Nothing] t2.at "ints" . to_vector . map .to_text . should_equal ["0.0", "0.0", "-0.0", "1.0", "-1.0", "1.0", "12345.0", "Nothing"] t3 = Table.new [["floats", ["0.0", "+0.0", "-0.0", "+1.0", "-1.0", "1.0", "0.0000", "10.", "12345."]]] - t4 = t3.parse_values type=Decimal + t4 = t3.parse type=Value_Type.Float t4.at "floats" . to_vector . should_equal [0, 0, 0, 1, -1, 1, 0, 10, 12345] t5 = Table.new [["floats", [".0", "0.", "1.", ".1", ".123", "-.1", "+.1", "+0.0", "0.1234", Nothing, "11111111.111"]]] - t6 = t5.parse_values type=Decimal + t6 = t5.parse type=Value_Type.Float t6.at "floats" . to_vector . should_equal [0.0, 0.0, 1.0, 0.1, 0.123, -0.1, 0.1, 0.0, 0.1234, Nothing, 11111111.111] Test.specify "should warn on leading zeros in numbers, if asked" <| @@ -37,56 +37,56 @@ spec = t1_parsed = [0, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, 12345, Nothing] t1_zeros = ["+00", "-00", "+01", "-01", "01", "000", "0010"] - t3 = t1.parse_values type=Integer + t3 = t1.parse type=Value_Type.Integer t3.at "ints" . to_vector . should_equal t1_parsed - Problems.get_attached_warnings t3 . should_equal [Leading_Zeros.Error "ints" Integer t1_zeros] + Problems.get_attached_warnings t3 . should_equal [Leading_Zeros.Error "ints" Value_Type.Integer t1_zeros] - t4 = t1.parse_values type=Decimal + t4 = t1.parse type=Value_Type.Float t4.at "ints" . to_vector . should_equal t1_parsed - Problems.get_attached_warnings t4 . should_equal [Leading_Zeros.Error "ints" Decimal t1_zeros] + Problems.get_attached_warnings t4 . should_equal [Leading_Zeros.Error "ints" Value_Type.Float t1_zeros] - t5 = t2.parse_values type=Decimal + t5 = t2.parse type=Value_Type.Float t5.at "floats" . to_vector . should_equal [0.0, 0.0, Nothing, Nothing, Nothing, 1.0] - Problems.get_attached_warnings t5 . should_equal [Leading_Zeros.Error "floats" Decimal ["00.", "01.0", '-0010.0000']] + Problems.get_attached_warnings t5 . should_equal [Leading_Zeros.Error "floats" Value_Type.Float ["00.", "01.0", '-0010.0000']] opts = Data_Formatter.Value allow_leading_zeros=True t1_parsed_zeros = [0, 0, 0, 1, -1, 1, 0, 10, 12345, Nothing] - t6 = t1.parse_values format=opts type=Integer + t6 = t1.parse format=opts type=Value_Type.Integer t6.at "ints" . to_vector . should_equal t1_parsed_zeros Problems.assume_no_problems t6 - t7 = t1.parse_values format=opts type=Decimal + t7 = t1.parse format=opts type=Value_Type.Float t7.at "ints" . to_vector . should_equal t1_parsed_zeros Problems.assume_no_problems t7 - t8 = t2.parse_values format=opts type=Decimal + t8 = t2.parse format=opts type=Value_Type.Float t8.at "floats" . to_vector . should_equal [0.0, 0.0, 0.0, 1.0, -10.0, 1.0] Problems.assume_no_problems t8 Test.specify "should correctly parse booleans" <| t1 = Table.new [["bools", ["true", "false", "True", "TRUE", "FALSE", Nothing, "False"]]] - t2 = t1.parse_values type=Boolean + t2 = t1.parse type=Value_Type.Boolean t2.at "bools" . to_vector . should_equal [True, False, True, True, False, Nothing, False] t3 = Table.new [["bools", ["1", "0", "true", "yes", "oui", "no", "NO!"]]] - t4 = t3.parse_values type=Boolean format="yes|no" + t4 = t3.parse type=Value_Type.Boolean format="yes|no" t4.at "bools" . to_vector . should_equal [Nothing, Nothing, Nothing, True, Nothing, False, Nothing] Test.specify "should correctly parse date and time" <| t1 = Table.new [["dates", ["2022-05-07", "2000-01-01", "2010-12-31"]]] - t2 = t1.parse_values type=Date + t2 = t1.parse type=Value_Type.Date t2.at "dates" . to_vector . should_equal [Date.new 2022 5 7, Date.new 2000 1 1, Date.new 2010 12 31] t3 = Table.new [["datetimes", ["2022-05-07 23:59:59", "2000-01-01 00:00:00", "2010-12-31 12:34:56"]]] - t4 = t3.parse_values type=Date_Time + t4 = t3.parse type=Value_Type.Date_Time t4.at "datetimes" . to_vector . should_equal [Date_Time.new 2022 5 7 23 59 59, Date_Time.new 2000 1 1, Date_Time.new 2010 12 31 12 34 56] t5 = Table.new [["times", ["23:59:59", "00:00:00", "12:34:56"]]] - t6 = t5.parse_values type=Time_Of_Day + t6 = t5.parse type=Value_Type.Time t6.at "times" . to_vector . should_equal [Time_Of_Day.new 23 59 59, Time_Of_Day.new, Time_Of_Day.new 12 34 56] t7 = Table.new [["dates", ["07/05/2022", "01/01/2001", "31/12/2010"]]] - t8 = t7.parse_values type=Date format="dd/MM/yyyy" + t8 = t7.parse type=Value_Type.Date format="dd/MM/yyyy" t8.at "dates" . value_type . should_equal Value_Type.Date t8.at "dates" . to_vector . should_equal [Date.new 2022 5 7, Date.new 2001 1 1, Date.new 2010 12 31] @@ -94,15 +94,15 @@ spec = opts = Data_Formatter.Value date_formats=["d.M.y", "d MMM y[ G]", "E, d MMM y"] datetime_formats=["yyyy-MM-dd'T'HH:mm:ss", "dd/MM/yyyy HH:mm"] time_formats=["H:mm:ss.n", "h:mma"] t1 = Table.new [["dates", ["1.2.476", "10 Jan 1900 AD", "Tue, 3 Jun 2008"]]] - t2 = t1.parse_values format=opts type=Date + t2 = t1.parse format=opts type=Value_Type.Date t2.at "dates" . to_vector . should_equal [Date.new 476 2 1, Date.new 1900 1 10, Date.new 2008 6 3] t3 = Table.new [["datetimes", ["2011-12-03T10:15:30", "31/12/2012 22:33"]]] - t4 = t3.parse_values format=opts type=Date_Time + t4 = t3.parse format=opts type=Value_Type.Date_Time t4.at "datetimes" . to_vector . should_equal [Date_Time.new 2011 12 3 10 15 30, Date_Time.new 2012 12 31 22 33] t5 = Table.new [["times", ["1:02:03.987654321", "1:30PM"]]] - t6 = t5.parse_values format=opts type=Time_Of_Day + t6 = t5.parse format=opts type=Value_Type.Time t6.at "times" . to_vector . should_equal [Time_Of_Day.new 1 2 3 nanosecond=987654321, Time_Of_Day.new 13 30 0 0] Test.specify "should warn when cells do not fit the expected format" <| @@ -113,51 +113,51 @@ spec = times = ["2001-01-01", "2001-01-01 12:34:56", "10:00:10", "Tuesday", "foobar", "", "10:99:99", "1/2/2003", "2001-30-10"] t = Table.new [ints, floats, bools, ["times", times]] - t0 = t.parse_values type=Boolean + t0 = t.parse type=Value_Type.Boolean t0.at "bools" . to_vector . should_equal [True, False, Nothing, Nothing, Nothing, Nothing, Nothing, True, Nothing] t0.at "ints" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing] - Problems.expect_warning (Invalid_Format.Error "bools" Boolean ["fAlSE", "foobar", "", "0", "1", "truefalse"]) t0 - Problems.expect_warning (Invalid_Format.Error "ints" Boolean ["0", "1", "1.0", "foobar", "", "--1", "+-1", "10", "-+1"]) t0 + Problems.expect_warning (Invalid_Format.Error "bools" Value_Type.Boolean ["fAlSE", "foobar", "", "0", "1", "truefalse"]) t0 + Problems.expect_warning (Invalid_Format.Error "ints" Value_Type.Boolean ["0", "1", "1.0", "foobar", "", "--1", "+-1", "10", "-+1"]) t0 - a1 = t.parse_values columns=["ints"] type=Integer on_problems=_ + a1 = t.parse columns=["ints"] type=Value_Type.Integer on_problems=_ t1 t = t.at "ints" . to_vector . should_equal [0, 1, Nothing, Nothing, Nothing, Nothing, Nothing, 10, Nothing] - p1 = [Invalid_Format.Error "ints" Integer ["1.0", "foobar", "", "--1", "+-1", "-+1"]] + p1 = [Invalid_Format.Error "ints" Value_Type.Integer ["1.0", "foobar", "", "--1", "+-1", "-+1"]] Problems.test_problem_handling a1 p1 t1 - a2 = t.parse_values columns=["floats"] type=Decimal on_problems=_ + a2 = t.parse columns=["floats"] type=Value_Type.Float on_problems=_ t2 t = t.at "floats" . to_vector . should_equal [0, 2, Nothing, Nothing, Nothing, Nothing, Nothing, 100, Nothing] - p2 = [Invalid_Format.Error "floats" Decimal ["1e6", "foobar", "", "--1", "+-1", "-+1"]] + p2 = [Invalid_Format.Error "floats" Value_Type.Float ["1e6", "foobar", "", "--1", "+-1", "-+1"]] Problems.test_problem_handling a2 p2 t2 - a3 = t.parse_values columns=["bools"] type=Boolean on_problems=_ + a3 = t.parse columns=["bools"] type=Value_Type.Boolean on_problems=_ t3 t = t.at "bools" . to_vector . should_equal [True, False, Nothing, Nothing, Nothing, Nothing, Nothing, True, Nothing] - p3 = [Invalid_Format.Error "bools" Boolean ["fAlSE", "foobar", "", "0", "1", "truefalse"]] + p3 = [Invalid_Format.Error "bools" Value_Type.Boolean ["fAlSE", "foobar", "", "0", "1", "truefalse"]] Problems.test_problem_handling a3 p3 t3 - a4 = t.parse_values columns=["times"] type=Date on_problems=_ + a4 = t.parse columns=["times"] type=Value_Type.Date on_problems=_ t4 t = t.at "times" . to_vector . should_equal [Date.new 2001 1 1, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing] - p4 = [Invalid_Format.Error "times" Date ["2001-01-01 12:34:56", "10:00:10", "Tuesday", "foobar", "", "10:99:99", "1/2/2003", "2001-30-10"]] + p4 = [Invalid_Format.Error "times" Value_Type.Date ["2001-01-01 12:34:56", "10:00:10", "Tuesday", "foobar", "", "10:99:99", "1/2/2003", "2001-30-10"]] Problems.test_problem_handling a4 p4 t4 - a5 = t.parse_values columns=["times"] type=Date_Time on_problems=_ + a5 = t.parse columns=["times"] type=Value_Type.Date_Time on_problems=_ t5 t = t.at "times" . to_vector . should_equal [Nothing, Date_Time.new 2001 1 1 12 34 56, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing] - p5 = [Invalid_Format.Error "times" Date_Time ["2001-01-01", "10:00:10", "Tuesday", "foobar", "", "10:99:99", "1/2/2003", "2001-30-10"]] + p5 = [Invalid_Format.Error "times" Value_Type.Date_Time ["2001-01-01", "10:00:10", "Tuesday", "foobar", "", "10:99:99", "1/2/2003", "2001-30-10"]] Problems.test_problem_handling a5 p5 t5 - a6 = t.parse_values columns=["times"] type=Time_Of_Day on_problems=_ + a6 = t.parse columns=["times"] type=Value_Type.Time on_problems=_ t6 t = t.at "times" . to_vector . should_equal [Nothing, Nothing, Time_Of_Day.new 10 0 10 0, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing] - p6 = [Invalid_Format.Error "times" Time_Of_Day ["2001-01-01", "2001-01-01 12:34:56", "Tuesday", "foobar", "", "10:99:99", "1/2/2003", "2001-30-10"]] + p6 = [Invalid_Format.Error "times" Value_Type.Time ["2001-01-01", "2001-01-01 12:34:56", "Tuesday", "foobar", "", "10:99:99", "1/2/2003", "2001-30-10"]] Problems.test_problem_handling a6 p6 t6 Test.specify "should leave not selected columns unaffected" <| t1 = Table.new [["A", ["1", "2"]], ["B", ["3", "4"]]] - t2 = t1.parse_values columns="B" + t2 = t1.parse columns="B" t2.at "A" . to_vector . should_equal ["1", "2"] t2.at "B" . to_vector . should_equal [3, 4] @@ -174,7 +174,7 @@ spec = c10 = ["mixeddates", ["2022-10-01", "2000-01-01 01:02:03", "01:02:03", Nothing]] c11 = ["text+ints", ["1", "2", " foobar", Nothing]] t = Table.new [c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11] - t2 = t.parse_values + t2 = t.parse Problems.assume_no_problems t2 t2.at "ints" . to_vector . should_equal [1, 2, -123, Nothing] @@ -191,19 +191,19 @@ spec = t2.at "text+ints" . to_vector . should_equal ["1", "2", "foobar", Nothing] # In Auto mode, integers take precedence over booleans. - t3 = Table.new [["bools", ["1", "0", "True"]], ["ints", ["1", "0", "0"]]] . parse_values format=(Data_Formatter.Value true_values=["1", "True"] false_values=["0", "False"]) + t3 = Table.new [["bools", ["1", "0", "True"]], ["ints", ["1", "0", "0"]]] . parse format=(Data_Formatter.Value true_values=["1", "True"] false_values=["0", "False"]) t3.at "bools" . to_vector . should_equal [True, False, True] t3.at "ints" . to_vector . should_equal [1, 0, 0] - t4 = Table.new [c2] . parse_values format=(Data_Formatter.Value allow_leading_zeros=True) + t4 = Table.new [c2] . parse format=(Data_Formatter.Value allow_leading_zeros=True) t4 . at "ints0" . to_vector . should_equal [1, 2, Nothing, -1] - t5 = t.parse_values columns="ints" type=Decimal + t5 = t.parse columns="ints" type=Value_Type.Float t5.at "ints" . to_vector . should_equal [1.0, 2.0, -123.0, Nothing] # `ints` are requested to be parsed as decimals. t5.at "ints" . to_vector . first . should_be_a Decimal - t6 = t.parse_values columns=["floats", "text+ints"] type=Auto + t6 = t.parse columns=["floats", "text+ints"] type=Auto # `floats` are auto-detected as decimals. t6.at "floats" . to_vector . should_equal [1.0, 2.2, Nothing, -1.0] # `text+ints` is attempted to be parsed (hence whitespace is stripped), but it only fits the text type. @@ -214,27 +214,27 @@ spec = Test.specify "should allow to specify a thousands separator and a custom decimal point" <| opts = Data_Formatter.Value decimal_point=',' thousand_separator='_' t1 = Table.new [["floats", ["0,0", "+0,0", "-0,0", "+1,5", "-1,2", "1,0", "0,0000", "10_000,", ",0"]]] - t2 = t1.parse_values format=opts + t2 = t1.parse format=opts t2.at "floats" . to_vector . should_equal [0.0, 0.0, 0.0, 1.5, -1.2, 1.0, 0.0, 10000.0, 0.0] t3 = Table.new [["xs", ["1,2", "1.3", "_0", "0_", "1_0_0"]]] - t4 = t3.parse_values format=opts type=Decimal + t4 = t3.parse format=opts type=Value_Type.Float t4.at "xs" . to_vector . should_equal [1.2, Nothing, Nothing, Nothing, 100.0] - Problems.get_attached_warnings t4 . should_equal [Invalid_Format.Error "xs" Decimal ["1.3", "_0", "0_"]] - t5 = t3.parse_values format=opts type=Integer + Problems.get_attached_warnings t4 . should_equal [Invalid_Format.Error "xs" Value_Type.Float ["1.3", "_0", "0_"]] + t5 = t3.parse format=opts type=Value_Type.Integer t5.at "xs" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing, 100] - Problems.get_attached_warnings t5 . should_equal [Invalid_Format.Error "xs" Integer ["1,2", "1.3", "_0", "0_"]] + Problems.get_attached_warnings t5 . should_equal [Invalid_Format.Error "xs" Value_Type.Integer ["1,2", "1.3", "_0", "0_"]] Test.specify "should allow to specify custom values for booleans" <| opts_1 = Data_Formatter.Value true_values=["1", "YES"] false_values=["0"] t1 = Table.new [["bools", ["1", "0", "YES", "1", "0"]]] - t2 = t1.parse_values format=opts_1 + t2 = t1.parse format=opts_1 t2.at "bools" . to_vector . should_equal [True, False, True, True, False] t3 = Table.new [["bools", ["1", "NO", "False", "True", "YES", "no", "oui", "0"]]] - t4 = t3.parse_values format=opts_1 type=Boolean + t4 = t3.parse format=opts_1 type=Value_Type.Boolean t4.at "bools" . to_vector . should_equal [True, Nothing, Nothing, Nothing, True, Nothing, Nothing, False] - Problems.get_attached_warnings t4 . should_equal [Invalid_Format.Error "bools" Boolean ["NO", "False", "True", "no", "oui"]] + Problems.get_attached_warnings t4 . should_equal [Invalid_Format.Error "bools" Value_Type.Boolean ["NO", "False", "True", "no", "oui"]] whitespace_table = ints = ["ints", ["0", "1 ", "0 1", " 2"]] @@ -246,62 +246,62 @@ spec = Table.new [ints, floats, bools, dates, datetimes, times] Test.specify "should trim input values by default" <| - t1 = whitespace_table.parse_values columns="ints" type=Integer + t1 = whitespace_table.parse columns="ints" type=Value_Type.Integer t1.at "ints" . to_vector . should_equal [0, 1, Nothing, 2] - Problems.expect_only_warning (Invalid_Format.Error "ints" Integer ["0 1"]) t1 + Problems.expect_only_warning (Invalid_Format.Error "ints" Value_Type.Integer ["0 1"]) t1 - t2 = whitespace_table.parse_values columns="floats" type=Decimal + t2 = whitespace_table.parse columns="floats" type=Value_Type.Float t2.at "floats" . to_vector . should_equal [0.0, 2.0, Nothing, 10.0] - Problems.expect_only_warning (Invalid_Format.Error "floats" Decimal ["- 1"]) t2 + Problems.expect_only_warning (Invalid_Format.Error "floats" Value_Type.Float ["- 1"]) t2 - t3 = whitespace_table.parse_values columns="bools" type=Boolean + t3 = whitespace_table.parse columns="bools" type=Value_Type.Boolean t3.at "bools" . to_vector . should_equal [True, False, Nothing, False] - Problems.expect_only_warning (Invalid_Format.Error "bools" Boolean ["t rue"]) t3 + Problems.expect_only_warning (Invalid_Format.Error "bools" Value_Type.Boolean ["t rue"]) t3 - t4 = whitespace_table.parse_values columns="dates" type=Date + t4 = whitespace_table.parse columns="dates" type=Value_Type.Date t4.at "dates" . to_vector . should_equal [Date.new 2022 1 1, Date.new 2022 7 17, Nothing, Nothing] - Problems.expect_only_warning (Invalid_Format.Error "dates" Date ["2022 - 07 - 17", ""]) t4 + Problems.expect_only_warning (Invalid_Format.Error "dates" Value_Type.Date ["2022 - 07 - 17", ""]) t4 - t5 = whitespace_table.parse_values columns="datetimes" type=Date_Time + t5 = whitespace_table.parse columns="datetimes" type=Value_Type.Date_Time t5.at "datetimes" . to_vector . should_equal [Date_Time.new 2022 1 1 11 59, Nothing, Nothing, Nothing] - Problems.expect_only_warning (Invalid_Format.Error "datetimes" Date_Time ["2022 - 07 - 17 1:2:3", "2022-01-01 11:59:00"]) t5 + Problems.expect_only_warning (Invalid_Format.Error "datetimes" Value_Type.Date_Time ["2022 - 07 - 17 1:2:3", "2022-01-01 11:59:00"]) t5 - t6 = whitespace_table.parse_values columns="times" type=Time_Of_Day + t6 = whitespace_table.parse columns="times" type=Value_Type.Time t6.at "times" . to_vector . should_equal [Time_Of_Day.new 11 0 0, Time_Of_Day.new, Nothing, Nothing] - Problems.expect_only_warning (Invalid_Format.Error "times" Time_Of_Day ["00 : 00 : 00"]) t6 + Problems.expect_only_warning (Invalid_Format.Error "times" Value_Type.Time ["00 : 00 : 00"]) t6 Test.specify "should fail to parse if whitespace is present and trimming is turned off" <| opts = Data_Formatter.Value trim_values=False - t1 = whitespace_table.parse_values format=opts columns="ints" type=Integer + t1 = whitespace_table.parse format=opts columns="ints" type=Value_Type.Integer t1.at "ints" . to_vector . should_equal [0, Nothing, Nothing, Nothing] - Problems.expect_only_warning (Invalid_Format.Error "ints" Integer ["1 ", "0 1", " 2"]) t1 + Problems.expect_only_warning (Invalid_Format.Error "ints" Value_Type.Integer ["1 ", "0 1", " 2"]) t1 - t2 = whitespace_table.parse_values format=opts columns="floats" type=Decimal + t2 = whitespace_table.parse format=opts columns="floats" type=Value_Type.Float t2.at "floats" . to_vector . should_equal [Nothing, Nothing, Nothing, 10.0] - Problems.expect_only_warning (Invalid_Format.Error "floats" Decimal ["0 ", " 2.0", "- 1"]) t2 + Problems.expect_only_warning (Invalid_Format.Error "floats" Value_Type.Float ["0 ", " 2.0", "- 1"]) t2 - t3 = whitespace_table.parse_values format=opts columns="bools" type=Boolean + t3 = whitespace_table.parse format=opts columns="bools" type=Value_Type.Boolean t3.at "bools" . to_vector . should_equal [Nothing, Nothing, Nothing, False] - Problems.expect_only_warning (Invalid_Format.Error "bools" Boolean ["True ", " false", "t rue"]) t3 + Problems.expect_only_warning (Invalid_Format.Error "bools" Value_Type.Boolean ["True ", " false", "t rue"]) t3 - t4 = whitespace_table.parse_values format=opts columns="dates" type=Date + t4 = whitespace_table.parse format=opts columns="dates" type=Value_Type.Date t4.at "dates" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing] - Problems.expect_only_warning (Invalid_Format.Error "dates" Date [" 2022-01-01", "2022-07-17 ", "2022 - 07 - 17", ""]) t4 + Problems.expect_only_warning (Invalid_Format.Error "dates" Value_Type.Date [" 2022-01-01", "2022-07-17 ", "2022 - 07 - 17", ""]) t4 - t5 = whitespace_table.parse_values format=opts columns="datetimes" type=Date_Time + t5 = whitespace_table.parse format=opts columns="datetimes" type=Value_Type.Date_Time t5.at "datetimes" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing] - Problems.expect_only_warning (Invalid_Format.Error "datetimes" Date_Time [" 2022-01-01 11:59:00 ", "2022 - 07 - 17 1:2:3 ", "2022-01-01 11:59:00"]) t5 + Problems.expect_only_warning (Invalid_Format.Error "datetimes" Value_Type.Date_Time [" 2022-01-01 11:59:00 ", "2022 - 07 - 17 1:2:3 ", "2022-01-01 11:59:00"]) t5 - t6 = whitespace_table.parse_values format=opts columns="times" type=Time_Of_Day + t6 = whitespace_table.parse format=opts columns="times" type=Value_Type.Time t6.at "times" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing] - Problems.expect_only_warning (Invalid_Format.Error "times" Time_Of_Day ["11:00:00 ", " 00:00:00", "00 : 00 : 00"]) t6 + Problems.expect_only_warning (Invalid_Format.Error "times" Value_Type.Time ["11:00:00 ", " 00:00:00", "00 : 00 : 00"]) t6 Test.specify "should fallback to text if whitespace is present and trimming is turned off" <| c1 = ["1", " +2", "-123", Nothing] c2 = [" 1.0 ", "2.2", Nothing, "-1.0"] c3 = ["true", " False", Nothing, "True"] t = Table.new [["ints", c1], ["floats", c2], ["bools", c3]] - t2 = t.parse_values format=(Data_Formatter.Value trim_values=False) + t2 = t.parse format=(Data_Formatter.Value trim_values=False) Warning.get_all t2 . should_equal [] t2.at "ints" . to_vector . should_equal c1 @@ -310,7 +310,7 @@ spec = Test.specify "should allow selecting columns by regex" <| t1 = Table.new [["An", ["1", "2", "3"]], ["Am", ["4", "5", "6"]], ["C", ["7", "8", "9"]], ["D", ["10", "11", "12"]]] - r1 = t1.parse_values columns=[Column_Selector.By_Name "A.*" use_regex=True] + r1 = t1.parse columns=[Column_Selector.By_Name "A.*" use_regex=True] r1.at "An" . to_vector . should_equal [1, 2, 3] r1.at "Am" . to_vector . should_equal [4, 5, 6] r1.at "C" . to_vector . should_equal ["7", "8", "9"] @@ -318,15 +318,15 @@ spec = Test.specify "should correctly handle problems: missing input columns" <| t1 = Table.new [["A", ["1", "2", "3"]]] - r1 = t1.parse_values columns=["A", "B", "C", "E"] on_problems=Problem_Behavior.Ignore + r1 = t1.parse columns=["A", "B", "C", "E"] on_problems=Problem_Behavior.Ignore r1.should_fail_with Missing_Input_Columns r1.catch.criteria . should_equal ["B", "C", "E"] - r2 = t1.parse_values columns=[Column_Selector.By_Name "A.+" use_regex=True] + r2 = t1.parse columns=[Column_Selector.By_Name "A.+" use_regex=True] r2.should_fail_with Missing_Input_Columns r2.catch.criteria . should_equal ["A.+"] - action = t1.parse_values columns=["A", "B", "C", "E"] error_on_missing_columns=False on_problems=_ + action = t1.parse columns=["A", "B", "C", "E"] error_on_missing_columns=False on_problems=_ tester table = table.at "A" . to_vector . should_equal [1, 2, 3] problems = [Missing_Input_Columns.Error ["B", "C", "E"]] @@ -334,11 +334,11 @@ spec = Test.specify "should correctly handle problems: out of bounds indices" <| t1 = Table.new [["A", ["1", "2", "3"]]] - r1 = t1.parse_values columns=[0, -1, 42, -5] + r1 = t1.parse columns=[0, -1, 42, -5] r1.should_fail_with Column_Indexes_Out_Of_Range r1.catch.indexes . should_equal [42, -5] - action = t1.parse_values columns=[0, -1, 42, -5] error_on_missing_columns=False on_problems=_ + action = t1.parse columns=[0, -1, 42, -5] error_on_missing_columns=False on_problems=_ tester table = table.at "A" . to_vector . should_equal [1, 2, 3] problems = [Column_Indexes_Out_Of_Range.Error [42, -5]] @@ -346,7 +346,7 @@ spec = Test.specify "should allow mixed column selectors" <| t1 = Table.new [["Am", ["1", "2", "3"]], ["B", ["4", "5", "6"]], ["C", ["7", "8", "9"]], ["D", ["10", "11", "12"]]] - r1 = t1.parse_values columns=[(Column_Selector.By_Name "A.*" use_regex=True), -2, "D"] + r1 = t1.parse columns=[(Column_Selector.By_Name "A.*" use_regex=True), -2, "D"] r1.at "Am" . to_vector . should_equal [1, 2, 3] r1.at "B" . to_vector . should_equal ["4", "5", "6"] r1.at "C" . to_vector . should_equal [7, 8, 9] @@ -354,7 +354,7 @@ spec = Test.specify "should handle edge-cases: overlapping selectors" <| t1 = Table.new [["Am", ["1", "2", "3"]], ["B", ["4", "5", "6"]], ["C", ["7", "8", "9"]], ["D", ["10", "11", "12"]]] - r1 = t1.parse_values columns=[(Column_Selector.By_Name "A.*" use_regex=True), 0, "D", -1, -1, 0, 3] + r1 = t1.parse columns=[(Column_Selector.By_Name "A.*" use_regex=True), 0, "D", -1, -1, 0, 3] r1.at "Am" . to_vector . should_equal [1, 2, 3] r1.at "B" . to_vector . should_equal ["4", "5", "6"] r1.at "C" . to_vector . should_equal ["7", "8", "9"] @@ -362,29 +362,29 @@ spec = Test.specify "should error if invalid target type is provided" <| t1 = Table.new [["A", ["1", "2", "3"]]] - t1.parse_values type=Nothing . should_fail_with Illegal_Argument + t1.parse type=Nothing . should_fail_with Illegal_Argument Test.specify "should error if the input column is not text" <| t1 = Table.new [["A", [1, 2, 3]], ["B", ["4", "5", "6"]], ["C", [7, 8, 9]], ["D", ["10", "11", "12"]]] - r1 = t1.parse_values columns=["A", "B", "C"] + r1 = t1.parse columns=["A", "B", "C"] r1.should_fail_with Invalid_Value_Type r1.catch.related_column . should_equal "A" r1.catch.expected.is_text.should_be_true Test.specify "should error if no input columns selected, unless error_on_missing_columns=False" <| t1 = Table.new [["A", ["1", "2", "3"]]] - r1 = t1.parse_values columns=[] + r1 = t1.parse columns=[] r1.should_fail_with No_Input_Columns_Selected - r2 = t1.parse_values columns=[] error_on_missing_columns=False + r2 = t1.parse columns=[] error_on_missing_columns=False r2 . should_equal t1 Problems.expect_warning No_Input_Columns_Selected r2 - r3 = t1.parse_values columns=[] error_on_missing_columns=False on_problems=Problem_Behavior.Ignore + r3 = t1.parse columns=[] error_on_missing_columns=False on_problems=Problem_Behavior.Ignore r3 . should_equal t1 Problems.assume_no_problems r3 - r4 = t1.parse_values columns=["nonexistent column :D", -42] error_on_missing_columns=False on_problems=Problem_Behavior.Report_Warning + r4 = t1.parse columns=["nonexistent column :D", -42] error_on_missing_columns=False on_problems=Problem_Behavior.Report_Warning r4 . should_equal t1 Problems.expect_warning No_Input_Columns_Selected r4 Problems.expect_warning (Missing_Input_Columns.Error ["nonexistent column :D"]) r4 @@ -393,87 +393,93 @@ spec = Test.group "Column.parse" <| Test.specify "should correctly parse integers" <| c1 = Column.from_vector "ints" ["0", "+0", "-0", "+1", "-1", "1", "000", "0010", "12345", Nothing] - c2 = c1.parse Integer + c2 = c1.parse type=Value_Type.Integer c2.name.should_equal c1.name c2 . to_vector . should_equal [0, 0, 0, 1, -1, 1, Nothing, Nothing, 12345, Nothing] + c2.value_type.should_equal Value_Type.Integer Problems.expect_warning Leading_Zeros c2 - c3 = c1.parse Integer format=(Data_Formatter.Value.with_number_formatting allow_leading_zeros=True) + c3 = c1.parse type=Value_Type.Integer format=(Data_Formatter.Value.with_number_formatting allow_leading_zeros=True) c3.to_vector . should_equal [0, 0, 0, 1, -1, 1, 0, 10, 12345, Nothing] Problems.assume_no_problems c3 Test.specify "should correctly parse decimals" <| c1 = Column.from_vector "ints" ["0", "+0", "-0", "+1", "-1", "1", "000", "0010", "12345", Nothing] - c2 = c1.parse Decimal + c2 = c1.parse Value_Type.Float c2.name.should_equal c1.name - c2 . to_vector . should_equal [0, 0, 0, 1, -1, 1, Nothing, Nothing, 12345, Nothing] + c2.to_vector . should_equal [0, 0, 0, 1, -1, 1, Nothing, Nothing, 12345, Nothing] + c2.value_type.should_equal Value_Type.Float c2.to_vector . map .to_text . should_equal ["0.0", "0.0", "-0.0", "1.0", "-1.0", "1.0", "Nothing", "Nothing", "12345.0", "Nothing"] Problems.expect_warning Leading_Zeros c2 c3 = Column.from_vector "floats" ["0.0", "+0.0", "-0.0", "+1.0", "-1.0", "1.0", "0.0000", "10.", "12345."] - c4 = c3.parse Decimal + c4 = c3.parse Value_Type.Float c4.to_vector . should_equal [0, 0, 0, 1, -1, 1, 0, 10, 12345] c4.value_type.is_floating_point.should_be_true Problems.assume_no_problems c4 c5 = Column.from_vector "floats" [".0", "0.", "1.", ".1", ".123", "-.1", "+.1", "+0.0", "0.1234", Nothing, "11111111.111"] - c6 = c5.parse Decimal + c6 = c5.parse Value_Type.Float c6.to_vector . should_equal [0.0, 0.0, 1.0, 0.1, 0.123, -0.1, 0.1, 0.0, 0.1234, Nothing, 11111111.111] Problems.assume_no_problems c6 Test.specify "should correctly parse booleans" <| c1 = Column.from_vector "bools" ["true", "false", "True", "TRUE", "FALSE", Nothing, "False"] - c2 = c1.parse Boolean + c2 = c1.parse type=Value_Type.Boolean c2.name.should_equal c1.name c2.to_vector . should_equal [True, False, True, True, False, Nothing, False] + c2.value_type.should_equal Value_Type.Boolean c1.parse . to_vector . should_equal [True, False, True, True, False, Nothing, False] c3 = Column.from_vector "bools" ["yes", "no", Nothing] - c4 = c3.parse Boolean "yes|no" + c4 = c3.parse type=Value_Type.Boolean "yes|no" c4.to_vector . should_equal [True, False, Nothing] c5 = Column.from_vector "bools" ["true", "yes", "false"] - c6 = c5.parse Boolean + c6 = c5.parse type=Value_Type.Boolean c6.to_vector . should_equal [True, Nothing, False] w = Problems.get_attached_warnings c6 . find w-> w.is_a Invalid_Format w.column.should_equal "bools" - w.datatype . should_equal Boolean + w.value_type . should_equal Value_Type.Boolean w.cells . should_equal ["yes"] Test.specify "should correctly parse date and time" <| c1 = Column.from_vector "date" ["2022-05-07", "2000-01-01", "2010-12-31"] - c2 = c1.parse Date + c2 = c1.parse type=Value_Type.Date c2.to_vector . should_equal [Date.new 2022 5 7, Date.new 2000 1 1, Date.new 2010 12 31] + c2.value_type.should_equal Value_Type.Date c3 = Column.from_vector "datetimes" ["2022-05-07 23:59:59", "2000-01-01 00:00:00", "2010-12-31 12:34:56", "2010-12-31T12:34:56", "2010-12-31 12:34:56.123"] - c4 = c3.parse Date_Time + c4 = c3.parse type=Value_Type.Date_Time c4.to_vector . should_equal [Date_Time.new 2022 5 7 23 59 59, Date_Time.new 2000 1 1, Date_Time.new 2010 12 31 12 34 56, Date_Time.new 2010 12 31 12 34 56, Date_Time.new 2010 12 31 12 34 56 123] + c4.value_type.should_equal Value_Type.Date_Time c5 = Column.from_vector "times" ["23:59:59", "00:00:00", "12:34:56"] - c6 = c5.parse Time_Of_Day + c6 = c5.parse type=Value_Type.Time c6.to_vector . should_equal [Time_Of_Day.new 23 59 59, Time_Of_Day.new, Time_Of_Day.new 12 34 56] + c6.value_type.should_equal Value_Type.Time c7 = Column.from_vector "foo" ["2022-05-07 23:59:59", "42", "2010-12-31"] - c8 = c7.parse Date_Time . to_vector . should_equal [Date_Time.new 2022 5 7 23 59 59, Nothing, Nothing] + c8 = c7.parse type=Value_Type.Date_Time . to_vector . should_equal [Date_Time.new 2022 5 7 23 59 59, Nothing, Nothing] w = Problems.get_attached_warnings c8 . find w-> w.is_a Invalid_Format w.column.should_equal "foo" - w.datatype . should_equal Date_Time + w.value_type . should_equal Value_Type.Date_Time w.cells . should_equal ["42", "2010-12-31"] Test.specify "should correctly parse date and time with format" <| c1 = Column.from_vector "date" ["5/7/2022", "1/1/2000", "12/31/2010"] - c2 = c1.parse Date "M/d/yyyy" + c2 = c1.parse type=Value_Type.Date "M/d/yyyy" c2.to_vector . should_equal [Date.new 2022 5 7, Date.new 2000 1 1, Date.new 2010 12 31] c3 = Column.from_vector "datetimes" ["5/7/2022 23:59:59", "1/1/2000 00:00:00", "12/31/2010 12:34:56"] - c4 = c3.parse Date_Time "M/d/yyyy HH:mm:ss" + c4 = c3.parse type=Value_Type.Date_Time "M/d/yyyy HH:mm:ss" c4.to_vector . should_equal [Date_Time.new 2022 5 7 23 59 59, Date_Time.new 2000 1 1, Date_Time.new 2010 12 31 12 34 56] Test.specify "should handle invalid format strings gracefully" <| c1 = Column.from_vector "date" ["5/7/2022", "1/1/2000", "12/31/2010"] - c1.parse Date "M/d/fqsrf" . should_fail_with Illegal_Argument - c1.parse Time_Of_Day "HH:mm:ss.fff" . should_fail_with Illegal_Argument - c1.parse Date_Time "M/d/fqsrf HH:mm:ss.fff" . should_fail_with Illegal_Argument + c1.parse type=Value_Type.Date "M/d/fqsrf" . should_fail_with Illegal_Argument + c1.parse type=Value_Type.Time "HH:mm:ss.fff" . should_fail_with Illegal_Argument + c1.parse type=Value_Type.Date_Time "M/d/fqsrf HH:mm:ss.fff" . should_fail_with Illegal_Argument Test.specify "should correctly work in Auto mode" <| c1 = Column.from_vector "A" ["1", "2", "3"] @@ -484,21 +490,26 @@ spec = c6 = Column.from_vector "F" ["this is here to ensure the column has type text... can be replaced one we have retyping"] c7 = Column.from_vector "G" ["true", "42"] c8 = Column.from_vector "H" ["text-to-force-value-type-to-be-text", Nothing, Nothing, Nothing] + c8.value_type . should_equal Value_Type.Char r1 = c1.parse r1.to_vector . should_equal [1, 2, 3] + r1.value_type.should_equal Value_Type.Integer Problems.assume_no_problems r1 r2 = c2.parse r2.to_vector . should_equal [1.0, 2.5, 3.0] + r2.value_type.should_equal Value_Type.Float Problems.assume_no_problems r2 r3 = c3.parse r3.to_vector . should_equal [Date.new 2022 5 7, Date.new 2000 1 1, Date.new 2010 12 31] + r3.value_type.should_equal Value_Type.Date Problems.assume_no_problems r3 r4 = c4.parse r4.to_vector . should_equal [True, False, Nothing] + r4.value_type.should_equal Value_Type.Boolean Problems.assume_no_problems r4 r5 = c5.parse @@ -508,25 +519,27 @@ spec = c5.parse format="yes|no" . should_fail_with Illegal_Argument r5_2 = c5.parse format=(Data_Formatter.Value.with_boolean_values ["yes"] ["no"]) r5_2.to_vector . should_equal [True, False] + r5_2.value_type . should_equal Value_Type.Boolean Problems.assume_no_problems r5_2 r6 = (c6.drop 1).parse r6.to_vector . should_equal [] - Test.with_clue "r6.value_type == "+r6.value_type.to_text+"; " <| - r6.value_type.is_text . should_be_true + r6.value_type . should_equal Value_Type.Char Problems.assume_no_problems r6 r7 = c7.parse r7.to_vector . should_equal ["true", "42"] + r7.value_type . should_equal Value_Type.Char Problems.assume_no_problems r7 r8 = c8.drop 1 . parse + r8.value_type . should_equal Value_Type.Char r8.to_vector . should_equal [Nothing, Nothing, Nothing] Problems.assume_no_problems r8 Test.specify "should error if invalid target type is provided" <| c1 = Column.from_vector "A" ["1", "2", "3"] - c1.parse Nothing . should_fail_with Illegal_Argument + c1.parse type=Nothing . should_fail_with Illegal_Argument Test.specify "should error if the input column is not text" <| c1 = Column.from_vector "A" [1, 2, 3] diff --git a/test/Table_Tests/src/Helpers/Value_Type_Spec.enso b/test/Table_Tests/src/Helpers/Value_Type_Spec.enso index 68a22d368f2d..e21101bba6e5 100644 --- a/test/Table_Tests/src/Helpers/Value_Type_Spec.enso +++ b/test/Table_Tests/src/Helpers/Value_Type_Spec.enso @@ -2,6 +2,7 @@ from Standard.Base import all import Standard.Table.Data.Type.Value_Type.Bits import Standard.Table.Data.Type.Value_Type.Value_Type +import Standard.Table.Data.Type.Value_Type_Helpers from Standard.Test import Test, Test_Suite import Standard.Test.Extensions @@ -14,7 +15,7 @@ spec = (Value_Type.Integer Bits.Bits_16).to_display_text . should_equal "Integer (16 bits)" Value_Type.Float.to_display_text . should_equal "Float (64 bits)" - Value_Type.Decimal.to_display_text . should_equal "Decimal (precision=Nothing, scale=0)" + Value_Type.Decimal.to_display_text . should_equal "Decimal (precision=Nothing, scale=Nothing)" Value_Type.Char.to_display_text . should_equal "Char (max_size=Nothing, variable_length=True)" (Value_Type.Binary 8 False).to_display_text . should_equal "Binary (max_size=8 bytes, variable_length=False)" @@ -27,4 +28,38 @@ spec = Value_Type.Unsupported_Data_Type.to_display_text . should_equal "Unsupported_Data_Type" (Value_Type.Unsupported_Data_Type "FOO-BAR").to_display_text . should_equal "Unsupported_Data_Type (FOO-BAR)" + Test.specify "should use correct in-memory logic to reconcile pairs of types for operations like union/iif" <| + Value_Type_Helpers.reconcile_types Value_Type.Boolean Value_Type.Boolean . should_equal Value_Type.Boolean + Value_Type_Helpers.reconcile_types Value_Type.Boolean Value_Type.Integer . should_equal Value_Type.Integer + + Value_Type_Helpers.reconcile_types (Value_Type.Integer Bits.Bits_16) (Value_Type.Integer Bits.Bits_32) . should_equal (Value_Type.Integer Bits.Bits_32) + Value_Type_Helpers.reconcile_types (Value_Type.Float Bits.Bits_32) (Value_Type.Float Bits.Bits_32) . should_equal (Value_Type.Float Bits.Bits_32) + Value_Type_Helpers.reconcile_types (Value_Type.Float Bits.Bits_32) (Value_Type.Float Bits.Bits_64) . should_equal (Value_Type.Float Bits.Bits_64) + + Value_Type_Helpers.reconcile_types Value_Type.Boolean Value_Type.Byte . should_equal Value_Type.Byte + Value_Type_Helpers.reconcile_types (Value_Type.Integer Bits.Bits_16) Value_Type.Byte . should_equal (Value_Type.Integer Bits.Bits_16) + # 64-bit floats are always used when unifying with integers + Value_Type_Helpers.reconcile_types (Value_Type.Float Bits.Bits_32) Value_Type.Byte . should_equal Value_Type.Float + Value_Type_Helpers.reconcile_types (Value_Type.Float Bits.Bits_32) Value_Type.Boolean . should_equal Value_Type.Float + + Value_Type_Helpers.reconcile_types (Value_Type.Char 10 False) (Value_Type.Char 10 False) . should_equal (Value_Type.Char 10 False) + Value_Type_Helpers.reconcile_types (Value_Type.Char 10 False) (Value_Type.Char 10 True) . should_equal (Value_Type.Char 10 True) + Value_Type_Helpers.reconcile_types (Value_Type.Char 100 False) (Value_Type.Char 10 True) . should_equal (Value_Type.Char 100 True) + Value_Type_Helpers.reconcile_types (Value_Type.Char 10 False) (Value_Type.Char 15 False) . should_equal (Value_Type.Char 15 True) + + Value_Type_Helpers.reconcile_types Value_Type.Date Value_Type.Date . should_equal Value_Type.Date + Value_Type_Helpers.reconcile_types Value_Type.Time Value_Type.Time . should_equal Value_Type.Time + Value_Type_Helpers.reconcile_types Value_Type.Date_Time Value_Type.Date_Time . should_equal Value_Type.Date_Time + ## Mixing date and time leads to mixed, if the user wants to convert date to at-midnight timestamp or + date-time to just date, they need to do it explicitly. + Value_Type_Helpers.reconcile_types Value_Type.Date Value_Type.Date_Time . should_equal Value_Type.Mixed + Value_Type_Helpers.reconcile_types Value_Type.Time Value_Type.Date_Time . should_equal Value_Type.Mixed + + Value_Type_Helpers.reconcile_types Value_Type.Float Value_Type.Integer . should_equal Value_Type.Float + Value_Type_Helpers.reconcile_types Value_Type.Char Value_Type.Integer . should_equal Value_Type.Mixed + Value_Type_Helpers.reconcile_types Value_Type.Float Value_Type.Char . should_equal Value_Type.Mixed + Value_Type_Helpers.reconcile_types Value_Type.Float Value_Type.Binary . should_equal Value_Type.Mixed + Value_Type_Helpers.reconcile_types Value_Type.Char Value_Type.Binary . should_equal Value_Type.Mixed + Value_Type_Helpers.reconcile_types Value_Type.Char Value_Type.Boolean . should_equal Value_Type.Mixed + main = Test_Suite.run_main spec diff --git a/test/Tests/src/Data/Text/Utils_Spec.enso b/test/Tests/src/Data/Text/Utils_Spec.enso index 5c98c5bf0bb8..671f06ff32ad 100644 --- a/test/Tests/src/Data/Text/Utils_Spec.enso +++ b/test/Tests/src/Data/Text/Utils_Spec.enso @@ -89,4 +89,27 @@ spec = Text_Utils.take_suffix (kshi+kshi+'a'+kshi) 2 . should_equal 'a'+kshi Text_Utils.take_suffix (kshi+kshi+'a'+kshi) 1 . should_equal kshi + Test.group "to_display_text" <| + Test.specify "simple conversion" <| + "Hello".to_display_text . should_equal "Hello" + + Test.specify "long text conversion" <| + long = "Hello World! ".repeat 1024 + disp = long.to_display_text + disp.length . should_equal 80 + disp.characters.take (First 5) . should_equal [ 'H', 'e', 'l', 'l', 'o' ] + disp.characters.take (Last 6) . should_equal ['l', 'd', '!', ' ', 'H', 'e'] + + Test.specify "grapheme 1 conversion" <| + txt = 'a\u0321\u0302'*100 + txt.to_display_text . should_equal 'a\u0321\u0302'*80 + + Test.specify "grapheme 2 conversion" <| + txt = '\u0915\u094D\u0937\u093F'*100 + txt.to_display_text . should_equal '\u0915\u094D\u0937\u093F'*80 + + Test.specify "grapheme 3 conversion" <| + txt = '\u{1F926}\u{1F3FC}\u200D\u2642\uFE0F'*100 + txt.to_display_text . should_equal '\u{1F926}\u{1F3FC}\u200D\u2642\uFE0F'*80 + main = Test_Suite.run_main spec diff --git a/test/Tests/src/Data/Time/Duration_Spec.enso b/test/Tests/src/Data/Time/Duration_Spec.enso index ca72cbe73d6f..bda77111a0f7 100644 --- a/test/Tests/src/Data/Time/Duration_Spec.enso +++ b/test/Tests/src/Data/Time/Duration_Spec.enso @@ -22,10 +22,6 @@ spec = (Duration.between (Date_Time.new 2001 1 1) (Date_Time.new 2001 1 7)).total_hours . should_equal (6 * 24) (Duration.between (Date_Time.new 2001 1 1 13) (Date_Time.new 2001 1 7 16)).total_hours . should_equal (3 + 6 * 24) - Test.specify "should check if empty" <| - interval = Duration.zero - interval.is_empty . should_be_true - Test.specify "should normalize periods" <| (Duration.new seconds=60).total_minutes . should_equal 1 (Duration.new milliseconds=1000).total_seconds . should_equal 1 diff --git a/test/Tests/src/Runtime/Lazy_Spec.enso b/test/Tests/src/Runtime/Lazy_Spec.enso index ef23b4bd00ed..e024ed534e37 100644 --- a/test/Tests/src/Runtime/Lazy_Spec.enso +++ b/test/Tests/src/Runtime/Lazy_Spec.enso @@ -1,12 +1,16 @@ from Standard.Base import all import Standard.Base.Runtime.Ref.Ref -import Standard.Base.Runtime.Lazy.Lazy import Standard.Base.Errors.Illegal_Argument.Illegal_Argument from Standard.Test import Test, Test_Suite import Standard.Test.Extensions +type Lazy + Value ~get + new ~computation = Lazy.Value computation + new_eager computation = Lazy.Value computation + spec = Test.group "Lazy" <| Test.specify "should compute the result only once" <| ref = Ref.new 0 diff --git a/test/Tests/src/Semantic/Runtime_Spec.enso b/test/Tests/src/Semantic/Runtime_Spec.enso index b50a5692a58c..0a947d7cf0da 100644 --- a/test/Tests/src/Semantic/Runtime_Spec.enso +++ b/test/Tests/src/Semantic/Runtime_Spec.enso @@ -1,16 +1,18 @@ import Standard.Base.Runtime import Standard.Base.Data.Numbers.Integer +import Standard.Base.Any.Any +import Standard.Base.Panic.Panic -from Standard.Base.Runtime.IO_Permissions import Input, Output +from Standard.Base.Runtime.Context import Input, Output -from Standard.Test import Test +from Standard.Test import Test, Test_Suite import Standard.Test.Extensions -in_fn : Integer -> Integer in Input -in_fn a = a * 2 +in_fn : Integer -> Integer +in_fn a = Input.if_enabled (a * 2) -out_fn : Integer -> Integer in Output -out_fn a = a + 1 +out_fn : Integer -> Integer +out_fn a = Output.if_enabled (a + 1) spec = Test.group "Inlining Helpers" <| @@ -20,9 +22,17 @@ spec = Test.specify "should allow to call a function" <| x = Runtime.no_inline_with_arg (x -> x + 4) 3 x . should_equal 7 - Test.group "IO Contexts" <| + Test.group "Contexts and Execution Environment" <| + Test.specify "should prevent execution in the default design environment" <| + res = Panic.catch Any (in_fn 1) p-> p.payload.to_text + res . should_equal "(Forbidden_Operation.Error 'Input')" Test.specify "should be configurable" <| - r = Runtime.allow_input_in "production" <| - Runtime.allow_output_in "production" <| - in_fn (out_fn 10) - r.should_equal 22 + r1 = Runtime.with_enabled_context Input <| + Runtime.with_enabled_context Output <| + in_fn (out_fn 10) + r1.should_equal 22 + + r2 = Panic.catch Any (Runtime.with_enabled_context Output <| in_fn (out_fn 10)) p-> p.payload.to_text + r2 . should_equal "(Forbidden_Operation.Error 'Input')" + +main = Test_Suite.run_main spec diff --git a/test/Visualization_Tests/src/Table_Spec.enso b/test/Visualization_Tests/src/Table_Spec.enso index 65d8e71a5287..8899dcb13432 100644 --- a/test/Visualization_Tests/src/Table_Spec.enso +++ b/test/Visualization_Tests/src/Table_Spec.enso @@ -89,7 +89,7 @@ visualization_spec connection = Test.specify "should visualize value type info" <| Value_Type.Boolean.to_json . should_equal '{"type":"Value_Type","constructor":"Boolean","_display_text_":"Boolean"}' Value_Type.Float.to_json . should_equal '{"type":"Value_Type","constructor":"Float","_display_text_":"Float (64 bits)","bits":64}' - Value_Type.Decimal.to_json . should_equal '{"type":"Value_Type","constructor":"Decimal","_display_text_":"Decimal (precision=Nothing, scale=0)","precision":null,"scale":0}' + Value_Type.Decimal.to_json . should_equal '{"type":"Value_Type","constructor":"Decimal","_display_text_":"Decimal (precision=Nothing, scale=Nothing)","precision":null,"scale":null}' Value_Type.Char.to_json . should_equal '{"type":"Value_Type","constructor":"Char","_display_text_":"Char (max_size=Nothing, variable_length=True)","size":null,"variable_length":true}' Value_Type.Unsupported_Data_Type.to_json . should_equal '{"type":"Value_Type","constructor":"Unsupported_Data_Type","_display_text_":"Unsupported_Data_Type","type_name":null}' diff --git a/test/Visualization_Tests/src/Widgets/Database_Widgets_Spec.enso b/test/Visualization_Tests/src/Widgets/Database_Widgets_Spec.enso index 762fb6ae7134..4c52332a93ad 100644 --- a/test/Visualization_Tests/src/Widgets/Database_Widgets_Spec.enso +++ b/test/Visualization_Tests/src/Widgets/Database_Widgets_Spec.enso @@ -25,7 +25,7 @@ spec = Test.group "Widgets for In-Database Connection with table name sets" <| Test.specify "works for `query` and `read`" <| - choices = ['a_table', 'another', 'mock_table'] . map n-> Choice.Option n n.pretty + choices = ['sqlite_schema', 'a_table', 'another', 'mock_table'] . map n-> Choice.Option n n.pretty expect = [["query", Widget.Single_Choice choices Nothing Display.Always]] . to_json Widgets.get_widget_json connection "query" ["query"] . should_equal expect Widgets.get_widget_json connection "read" ["query"] . should_equal expect diff --git a/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso index 69e9f40d50e0..2bc6cec1bb3b 100644 --- a/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso +++ b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Errors/Common.enso @@ -21,3 +21,7 @@ type Inexhaustive_Pattern_Match @Builtin_Type type Arity_Error Error expected_min expected_max actual + +@Builtin_Type +type Forbidden_Operation + Error operation diff --git a/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Function.enso b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Function.enso new file mode 100644 index 000000000000..ef6a107c1b4b --- /dev/null +++ b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Function.enso @@ -0,0 +1,2 @@ +@Builtin_Type +type Function diff --git a/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Runtime.enso b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Runtime.enso index 720c416ee03a..8540bb985077 100644 --- a/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Runtime.enso +++ b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Runtime.enso @@ -1,7 +1,43 @@ -type IO_Permissions +import project.Any.Any +import project.Data.Text.Text +import project.Data.Boolean.Boolean +import project.Errors.Common.Forbidden_Operation +import project.Function.Function +import project.Panic.Panic + +from project.Runtime.Context import Input,Output + +@Builtin_Type +type Context Input Output + name : Text + name self = + case self of + Input -> "Input" + Output -> "Output" + + if_enabled : Function -> Text -> Any + if_enabled self ~action environment="design" = + if self.is_enabled environment then action else Panic.throw (Forbidden_Operation.Error self.name) + + is_enabled : Text -> Boolean + is_enabled self environment="design" = @Builtin_Method "Context.is_enabled" + +current_execution_environment : Text +current_execution_environment = @Builtin_Method "Runtime.current_execution_environment" + +with_enabled_context : Context -> Function -> Text -> Any +with_enabled_context context ~action environment=Runtime.current_execution_environment = with_enabled_context_builtin context action environment + +with_enabled_context_builtin : Context -> Function -> Text -> Any +with_enabled_context_builtin context ~action environment = @Builtin_Method "Runtime.with_enabled_context_builtin" + +with_disabled_context : Context -> Function -> Text -> Any +with_disabled_context context ~action environment=Runtime.current_execution_environment = with_disabled_context_builtin context action environment + +with_disabled_context_builtin : Context -> Function -> Text -> Any +with_disabled_context_builtin context ~action environment = @Builtin_Method "Runtime.with_disabled_context_builtin" + primitive_get_stack_trace = @Builtin_Method "Runtime.primitive_get_stack_trace" -allow_input_in env = @Builtin_Method "Runtime.allow_input_in" -allow_output_in env = @Builtin_Method "Runtime.allow_output_in" diff --git a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-add b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-add deleted file mode 100644 index fc294a491ea7..000000000000 --- a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-add +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2007 David Crawshaw - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-ignore b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-ignore deleted file mode 100644 index b4b84603f502..000000000000 --- a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-ignore +++ /dev/null @@ -1,4 +0,0 @@ -copyright notice and this permission notice appear in all copies. -this work for additional information regarding copyright ownership. -this work for additional information regarding copyright ownership. -copyright notice and this permission notice appear in all copies. diff --git a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep deleted file mode 100644 index b04d73e63cee..000000000000 --- a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep +++ /dev/null @@ -1,6 +0,0 @@ -Copyright (c) 2007 David Crawshaw -Copyright 2007 Taro L. Saito -Copyright 2008 Taro L. Saito -Copyright 2009 Taro L. Saito -Copyright 2010 Taro L. Saito -Copyright 2016 Magnus Reftel diff --git a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep-context b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep-context deleted file mode 100644 index 25f623ea1af9..000000000000 --- a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep-context +++ /dev/null @@ -1 +0,0 @@ -Copyright (c) 2021 Gauthier Roebroeck diff --git a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/copyright-ignore similarity index 55% rename from tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep rename to tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/copyright-ignore index 915c5e9ad45c..df4a4d2bf496 100644 --- a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep +++ b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/copyright-ignore @@ -1,4 +1,3 @@ Copyright 2008 Taro L. Saito Copyright 2009 Taro L. Saito Copyright 2010 Taro L. Saito -Copyright (c) 2021 Gauthier Roebroeck diff --git a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-ignore b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/copyright-keep similarity index 59% rename from tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-ignore rename to tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/copyright-keep index d9997fce671f..94ec1d4ac39d 100644 --- a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-ignore +++ b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/copyright-keep @@ -1,3 +1,6 @@ Copyright (c) 2007 David Crawshaw +Copyright (c) 2021 Gauthier Roebroeck +Copyright 2007 Taro L. Saito +Copyright 2016 Magnus Reftel copyright notice and this permission notice appear in all copies. this work for additional information regarding copyright ownership. diff --git a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/custom-license b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/custom-license similarity index 100% rename from tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/custom-license rename to tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/custom-license diff --git a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/files-keep b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/files-keep similarity index 100% rename from tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/files-keep rename to tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/files-keep index 348b16cee494..e724e5582868 100644 --- a/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/files-keep +++ b/tools/legal-review/Database/org.xerial.sqlite-jdbc-3.41.2.1/files-keep @@ -1,2 +1,2 @@ -META-INF/maven/org.xerial/sqlite-jdbc/LICENSE META-INF/maven/org.xerial/sqlite-jdbc/LICENSE.zentus +META-INF/maven/org.xerial/sqlite-jdbc/LICENSE diff --git a/tools/legal-review/Database/report-state b/tools/legal-review/Database/report-state index 2d8a2833ff1b..e1e69c1dc970 100644 --- a/tools/legal-review/Database/report-state +++ b/tools/legal-review/Database/report-state @@ -1,3 +1,3 @@ -87AFCC58BE0E7EE9CF656C3750BE38C23B6DEC25E1918CB469E77287E9F654C0 -99D73D03217F8E7D41288717A740C13DB765186A8246C4C6813E24ED23142A4C +F04556111778820D07D28B45846E7DB8144686C5571EBEFDCF1D3A23E98DA640 +48C573837ADE7AA7B2DF62F71F6522827EEB47194545260798FB8651A7DEB11F 0 diff --git a/tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.3.3/custom-license b/tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.4.1/custom-license similarity index 100% rename from tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.3.3/custom-license rename to tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.4.1/custom-license diff --git a/tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.3.3/files-add/LICENSE.txt b/tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.4.1/files-add/LICENSE.txt similarity index 97% rename from tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.3.3/files-add/LICENSE.txt rename to tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.4.1/files-add/LICENSE.txt index 05b7a25dbccd..0fb1082e7314 100644 --- a/tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.3.3/files-add/LICENSE.txt +++ b/tools/legal-review/engine/com.typesafe.slick.slick_2.13-3.4.1/files-add/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright 2011-2016 Typesafe, Inc. +Copyright 2011-2021 Lightbend, Inc. All rights reserved. diff --git a/tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.4/copyright-keep-context b/tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.4/copyright-keep-context new file mode 100644 index 000000000000..2d1291cd5315 --- /dev/null +++ b/tools/legal-review/engine/org.reactivestreams.reactive-streams-1.0.4/copyright-keep-context @@ -0,0 +1 @@ +this code has waived all copyright and related or neighboring * diff --git a/tools/legal-review/engine/org.scala-lang.modules.scala-collection-compat_2.13-2.0.0/copyright-add b/tools/legal-review/engine/org.scala-lang.modules.scala-collection-compat_2.13-2.8.1/copyright-add similarity index 100% rename from tools/legal-review/engine/org.scala-lang.modules.scala-collection-compat_2.13-2.0.0/copyright-add rename to tools/legal-review/engine/org.scala-lang.modules.scala-collection-compat_2.13-2.8.1/copyright-add diff --git a/tools/legal-review/engine/org.scala-lang.modules.scala-collection-compat_2.13-2.0.0/copyright-keep-context b/tools/legal-review/engine/org.scala-lang.modules.scala-collection-compat_2.13-2.8.1/copyright-keep-context similarity index 100% rename from tools/legal-review/engine/org.scala-lang.modules.scala-collection-compat_2.13-2.0.0/copyright-keep-context rename to tools/legal-review/engine/org.scala-lang.modules.scala-collection-compat_2.13-2.8.1/copyright-keep-context diff --git a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-add b/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-add deleted file mode 100644 index 1f5cc2a40b2d..000000000000 --- a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-add +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright (c) 2007 David Crawshaw - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ diff --git a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep-context b/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep-context deleted file mode 100644 index 89b10836aadb..000000000000 --- a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep-context +++ /dev/null @@ -1,2 +0,0 @@ -Copyright 2007 Taro L. Saito -Copyright 2016 Magnus Reftel diff --git a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/copyright-ignore b/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/copyright-ignore new file mode 100644 index 000000000000..df4a4d2bf496 --- /dev/null +++ b/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/copyright-ignore @@ -0,0 +1,3 @@ +Copyright 2008 Taro L. Saito +Copyright 2009 Taro L. Saito +Copyright 2010 Taro L. Saito diff --git a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/copyright-keep b/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/copyright-keep new file mode 100644 index 000000000000..94ec1d4ac39d --- /dev/null +++ b/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/copyright-keep @@ -0,0 +1,6 @@ +Copyright (c) 2007 David Crawshaw +Copyright (c) 2021 Gauthier Roebroeck +Copyright 2007 Taro L. Saito +Copyright 2016 Magnus Reftel +copyright notice and this permission notice appear in all copies. +this work for additional information regarding copyright ownership. diff --git a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/custom-license b/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/custom-license similarity index 100% rename from tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/custom-license rename to tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/custom-license diff --git a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/files-keep b/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/files-keep similarity index 100% rename from tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/files-keep rename to tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/files-keep index 348b16cee494..e724e5582868 100644 --- a/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.36.0.3/files-keep +++ b/tools/legal-review/engine/org.xerial.sqlite-jdbc-3.41.2.1/files-keep @@ -1,2 +1,2 @@ -META-INF/maven/org.xerial/sqlite-jdbc/LICENSE META-INF/maven/org.xerial/sqlite-jdbc/LICENSE.zentus +META-INF/maven/org.xerial/sqlite-jdbc/LICENSE diff --git a/tools/legal-review/engine/report-state b/tools/legal-review/engine/report-state index db03db955f42..8906949a45e0 100644 --- a/tools/legal-review/engine/report-state +++ b/tools/legal-review/engine/report-state @@ -1,3 +1,3 @@ -7520855014B1EA606329CFE36E11F4D56759A574DCFCC9F0DF26118104236E5F -703351A5878198A8B8742CB0B36FAA59C9877C59242A447AB89BF11437BEB70D +4A60E6DF1A26DB1AFA55D834537366D7FD5260B08A6DFD1E507D2AFFEE659DE2 +C5B585894062096A4041C121B962E581E1B797922B192935C959B1C464DED266 0 diff --git a/tools/legal-review/engine/reviewed-licenses/MIT-0 b/tools/legal-review/engine/reviewed-licenses/MIT-0 new file mode 100644 index 000000000000..8264983091cf --- /dev/null +++ b/tools/legal-review/engine/reviewed-licenses/MIT-0 @@ -0,0 +1 @@ +tools/legal-review/license-texts/MIT-0 diff --git a/tools/legal-review/license-texts/MIT-0 b/tools/legal-review/license-texts/MIT-0 new file mode 100644 index 000000000000..a4e9dc906188 --- /dev/null +++ b/tools/legal-review/license-texts/MIT-0 @@ -0,0 +1,16 @@ +MIT No Attribution + +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tools/performance/engine-benchmarks/bench_download.py b/tools/performance/engine-benchmarks/bench_download.py index 2b1abd69ba27..1794b47ee0dc 100755 --- a/tools/performance/engine-benchmarks/bench_download.py +++ b/tools/performance/engine-benchmarks/bench_download.py @@ -569,6 +569,45 @@ def render_html(jinja_data: JinjaData, template_file: str, html_out_fname: str) html_file.write(generated_html) +def compare_runs(bench_run_id_1: str, bench_run_id_2: str, cache: Cache, tmp_dir: str) -> None: + def perc_str(perc: float) -> str: + s = "+" if perc > 0 else "" + s += "{:.5f}".format(perc) + s += "%" + return s + + def percent_change(old_val: float, new_val: float) -> float: + return ((new_val - old_val) / old_val) * 100 + + def commit_to_str(commit: Commit) -> str: + return f"{commit.timestamp} {commit.author.name} '{commit.message.splitlines()[0]}'" + + res_1 = _invoke_gh_api(f"/actions/runs/{bench_run_id_1}") + bench_run_1 = _parse_bench_run_from_json(res_1) + res_2 = _invoke_gh_api(f"/actions/runs/{bench_run_id_2}") + bench_run_2 = _parse_bench_run_from_json(res_2) + bench_report_1 = get_bench_report(bench_run_1, cache, tmp_dir) + bench_report_2 = get_bench_report(bench_run_2, cache, tmp_dir) + bench_labels: List[str] = list(bench_report_1.label_score_dict.keys()) + df = pd.DataFrame(columns=["bench_label", "score-run-1", "score-run-2"]) + for bench_label in bench_labels: + df = pd.concat([df, pd.DataFrame([{ + "bench_label": bench_label, + "score-run-1": bench_report_1.label_score_dict[bench_label], + "score-run-2": bench_report_2.label_score_dict[bench_label], + }])], ignore_index=True) + df["score-diff"] = np.diff(df[["score-run-1", "score-run-2"]], axis=1) + df["score-diff-perc"] = df.apply(lambda row: perc_str(percent_change(row["score-run-1"], row["score-run-2"])), + axis=1) + print("================================") + print(df.to_string(index=False, header=True, justify="center", float_format="%.5f")) + print("================================") + print("Latest commit on bench-run-id-1: ") + print(" " + commit_to_str(bench_run_1.head_commit)) + print("Latest commit on bench-run-id-2: ") + print(" " + commit_to_str(bench_run_2.head_commit)) + + if __name__ == '__main__': default_since = datetime.now() - timedelta(days=14) default_until = datetime.now() @@ -591,6 +630,13 @@ def render_html(jinja_data: JinjaData, template_file: str, html_out_fname: str) help=f"The date until which the benchmark results will be gathered. " f"Format is {date_format_help}. " f"The default is today") + arg_parser.add_argument("--compare", + nargs=2, + default=[], + metavar=("", ""), + help="Compare two benchmark actions runs. Choose an action from https://github.com/enso-org/enso/actions/workflows/benchmark.yml, " + "and copy its ID from the URL. For example ID 4602465427 from URL https://github.com/enso-org/enso/actions/runs/4602465427. " + "This option excludes --since, --until, --output, and --create-csv options.") arg_parser.add_argument("-o", "--output", default="Engine_Benchs/data/benchs.csv", metavar="CSV_OUTPUT", @@ -621,15 +667,8 @@ def render_html(jinja_data: JinjaData, template_file: str, html_out_fname: str) else: log_level = logging.INFO logging.basicConfig(level=log_level, stream=sys.stdout) - if not args.since: - logging.error("--since option not specified") - arg_parser.print_help() - exit(1) + since: datetime = args.since - if not args.until: - logging.error(f"--until option not specified") - arg_parser.print_help() - exit(1) until: datetime = args.until cache_dir: str = args.cache if not args.tmp_dir: @@ -640,48 +679,50 @@ def render_html(jinja_data: JinjaData, template_file: str, html_out_fname: str) assert cache_dir and temp_dir csv_fname: str = args.output create_csv: bool = args.create_csv + compare: List[str] = args.compare logging.info(f"parsed args: since={since}, until={until}, cache_dir={cache_dir}, " f"temp_dir={temp_dir}, use_cache={use_cache}, output={csv_fname}, " - f"create_csv={create_csv}") + f"create_csv={create_csv}, compare={compare}") if use_cache: cache = populate_cache(cache_dir) else: cache = FakeCache() - bench_runs = get_bench_runs(since, until) - if len(bench_runs) == 0: - print(f"No successful benchmarks found within period since {since} until {until}") - exit(1) - job_reports: List[JobReport] = [] - for bench_run in bench_runs: - job_report = get_bench_report(bench_run, cache, temp_dir) - if job_report: - job_reports.append(job_report) - logging.debug(f"Got {len(job_reports)} job reports") - if create_csv: - write_bench_reports_to_csv(job_reports, csv_fname) - logging.info(f"Benchmarks written to {csv_fname}") - print(f"The generated CSV is in {csv_fname}") - - # Create a separate datatable for each benchmark label - # with 'label' and 'commit_timestamp' as columns. - bench_labels: List[str] = list(job_reports[0].label_score_dict.keys()) - template_bench_datas: List[TemplateBenchData] = [] - for bench_label in bench_labels: - bench_data = create_data_for_benchmark(job_reports, bench_label) - template_bench_datas.append(create_template_data(bench_data)) - jinja_data = JinjaData( - since=since, - until=until, - bench_datas=template_bench_datas - ) - - # Render Jinja template with jinja_data - render_html(jinja_data, JINJA_TEMPLATE, "index.html") - index_html_path = os.path.join(os.getcwd(), "index.html") - - print(f"The generated HTML is in {index_html_path}") - print(f"Open file://{index_html_path} in the browser") + if len(compare) > 0: + compare_runs(compare[0], compare[1], cache, temp_dir) + else: + bench_runs = get_bench_runs(since, until) + if len(bench_runs) == 0: + print(f"No successful benchmarks found within period since {since} until {until}") + exit(1) + job_reports: List[JobReport] = [] + for bench_run in bench_runs: + job_report = get_bench_report(bench_run, cache, temp_dir) + if job_report: + job_reports.append(job_report) + logging.debug(f"Got {len(job_reports)} job reports") + if create_csv: + write_bench_reports_to_csv(job_reports, csv_fname) + logging.info(f"Benchmarks written to {csv_fname}") + print(f"The generated CSV is in {csv_fname}") + + # Create a separate datatable for each benchmark label + # with 'label' and 'commit_timestamp' as columns. + bench_labels: List[str] = list(job_reports[0].label_score_dict.keys()) + template_bench_datas: List[TemplateBenchData] = [] + for bench_label in bench_labels: + bench_data = create_data_for_benchmark(job_reports, bench_label) + template_bench_datas.append(create_template_data(bench_data)) + jinja_data = JinjaData( + since=since, + until=until, + bench_datas=template_bench_datas + ) + # Render Jinja template with jinja_data + render_html(jinja_data, JINJA_TEMPLATE, "index.html") + index_html_path = os.path.join(os.getcwd(), "index.html") + print(f"The generated HTML is in {index_html_path}") + print(f"Open file://{index_html_path} in the browser") From 2a3b42bd9c403fbd118be887a0bf02f4c19cb4fb Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 10:59:20 -0400 Subject: [PATCH 27/64] expect_text throws --- .../0.0.0-dev/src/Internal/Split_Tokenize.enso | 16 +++++++--------- .../src/In_Memory/Split_Tokenaize_Spec.enso | 10 ++-------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 752dcfcc758f..b019e5c14ae9 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -15,7 +15,7 @@ from Standard.Table.Errors import Column_Count_Exceeded, Invalid_Value_Type, Mis See `Table.split_to_columns`. split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = - expect_text_column table column on_problems <| + expect_text_column table column <| fan_out_to_columns table column (ignore_nothing (_.split delimiter)) "split" column_count on_problems ## PRIVATE @@ -24,7 +24,7 @@ split_to_columns table column delimiter="," column_count=Auto on_problems=Report split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows table column delimiter="," on_problems=Report_Error = _ = [on_problems] - expect_text_column table column on_problems <| + expect_text_column table column <| fan_out_to_rows table column (ignore_nothing (_.split delimiter)) ## PRIVATE @@ -33,7 +33,7 @@ split_to_rows table column delimiter="," on_problems=Report_Error = See `Table.tokenize_to_columns`. tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns table column pattern case_sensitivity column_count on_problems = - expect_text_column table column on_problems <| + expect_text_column table column <| fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) "tokenize" column_count on_problems ## PRIVATE @@ -43,7 +43,7 @@ tokenize_to_columns table column pattern case_sensitivity column_count on_proble tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = _ = [on_problems] - expect_text_column table column on_problems <| + expect_text_column table column <| fan_out_to_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) ## PRIVATE @@ -191,16 +191,14 @@ ignore_nothing function = x-> case x of ## PRIVATE Asserts that a column exists in the table and is a text column. expect_text_column : Table -> Text | Integer -> Problem_Behavior -> Any -> Any -expect_text_column table column_id on_problems ~action = +expect_text_column table column_id ~action = column = table.get column_id case column of Nothing -> - problem = Missing_Input_Columns.Error [column_id] - on_problems.attach_problem_after Nothing problem + Error.throw <| Missing_Input_Columns.Error [column_id] _ : Column -> case Value_Type.is_text column.value_type of False -> - problem = Invalid_Value_Type.Error Value_Type.Char column.value_type column_id - on_problems.attach_problem_after Nothing problem + Error.throw <| Invalid_Value_Type.Error Value_Type.Char column.value_type column_id True -> action diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index a75a7c03b0cd..2862adf82d50 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -97,18 +97,12 @@ spec = Test.specify "*_to_columns handles missing input column" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols - action = t.tokenize_to_columns 'invalid_name' "([a-z]).(\d+)" on_problems=_ - tester = _.should_equal Nothing - problems = [Missing_Input_Columns.Error ['invalid_name']] - Problems.test_problem_handling action problems tester + t.tokenize_to_columns 'invalid_name' "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns Test.specify "*_to_rows handles missing input column" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols - action = t.tokenize_to_rows 'invalid_name' "([a-z]).(\d+)" on_problems=_ - tester = _.should_equal Nothing - problems = [Missing_Input_Columns.Error ['invalid_name']] - Problems.test_problem_handling action problems tester + t.tokenize_to_rows 'invalid_name' "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns Test.group "name conflicts" <| Test.specify 'will make column names unique' <| From a838918c3ccef23e46afbc3d81ab689dd2e8d92f Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 11:48:55 -0400 Subject: [PATCH 28/64] tests for cols --- .../src/In_Memory/Split_Tokenaize_Spec.enso | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index 2862adf82d50..a8da9d1be4dd 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -69,31 +69,44 @@ spec = t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" tables_should_be_equal t2 expected - Test.group 'errors' <| - Test.specify "won't work on a non-text column" <| - cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + Test.group 'column count' <| + Test.specify 'should generate extra empty columns if column_count is set' <| + cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols - t.split_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type - t.split_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type - t.tokenize_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type - t.tokenize_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type + expected_rows = [[0, 'a', 'c', Nothing, Nothing], [1, 'c', 'd', 'ef', Nothing], [2, 'gh', 'ij', 'u', Nothing]] + expected = Table.from_rows ['foo', 'split_bar', 'split_bar_1', 'split_bar_2', 'split_bar_3'] expected_rows + t2 = t.split_to_columns 'bar' 'b' column_count=4 + tables_should_be_equal t2 expected - Test.specify "split should return problems when exceeding the column limit" <| + Test.specify "split should limit columns and return problems when exceeding the column limit" <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols - action = t.split_to_columns 'bar' 'b' column_count=1 on_problems=_ - tester = _-> True # _.should_be_a Table + expected_rows = [[0, 'a', 'c'], [1, 'c', 'd'], [2, 'gh', 'ij']] + expected = Table.from_rows ['foo', 'split_bar', 'split_bar_1'] expected_rows + action = t.split_to_columns 'bar' 'b' column_count=2 on_problems=_ + tester = _.should_equal expected problems = [Column_Count_Exceeded.Error 1 3] Problems.test_problem_handling action problems tester - Test.specify "tokenize should return problems when exceeding the column limit" <| + Test.specify "tokenize should limit columns and return problems when exceeding the column limit" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols - action = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" column_count=1 on_problems=_ - tester = _.should_be_a Table + expected_rows = [[0, 'a', 'c'], [1, 'c', 'd'], [2, 'gh', 'ij']] + expected = Table.from_rows ['foo', 'split_bar', 'split_bar_1'] expected_rows + action = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" column_count=2 on_problems=_ + tester = _.should_equal expected problems = [Column_Count_Exceeded.Error 1 3] Problems.test_problem_handling action problems tester + Test.group 'errors' <| + Test.specify "won't work on a non-text column" <| + cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + t = Table.new cols + t.split_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type + t.split_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type + Test.specify "*_to_columns handles missing input column" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols From 9e7356a0e6b2857f974a811c45e4bd5ea3c02084 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:11:12 -0400 Subject: [PATCH 29/64] transpose with less copying --- .../src/Internal/Split_Tokenize.enso | 217 ++++++++++++++---- .../src/In_Memory/Split_Tokenaize_Spec.enso | 38 +-- 2 files changed, 186 insertions(+), 69 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index b019e5c14ae9..1c5a22cb0fe0 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -3,12 +3,14 @@ from Standard.Base import all import project.Data.Column.Column import project.Data.Set_Mode.Set_Mode import project.Data.Table.Table +import project.Internal.Java_Exports import project.Internal.Problem_Builder.Problem_Builder import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy from project.Data.Type.Value_Type import Auto from Standard.Table import Value_Type -from Standard.Table.Errors import Column_Count_Exceeded, Invalid_Value_Type, Missing_Input_Columns +from Standard.Table.Errors import Column_Count_Exceeded, Duplicate_Output_Column_Names, Invalid_Value_Type, Missing_Input_Columns +from project.Internal.Java_Exports import make_string_builder ## PRIVATE Splits a column of text into a set of new columns. @@ -16,7 +18,7 @@ from Standard.Table.Errors import Column_Count_Exceeded, Invalid_Value_Type, Mis split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = expect_text_column table column <| - fan_out_to_columns table column (ignore_nothing (_.split delimiter)) "split" column_count on_problems + fan_out_to_columns table column (ignore_nothing (_.split delimiter)) column_count on_problems ## PRIVATE Splits a column of text into a set of new rows. @@ -34,7 +36,7 @@ split_to_rows table column delimiter="," on_problems=Report_Error = tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns table column pattern case_sensitivity column_count on_problems = expect_text_column table column <| - fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) "tokenize" column_count on_problems + fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems ## PRIVATE Tokenizes a column of text into a set of new rows using a regular @@ -56,18 +58,23 @@ tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sens - input_column: The column to transform. - function: A function that transforms a single element of `input_column` to multiple values. -fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Text -> Auto | Integer -> Problem_Behavior -> Table | Nothing -fan_out_to_columns table column function function_name column_count=Auto on_problems=Report_Error = +fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Auto | Integer -> Problem_Behavior -> Table | Nothing +fan_out_to_columns table column function column_count=Auto on_problems=Report_Error = input_column = table.get column - new_column_vectors = map_column_vector_to_multiple input_column.to_vector function - new_columns = build_and_name_columns table column function_name new_column_vectors - num_new_columns = new_columns.length - too_many_columns = column_count != Auto && num_new_columns > column_count + problem_builder = Problem_Builder.new + new_columns_unrenamed = map_columns_to_multiple input_column function column_count problem_builder + new_columns = rename_new_columns table new_columns_unrenamed problem_builder + #new_column_vectors = map_column_vector_to_multiple input_column.to_vector function + #new_columns = build_and_name_columns table column function_name new_column_vectors + #num_new_columns = new_columns.length + #too_many_columns = column_count != Auto && num_new_columns > column_count new_table = replace_column_with_columns table column new_columns + problem_builder.attach_problems_after on_problems new_table - if too_many_columns.not then new_table else - problem = Column_Count_Exceeded.Error column_count num_new_columns - on_problems.attach_problem_after new_table problem + ## + if too_many_columns.not then new_table else + problem = Column_Count_Exceeded.Error column_count num_new_columns + on_problems.attach_problem_after new_table problem ## PRIVATE Transform a column by applying the given function to the values in the @@ -113,6 +120,113 @@ fan_out_to_rows table column function = Column.from_vector column.name (builder.to_vector) Table.new new_columns +## PRIVATE + + Map a multi-valued function over a column and return the results as set of + output columns. + + Returns a Pair of a Vector of Columns and a Vector of problems. + + Arguments: + - input_column: The column to transform. + - function: A function that transforms a single element of `input_column` + to multiple values. + - column_count: The number of columns to split to. + If `Auto` then columns will be added to fit all data. + If the data exceeds the `column_count`, a `Column_Count_Exceeded` error + will follow the `on_problems` behavior. + - on_problems: Specifies the behavior when a problem occurs. +map_columns_to_multiple : Column -> (Any -> Vector Any) -> Auto | Integer -> Problem_Builder -> Vector Column +map_columns_to_multiple input_column function column_count problem_builder = + num_rows = input_column.length + input_storage = input_column.java_column.getStorage + + builders = case column_count of + Auto -> + builders = Vector.new_builder + + 0.up_to num_rows . map i-> + input_value = input_storage.getItemBoxed i + output_values = function input_value + + # Add more builders if necessary to accommodate `output_values`. + if output_values.length > builders.length then + num_builders_needed = output_values.length - builders.length + 0.up_to num_builders_needed . map _-> + builder = make_string_builder num_rows + + # Pad the new builder with nulls + num_nulls_needed = i + builder.appendNulls num_nulls_needed + + builders.append <| builder + + ## Add `output_values` to builders; if there are more builders + than `output_values`, pad with nulll. + 0.up_to builders.length . map i-> + builders.at i . appendNoGrow (output_values.get i Nothing) + + builders.to_vector + + _ : Integer -> + builders = Vector.new column_count (_-> make_string_builder num_rows) + + output_lengths = 0.up_to num_rows . map i-> + input_value = input_storage.getItemBoxed i + output_values = function input_value + + ## Add `output_values` to builders; if there are more builders + than `output_values`, pad with null. + 0.up_to builders.length . map i-> + builders.at i . appendNoGrow (output_values.get i Nothing) + + output_values.length + + max_output_length = output_lengths.fold 0 .max + + if max_output_length > column_count then + problem = Column_Count_Exceeded.Error column_count max_output_length + problem_builder.report_other_warning problem + + builders + + # Build Columns. + builders.map .seal . map_with_index i->storage-> + name = input_column.name + "_" + i.to_text + Column.from_storage name storage + + ## + IO.println "IS" + IO.println input_storage + 0.up_to num_rows . map i-> + input_value = input_storage.getItemBoxed i + IO.println "INP" + IO.println i + IO.println input_value + builder = make_string_builder num_rows + IO.println builder + IO.println builder.getCurrentSize + builder.appendNulls 2 + builder.appendNoGrow "asdf" + st = builder.seal + c = Column.from_storage "nm" st + IO.println c + 0.up_to num_rows . map i-> + IO.println (c.at i) + Nothing + +## PRIVATE + Name a set of column vectors to be unique when added to a table. +rename_new_columns : Table -> Vector Column -> Problem_Builder -> Vector Column +rename_new_columns table columns problem_builder = + unique = Unique_Name_Strategy.new + unique.mark_used <| table.columns.map .name + new_columns = columns.map column-> + new_name = unique.make_unique column.name + column.rename new_name + problem_builder.report_unique_name_strategy unique + new_columns + ## PRIVATE Transform a column vector into a set of column vectors. Takes a function that maps a single element of the input column vector to a vector of output @@ -123,10 +237,10 @@ fan_out_to_rows table column function = - input_column: The column to transform. - function: A function that transforms a single element of `input_column` to multiple values. -map_column_vector_to_multiple : Vector Any -> (Any -> Vector Any) -> Vector (Vector Any) -map_column_vector_to_multiple input_vector function = - result_row_vectors = input_vector.map value-> function value - transpose_with_pad result_row_vectors + map_column_vector_to_multiple : Vector Any -> (Any -> Vector Any) -> Vector (Vector Any) + map_column_vector_to_multiple input_vector function = + result_row_vectors = input_vector.map value-> function value + transpose_with_pad result_row_vectors ## PRIVATE Remove a column and add new columns. @@ -135,44 +249,45 @@ replace_column_with_columns table old_column new_columns = with_column_removed = table.remove_columns old_column error_on_missing_columns=True new_columns.fold with_column_removed (t-> c-> t.set c set_mode=Set_Mode.Add) -## PRIVATE - Swap rows and columns of a vector-of-vectors, padding each vector to be the - same length first. - Assumes both dimensions are non-zero. -transpose_with_pad : Vector (Vector Any) -> Vector (Vector Any) -transpose_with_pad vecs = transpose (pad_vectors vecs Nothing) +## + ## PRIVATE + Swap rows and columns of a vector-of-vectors, padding each vector to be the + same length first. + Assumes both dimensions are non-zero. + transpose_with_pad : Vector (Vector Any) -> Vector (Vector Any) + transpose_with_pad vecs = transpose (pad_vectors vecs Nothing) -## PRIVATE - Swap rows and columns of a vector-of-vectors. - Assumes both dimensions are non-zero. -transpose : Vector (Vector Any) -> Vector (Vector Any) -transpose vecs = - num_output_rows = vecs.length - num_output_cols = vecs.first.length - builders = (0.up_to num_output_cols).map _-> Vector.new_builder num_output_rows - vecs.map vec-> - vec.map_with_index i-> v-> - builders.at i . append v - builders.map .to_vector + ## PRIVATE + Swap rows and columns of a vector-of-vectors. + Assumes both dimensions are non-zero. + transpose : Vector (Vector Any) -> Vector (Vector Any) + transpose vecs = + num_output_rows = vecs.length + num_output_cols = vecs.first.length + builders = (0.up_to num_output_cols).map _-> Vector.new_builder num_output_rows + vecs.map vec-> + vec.map_with_index i-> v-> + builders.at i . append v + builders.map .to_vector -## PRIVATE - Pad vectors so they have the same length. -pad_vectors : Vector (Vector Any) -> Any -> Vector (Vector Any) -pad_vectors vecs pad_value = - length = maximum <| vecs.map .length - vecs.map v-> v.pad length pad_value + ## PRIVATE + Pad vectors so they have the same length. + pad_vectors : Vector (Vector Any) -> Any -> Vector (Vector Any) + pad_vectors vecs pad_value = + length = maximum <| vecs.map .length + vecs.map v-> v.pad length pad_value -## PRIVATE - Name a set of column vectors to be unique when added to a table. Base the - new names on the name of the original column from which they were derived. -build_and_name_columns : Table -> Text -> Text -> Vector (Vector Any) -> Vector Column -build_and_name_columns table original_column_name function_name vectors = - ## Start with copies of the original column name. - old_names = 0.up_to vectors.length . map _-> original_column_name - table_column_names = table.columns . map .name - unique = Unique_Name_Strategy.new - new_names = unique.combine_with_prefix table_column_names old_names (function_name + "_") - vectors.map_with_index i-> vector-> Column.from_vector (new_names.at i) vector + ## PRIVATE + Name a set of column vectors to be unique when added to a table. Base the + new names on the name of the original column from which they were derived. + build_and_name_columns : Table -> Text -> Text -> Vector (Vector Any) -> Vector Column + build_and_name_columns table original_column_name function_name vectors = + ## Start with copies of the original column name. + old_names = 0.up_to vectors.length . map _-> original_column_name + table_column_names = table.columns . map .name + unique = Unique_Name_Strategy.new + new_names = unique.combine_with_prefix table_column_names old_names (function_name + "_") + vectors.map_with_index i-> vector-> Column.from_vector (new_names.at i) vector ## PRIVATE Return the maximum value of the vector. diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index a8da9d1be4dd..c4267a965a94 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -5,7 +5,7 @@ import Standard.Base.Errors.Problem_Behavior.Problem_Behavior import Standard.Test.Extensions from Standard.Table import Table, Column -from Standard.Table.Errors import Invalid_Value_Type, Column_Count_Exceeded, Missing_Input_Columns +from Standard.Table.Errors import Invalid_Value_Type, Column_Count_Exceeded, Duplicate_Output_Column_Names, Missing_Input_Columns from Standard.Test import Test, Test_Suite, Problems spec = @@ -16,7 +16,7 @@ spec = tables_should_be_equal actual expected = equal = tables_equal actual expected if equal.not then - msg = 'Tables differ. Actual:\n' + actual.display + '\nExpected:\n' + expected.display + msg = 'Tables differ.\nActual:\n' + actual.display + '\nExpected:\n' + expected.display Test.fail msg Test.group 'split' <| @@ -24,7 +24,7 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] - expected = Table.from_rows ['foo', 'split_bar', 'split_bar_1', 'split_bar_2'] expected_rows + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.split_to_columns 'bar' 'b' tables_should_be_equal t2 expected @@ -41,7 +41,7 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] t = Table.new cols expected_rows = [[0, '12', '34', '5'], [1, '23', Nothing, Nothing], [2, '2', '4', '55']] - expected = Table.from_rows ['foo', 'tokenize_bar', 'tokenize_bar_1', 'tokenize_bar_2'] expected_rows + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.tokenize_to_columns 'bar' "\d+" tables_should_be_equal t2 expected @@ -57,7 +57,7 @@ spec = cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] - expected = Table.from_rows ['foo', 'tokenize_bar', 'tokenize_bar_1', 'tokenize_bar_2'] expected_rows + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" tables_should_be_equal t2 expected @@ -74,7 +74,7 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols expected_rows = [[0, 'a', 'c', Nothing, Nothing], [1, 'c', 'd', 'ef', Nothing], [2, 'gh', 'ij', 'u', Nothing]] - expected = Table.from_rows ['foo', 'split_bar', 'split_bar_1', 'split_bar_2', 'split_bar_3'] expected_rows + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2', 'bar_3'] expected_rows t2 = t.split_to_columns 'bar' 'b' column_count=4 tables_should_be_equal t2 expected @@ -82,20 +82,20 @@ spec = cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] t = Table.new cols expected_rows = [[0, 'a', 'c'], [1, 'c', 'd'], [2, 'gh', 'ij']] - expected = Table.from_rows ['foo', 'split_bar', 'split_bar_1'] expected_rows + expected = Table.from_rows ['foo', 'bar_0', 'bar_1'] expected_rows action = t.split_to_columns 'bar' 'b' column_count=2 on_problems=_ - tester = _.should_equal expected - problems = [Column_Count_Exceeded.Error 1 3] + tester = t-> tables_should_be_equal t expected + problems = [Column_Count_Exceeded.Error 2 3] Problems.test_problem_handling action problems tester Test.specify "tokenize should limit columns and return problems when exceeding the column limit" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols - expected_rows = [[0, 'a', 'c'], [1, 'c', 'd'], [2, 'gh', 'ij']] - expected = Table.from_rows ['foo', 'split_bar', 'split_bar_1'] expected_rows + expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1'] expected_rows action = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" column_count=2 on_problems=_ - tester = _.should_equal expected - problems = [Column_Count_Exceeded.Error 1 3] + tester = t-> tables_should_be_equal t expected + problems = [Column_Count_Exceeded.Error 2 3] Problems.test_problem_handling action problems tester Test.group 'errors' <| @@ -119,11 +119,13 @@ spec = Test.group "name conflicts" <| Test.specify 'will make column names unique' <| - cols = [['split_bar', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']], ['bar_1', ['a', 'b', 'c']]] t = Table.new cols - expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] - expected = Table.from_rows ['split_bar', 'split_bar_1', 'split_bar_2', 'split_bar_3'] expected_rows - t2 = t.split_to_columns 'bar' 'b' - tables_should_be_equal t2 expected + expected_rows = [[0, 'a', 'a', 'c', Nothing], [1, 'b', 'c', 'd', 'ef'], [2, 'c', 'gh', 'ij', 'u']] + expected = Table.from_rows ['foo', 'bar_1', 'bar_0', 'bar_1_1', 'bar_2'] expected_rows + action = t.split_to_columns 'bar' 'b' on_problems=_ + tester = t-> tables_should_be_equal t expected + problems = [Duplicate_Output_Column_Names.Error ['bar_1']] + Problems.test_problem_handling action problems tester main = Test_Suite.run_main spec From e48e901bb6894e753a73ed3ffd3c7984a162c40c Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:25:47 -0400 Subject: [PATCH 30/64] column order --- .../0.0.0-dev/src/Internal/Split_Tokenize.enso | 11 +++-------- .../src/In_Memory/Split_Tokenaize_Spec.enso | 13 +++++++++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 1c5a22cb0fe0..670bfb3ac080 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -64,11 +64,7 @@ fan_out_to_columns table column function column_count=Auto on_problems=Report_Er problem_builder = Problem_Builder.new new_columns_unrenamed = map_columns_to_multiple input_column function column_count problem_builder new_columns = rename_new_columns table new_columns_unrenamed problem_builder - #new_column_vectors = map_column_vector_to_multiple input_column.to_vector function - #new_columns = build_and_name_columns table column function_name new_column_vectors - #num_new_columns = new_columns.length - #too_many_columns = column_count != Auto && num_new_columns > column_count - new_table = replace_column_with_columns table column new_columns + new_table = replace_column_with_columns table input_column new_columns problem_builder.attach_problems_after on_problems new_table ## @@ -243,11 +239,10 @@ rename_new_columns table columns problem_builder = transpose_with_pad result_row_vectors ## PRIVATE - Remove a column and add new columns. + Replace a single column in a table with new columns. replace_column_with_columns : Table -> Column -> Vector Column -> Table replace_column_with_columns table old_column new_columns = - with_column_removed = table.remove_columns old_column error_on_missing_columns=True - new_columns.fold with_column_removed (t-> c-> t.set c set_mode=Set_Mode.Add) + Table.new ((table.columns.map (c-> if c.name == old_column.name then new_columns else [c])).flatten) ## ## PRIVATE diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index c4267a965a94..5ba11451ab0d 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -121,11 +121,20 @@ spec = Test.specify 'will make column names unique' <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']], ['bar_1', ['a', 'b', 'c']]] t = Table.new cols - expected_rows = [[0, 'a', 'a', 'c', Nothing], [1, 'b', 'c', 'd', 'ef'], [2, 'c', 'gh', 'ij', 'u']] - expected = Table.from_rows ['foo', 'bar_1', 'bar_0', 'bar_1_1', 'bar_2'] expected_rows + expected_rows = [[0, 'a', 'c', Nothing, 'a'], [1, 'c', 'd', 'ef', 'b'], [2, 'gh', 'ij', 'u', 'c']] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1_1', 'bar_2', 'bar_1'] expected_rows action = t.split_to_columns 'bar' 'b' on_problems=_ tester = t-> tables_should_be_equal t expected problems = [Duplicate_Output_Column_Names.Error ['bar_1']] Problems.test_problem_handling action problems tester + Test.group "column order" <| + Test.specify 'preserves column order' <| + cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']], ['baz', [1, 2, 3]]] + t = Table.new cols + expected_rows = [[0, 'a', 'c', Nothing, 1], [1, 'c', 'd', 'ef', 2], [2, 'gh', 'ij', 'u', 3]] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2', 'baz'] expected_rows + t2 = t.split_to_columns 'bar' 'b' + tables_should_be_equal t2 expected + main = Test_Suite.run_main spec From 6fcb6ca6c229f5d9592609113db7003704fc28ab Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:29:10 -0400 Subject: [PATCH 31/64] reverse jagged --- test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index 5ba11451ab0d..a2b07616d6f3 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -98,6 +98,14 @@ spec = problems = [Column_Count_Exceeded.Error 2 3] Problems.test_problem_handling action problems tester + Test.specify 'should generate extra empty columns if column_count is set (reverse row order)' <| + cols = [['foo', [0, 1, 2]], ['bar', ['ghbijbu', 'cbdbef', 'abc']]] + t = Table.new cols + expected_rows = [[0, 'gh', 'ij', 'u', Nothing], [1, 'c', 'd', 'ef', Nothing], [2, 'a', 'c', Nothing, Nothing]] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2', 'bar_3'] expected_rows + t2 = t.split_to_columns 'bar' 'b' column_count=4 + tables_should_be_equal t2 expected + Test.group 'errors' <| Test.specify "won't work on a non-text column" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] From 2d747ed33f6449626fe83eff68e17e5a2b162428 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:31:17 -0400 Subject: [PATCH 32/64] remove old code --- .../src/Internal/Split_Tokenize.enso | 82 +------------------ 1 file changed, 1 insertion(+), 81 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 670bfb3ac080..b9da659d0834 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -67,11 +67,6 @@ fan_out_to_columns table column function column_count=Auto on_problems=Report_Er new_table = replace_column_with_columns table input_column new_columns problem_builder.attach_problems_after on_problems new_table - ## - if too_many_columns.not then new_table else - problem = Column_Count_Exceeded.Error column_count num_new_columns - on_problems.attach_problem_after new_table problem - ## PRIVATE Transform a column by applying the given function to the values in the column. The function produces multiple outputs, so each row is duplicated, @@ -158,7 +153,7 @@ map_columns_to_multiple input_column function column_count problem_builder = builders.append <| builder ## Add `output_values` to builders; if there are more builders - than `output_values`, pad with nulll. + than `output_values`, pad with null. 0.up_to builders.length . map i-> builders.at i . appendNoGrow (output_values.get i Nothing) @@ -191,26 +186,6 @@ map_columns_to_multiple input_column function column_count problem_builder = name = input_column.name + "_" + i.to_text Column.from_storage name storage - ## - IO.println "IS" - IO.println input_storage - 0.up_to num_rows . map i-> - input_value = input_storage.getItemBoxed i - IO.println "INP" - IO.println i - IO.println input_value - builder = make_string_builder num_rows - IO.println builder - IO.println builder.getCurrentSize - builder.appendNulls 2 - builder.appendNoGrow "asdf" - st = builder.seal - c = Column.from_storage "nm" st - IO.println c - 0.up_to num_rows . map i-> - IO.println (c.at i) - Nothing - ## PRIVATE Name a set of column vectors to be unique when added to a table. rename_new_columns : Table -> Vector Column -> Problem_Builder -> Vector Column @@ -223,67 +198,12 @@ rename_new_columns table columns problem_builder = problem_builder.report_unique_name_strategy unique new_columns -## PRIVATE - Transform a column vector into a set of column vectors. Takes a function - that maps a single element of the input column vector to a vector of output - values. The vectors of output values are padded with Nothing to be the same - length. - - Arguments: - - input_column: The column to transform. - - function: A function that transforms a single element of `input_column` - to multiple values. - map_column_vector_to_multiple : Vector Any -> (Any -> Vector Any) -> Vector (Vector Any) - map_column_vector_to_multiple input_vector function = - result_row_vectors = input_vector.map value-> function value - transpose_with_pad result_row_vectors - ## PRIVATE Replace a single column in a table with new columns. replace_column_with_columns : Table -> Column -> Vector Column -> Table replace_column_with_columns table old_column new_columns = Table.new ((table.columns.map (c-> if c.name == old_column.name then new_columns else [c])).flatten) -## - ## PRIVATE - Swap rows and columns of a vector-of-vectors, padding each vector to be the - same length first. - Assumes both dimensions are non-zero. - transpose_with_pad : Vector (Vector Any) -> Vector (Vector Any) - transpose_with_pad vecs = transpose (pad_vectors vecs Nothing) - - ## PRIVATE - Swap rows and columns of a vector-of-vectors. - Assumes both dimensions are non-zero. - transpose : Vector (Vector Any) -> Vector (Vector Any) - transpose vecs = - num_output_rows = vecs.length - num_output_cols = vecs.first.length - builders = (0.up_to num_output_cols).map _-> Vector.new_builder num_output_rows - vecs.map vec-> - vec.map_with_index i-> v-> - builders.at i . append v - builders.map .to_vector - - ## PRIVATE - Pad vectors so they have the same length. - pad_vectors : Vector (Vector Any) -> Any -> Vector (Vector Any) - pad_vectors vecs pad_value = - length = maximum <| vecs.map .length - vecs.map v-> v.pad length pad_value - - ## PRIVATE - Name a set of column vectors to be unique when added to a table. Base the - new names on the name of the original column from which they were derived. - build_and_name_columns : Table -> Text -> Text -> Vector (Vector Any) -> Vector Column - build_and_name_columns table original_column_name function_name vectors = - ## Start with copies of the original column name. - old_names = 0.up_to vectors.length . map _-> original_column_name - table_column_names = table.columns . map .name - unique = Unique_Name_Strategy.new - new_names = unique.combine_with_prefix table_column_names old_names (function_name + "_") - vectors.map_with_index i-> vector-> Column.from_vector (new_names.at i) vector - ## PRIVATE Return the maximum value of the vector. Throws Empty_Error if the vector is empty. From e5426a5071728dd69edc6ff607aa32fc2ab515dc Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:36:13 -0400 Subject: [PATCH 33/64] Column_Count_Exceeded members private --- distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso index d447ccc9fdc8..a7d7ce13135c 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso @@ -554,10 +554,14 @@ type Invalid_Value_For_Type "The value ["+self.value.to_text+"] is not valid for the column type ["+self.value_type.to_text+"]." type Column_Count_Exceeded - ## Indicates that an operation generating new columns produced more columns + ## PRIVATE + Indicates that an operation generating new columns produced more columns than allowed by the limit. Error (limit : Integer) (column_count : Integer) + ## PRIVATE + + Create a human-readable version of the error. to_display_text : Text to_display_text self = "The operation produced more columns than the specified limit. The limit is "+self.limit.to_text+" and the number of new columns was "+self.column_count.to_text+". The limit may be turned off by setting the `limit` option to `Auto`." From 07ae867beaddc31cb62c702af27b8e8081a2ae62 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:38:31 -0400 Subject: [PATCH 34/64] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbbe085dbe63..778f0aa352f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -378,6 +378,7 @@ `year`/`month`/`day` operations to Table columns.][6153] - [`Text.split` can now take a vector of delimiters.][6156] - [Implemented `Table.union` for the Database backend.][6204] +- [Implemented `Table.split` and `Table.tokenize` for in-memory tables.][6233] [debug-shortcuts]: https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug @@ -573,6 +574,7 @@ [6156]: https://github.com/enso-org/enso/pull/6156 [6204]: https://github.com/enso-org/enso/pull/6204 [6077]: https://github.com/enso-org/enso/pull/6077 +[6233]: https://github.com/enso-org/enso/pull/6233 #### Enso Compiler From 17fe3c1a63ea03266fa7df97588027630dd15bbb Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:40:28 -0400 Subject: [PATCH 35/64] mixed column --- test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index a2b07616d6f3..71fdeba75ca9 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -115,6 +115,14 @@ spec = t.tokenize_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type t.tokenize_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type + Test.specify "won't work on a mixed column" <| + cols = [['foo', [0, 1]], ['bar', [500, 'ab-10:bc-20c']]] + t = Table.new cols + t.split_to_columns 'bar' "x" . should_fail_with Invalid_Value_Type + t.split_to_rows 'bar' "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_columns 'bar' "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_rows 'bar' "x" . should_fail_with Invalid_Value_Type + Test.specify "*_to_columns handles missing input column" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] t = Table.new cols From 21c484ca3b7c9b93d21cc8a66db90d77370de070 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:48:34 -0400 Subject: [PATCH 36/64] comment --- .../lib/Standard/Database/0.0.0-dev/src/Data/Table.enso | 4 ++-- distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index aa8b480738f3..ce1426256357 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1425,7 +1425,7 @@ type Table ## Tokenizes a column of text into a set of new columns using a regular expression. If the pattern contains marked groups, the values are concatenated - together otherwise the whole match is returned. + together; otherwise the whole match is returned. The original column will be removed from the table. The new columns will be named with the name of the input with a incrementing number after. @@ -1448,7 +1448,7 @@ type Table ## Tokenizes a column of text into a set of new rows using a regular expression. If the pattern contains marked groups, the values are concatenated - together otherwise the whole match is returned. + together; otherwise the whole match is returned. The values of other columns are repeated for the new rows. Arguments: diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index f489a70e8202..018b1dd61136 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -950,7 +950,7 @@ type Table ## Tokenizes a column of text into a set of new columns using a regular expression. If the pattern contains marked groups, the values are concatenated - together otherwise the whole match is returned. + together; otherwise the whole match is returned. The original column will be removed from the table. The new columns will be named with the name of the input with a incrementing number after. @@ -972,7 +972,7 @@ type Table ## Tokenizes a column of text into a set of new rows using a regular expression. If the pattern contains marked groups, the values are concatenated - together otherwise the whole match is returned. + together; otherwise the whole match is returned. The values of other columns are repeated for the new rows. Arguments: From ee80885778a1b8460a67a5a449e819052096529e Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:52:01 -0400 Subject: [PATCH 37/64] unused import --- .../Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 1 - 1 file changed, 1 deletion(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index b9da659d0834..eea37c00713d 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -1,7 +1,6 @@ from Standard.Base import all import project.Data.Column.Column -import project.Data.Set_Mode.Set_Mode import project.Data.Table.Table import project.Internal.Java_Exports import project.Internal.Problem_Builder.Problem_Builder From d23f16331a11c08ddd8f05eeda33f3ba39344944 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Apr 2023 16:53:48 -0400 Subject: [PATCH 38/64] restored accidentally-deleted import --- test/Table_Tests/src/In_Memory/Table_Spec.enso | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Table_Tests/src/In_Memory/Table_Spec.enso b/test/Table_Tests/src/In_Memory/Table_Spec.enso index 47e51e0e328e..3361aeaaf6dd 100644 --- a/test/Table_Tests/src/In_Memory/Table_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Table_Spec.enso @@ -8,6 +8,7 @@ from Standard.Table import Table, Column, Sort_Column, Column_Selector, Aggregat import Standard.Table.Main as Table_Module from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all hiding First, Last import Standard.Table.Data.Type.Value_Type.Value_Type +from Standard.Table.Errors import Invalid_Output_Column_Names, Duplicate_Output_Column_Names, No_Input_Columns_Selected, Missing_Input_Columns, No_Such_Column, Floating_Point_Equality, Invalid_Value_Type import Standard.Visualization From 95394489898d2321cf879c3e4c0d7945f0fa5763 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Apr 2023 10:06:44 -0400 Subject: [PATCH 39/64] unused import --- test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso index 71fdeba75ca9..448e47694884 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso @@ -1,10 +1,8 @@ from Standard.Base import all -import Standard.Base.Errors.Illegal_Argument.Illegal_Argument -import Standard.Base.Errors.Problem_Behavior.Problem_Behavior import Standard.Test.Extensions -from Standard.Table import Table, Column +from Standard.Table import Table from Standard.Table.Errors import Invalid_Value_Type, Column_Count_Exceeded, Duplicate_Output_Column_Names, Missing_Input_Columns from Standard.Test import Test, Test_Suite, Problems From 4f754b9ca8f48f8ea8a56197a0e5c98e3eefbe7b Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Apr 2023 11:27:51 -0400 Subject: [PATCH 40/64] fan_out_to_rows use storage --- .../src/Internal/Split_Tokenize.enso | 61 +++++++++++-------- ...ize_Spec.enso => Split_Tokenize_Spec.enso} | 0 2 files changed, 35 insertions(+), 26 deletions(-) rename test/Table_Tests/src/In_Memory/{Split_Tokenaize_Spec.enso => Split_Tokenize_Spec.enso} (100%) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index eea37c00713d..889a9b9c0e6b 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -11,6 +11,8 @@ from Standard.Table import Value_Type from Standard.Table.Errors import Column_Count_Exceeded, Duplicate_Output_Column_Names, Invalid_Value_Type, Missing_Input_Columns from project.Internal.Java_Exports import make_string_builder +polyglot java import org.enso.table.data.mask.OrderMask + ## PRIVATE Splits a column of text into a set of new columns. See `Table.split_to_columns`. @@ -78,36 +80,43 @@ fan_out_to_columns table column function column_count=Auto on_problems=Report_Er - function: A function that transforms a single element of `input_column` to multiple values. fan_out_to_rows : Table -> Text | Integer -> (Any -> Vector Any) -> Table -fan_out_to_rows table column function = - input_column = table.get column +fan_out_to_rows table input_column_name function = + input_column = table.at input_column_name + input_storage = input_column.java_column.getStorage + num_input_rows = input_storage.size + + # Accumulates the output of the output column values. + output_column_builder = make_string_builder 0 + # Accumulates repeated position indices for the order mask. + order_mask_positions = Vector.new_builder 0 - # Transform each input value to an output set. - output_value_sets = input_column.to_vector.map function - output_value_set_sizes = output_value_sets.map .length - num_input_rows = table.row_count - num_output_rows = output_value_set_sizes.fold 0 (+) - - ## Generate new columns for the output table. For the input_column, the - new column consists of the concatenation of the output value sets. For - each of the other columns, it consists of the elements of the input - column repeated a number times, with that number equal to the size of - the corresponding output value set. + 0.up_to num_input_rows . map i-> + input_value = input_storage.getItemBoxed i + output_values = function input_value + # Append each value. + output_values.map v-> output_column_builder.append v + # Append n copies of the input row position, n = # of output values. + 0.up_to output_values.length . map _-> order_mask_positions.append i + + # Build the output column + output_storage = output_column_builder.seal + output_column = Column.from_storage input_column_name output_storage + + # Build the order mask. + order_mask = OrderMask.new (order_mask_positions.to_vector) + + # Build the other columns, and include the output_column while doing it. new_columns = table.columns.map column-> - builder = Vector.new_builder num_output_rows - case column.name == input_column.name of + case column.name == input_column_name of True -> - # The transformed column: concatenate output value sets. - output_value_sets.map values-> - builder.append_vector_range values + # Replace the input column with the output column. + output_column False -> - # The other columns: repeat the input values. - column_vector = column.to_vector - 0.up_to num_input_rows . map i-> - repetition_count = output_value_set_sizes.at i - input_value = column_vector.at i - 0.up_to repetition_count . map _-> - builder.append input_value - Column.from_vector column.name (builder.to_vector) + # Build a new column from the old one with the mask + old_storage = column.java_column.getStorage + new_storage = old_storage.applyMask order_mask + Column.from_storage column.name new_storage + Table.new new_columns ## PRIVATE diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso similarity index 100% rename from test/Table_Tests/src/In_Memory/Split_Tokenaize_Spec.enso rename to test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso From f32ebb57d99aa4b73dc1094ecafdf4201246c0f2 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Apr 2023 11:33:34 -0400 Subject: [PATCH 41/64] repeat --- .../Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 889a9b9c0e6b..a8ac66b269e3 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -96,7 +96,7 @@ fan_out_to_rows table input_column_name function = # Append each value. output_values.map v-> output_column_builder.append v # Append n copies of the input row position, n = # of output values. - 0.up_to output_values.length . map _-> order_mask_positions.append i + repeat output_values.length <| order_mask_positions.append i # Build the output column output_storage = output_column_builder.seal @@ -151,7 +151,7 @@ map_columns_to_multiple input_column function column_count problem_builder = # Add more builders if necessary to accommodate `output_values`. if output_values.length > builders.length then num_builders_needed = output_values.length - builders.length - 0.up_to num_builders_needed . map _-> + repeat num_builders_needed <| builder = make_string_builder num_rows # Pad the new builder with nulls @@ -240,3 +240,7 @@ expect_text_column table column_id ~action = Error.throw <| Invalid_Value_Type.Error Value_Type.Char column.value_type column_id True -> action + +## PRIVATE + Repeat a computation n times. +repeat n ~action = 0.up_to n . map _-> action From 414c713f026c9f43b9086b7f9d2fec966c9b3ec9 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Apr 2023 11:37:17 -0400 Subject: [PATCH 42/64] naming --- .../src/Internal/Split_Tokenize.enso | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index a8ac66b269e3..1a8944bcb20b 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -17,37 +17,37 @@ polyglot java import org.enso.table.data.mask.OrderMask Splits a column of text into a set of new columns. See `Table.split_to_columns`. split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table -split_to_columns table column delimiter="," column_count=Auto on_problems=Report_Error = - expect_text_column table column <| - fan_out_to_columns table column (ignore_nothing (_.split delimiter)) column_count on_problems +split_to_columns table input_column_name delimiter="," column_count=Auto on_problems=Report_Error = + expect_text_column table input_column_name <| + fan_out_to_columns table input_column_name (ignore_nothing (_.split delimiter)) column_count on_problems ## PRIVATE Splits a column of text into a set of new rows. See `Table.split_to_rows`. split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table -split_to_rows table column delimiter="," on_problems=Report_Error = +split_to_rows table input_column_name delimiter="," on_problems=Report_Error = _ = [on_problems] - expect_text_column table column <| - fan_out_to_rows table column (ignore_nothing (_.split delimiter)) + expect_text_column table input_column_name <| + fan_out_to_rows table input_column_name (ignore_nothing (_.split delimiter)) ## PRIVATE Tokenizes a column of text into a set of new columns using a regular expression. See `Table.tokenize_to_columns`. tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table -tokenize_to_columns table column pattern case_sensitivity column_count on_problems = - expect_text_column table column <| - fan_out_to_columns table column (ignore_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems +tokenize_to_columns table input_column_name pattern case_sensitivity column_count on_problems = + expect_text_column table input_column_name <| + fan_out_to_columns table input_column_name (ignore_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems ## PRIVATE Tokenizes a column of text into a set of new rows using a regular expression. See `Table.tokenize_to_rows`. tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table -tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = +tokenize_to_rows table input_column_name pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = _ = [on_problems] - expect_text_column table column <| - fan_out_to_rows table column (ignore_nothing (_.tokenize pattern case_sensitivity)) + expect_text_column table input_column_name <| + fan_out_to_rows table input_column_name (ignore_nothing (_.tokenize pattern case_sensitivity)) ## PRIVATE Transform a table by transforming a column into a set of columns. Takes a @@ -60,8 +60,8 @@ tokenize_to_rows table column pattern="." case_sensitivity=Case_Sensitivity.Sens - function: A function that transforms a single element of `input_column` to multiple values. fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Auto | Integer -> Problem_Behavior -> Table | Nothing -fan_out_to_columns table column function column_count=Auto on_problems=Report_Error = - input_column = table.get column +fan_out_to_columns table input_column_name function column_count=Auto on_problems=Report_Error = + input_column = table.get input_column_name problem_builder = Problem_Builder.new new_columns_unrenamed = map_columns_to_multiple input_column function column_count problem_builder new_columns = rename_new_columns table new_columns_unrenamed problem_builder From 7acdd0037a5084938571ce2fea417a9d4cb94d62 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Apr 2023 11:49:58 -0400 Subject: [PATCH 43/64] tok case-sens --- .../src/In_Memory/Split_Tokenize_Spec.enso | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index 448e47694884..9dea97a6ed04 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -1,5 +1,6 @@ from Standard.Base import all +import Standard.Base.Data.Text.Case_Sensitivity.Case_Sensitivity import Standard.Test.Extensions from Standard.Table import Table @@ -67,6 +68,22 @@ spec = t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" tables_should_be_equal t2 expected + Test.specify 'can do tokenize_to_columns case-insensitively' <| + cols = [['foo', [0, 1, 2]], ['bar', ['aBqcE', 'qcBr', 'cCb']]] + t = Table.new cols + expected_rows = [[0, 'B', 'c', Nothing], [1, 'c', 'B', Nothing], [2, 'c', 'C', 'b']] + expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows + t2 = t.tokenize_to_columns 'bar' "[bc]" case_sensitivity=Case_Sensitivity.Insensitive + tables_should_be_equal t2 expected + + Test.specify 'can do tokenize_to_rows case-insensitively' <| + cols = [['foo', [0, 1, 2]], ['bar', ['aBqcE', 'qcBr', 'cCb']]] + t = Table.new cols + expected_rows = [[0, 'B'], [0, 'c'], [1, 'c'], [1, 'B'], [2, 'c'], [2, 'C'], [2, 'b']] + expected = Table.from_rows ['foo', 'bar'] expected_rows + t2 = t.tokenize_to_rows 'bar' "[bc]" case_sensitivity=Case_Sensitivity.Insensitive + tables_should_be_equal t2 expected + Test.group 'column count' <| Test.specify 'should generate extra empty columns if column_count is set' <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] From f06db7846bd024ff9a45c1ff8f10e471b39b2e89 Mon Sep 17 00:00:00 2001 From: GregoryTravis Date: Thu, 13 Apr 2023 11:26:45 -0400 Subject: [PATCH 44/64] Update distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Radosław Waśko --- .../lib/Standard/Database/0.0.0-dev/src/Data/Table.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index ce1426256357..88b33fbf0c36 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1394,7 +1394,7 @@ type Table ## Splits a column of text into a set of new columns. The original column will be removed from the table. - The new columns will be named with the name of the input with a + The new columns will be named with the name of the input column with a incrementing number after. Arguments: From 1e15e9fd723c0b4636bd4c75967f52d923689e50 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 11:33:59 -0400 Subject: [PATCH 45/64] review --- .../Database/0.0.0-dev/src/Data/Table.enso | 24 ++++++++++--------- .../Table/0.0.0-dev/src/Data/Table.enso | 12 +++++----- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index 88b33fbf0c36..e72c3ad80974 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1398,40 +1398,42 @@ type Table incrementing number after. Arguments: - - column: The column to split the text of. + - column: The name or index of the column to split the text of. - delimiter: The term or terms used to split the text. - column_count: The number of columns to split to. If `Auto` then columns will be added to fit all data. - If the data exceeds the `column_count`, a `Column_Count_Exceeded` error - will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. + + ! Error Conditions + If the data exceeds the `column_count`, a `Column_Count_Exceeded` will + be reported according to the `on_problems` behavior. split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = _ = [column delimiter column_count on_problems] - Unimplemented.throw "This is an interface only." + Error.throw (Unsupported_Database_Operation.Error "Table.split_to_columns is not implemented yet for the Database backends.") ## Splits a column of text into a set of new rows. The values of other columns are repeated for the new rows. Arguments: - - column: The column to split the text of. + - column: The name or index of the column to split the text of. - delimiter: The term or terms used to split the text. - on_problems: Specifies the behavior when a problem occurs. split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table split_to_rows self column delimiter="," on_problems=Report_Error = _ = [column delimiter on_problems] - Unimplemented.throw "This is an interface only." + Error.throw (Unsupported_Database_Operation.Error "Table.split_to_rows is not implemented yet for the Database backends.") ## Tokenizes a column of text into a set of new columns using a regular expression. If the pattern contains marked groups, the values are concatenated together; otherwise the whole match is returned. The original column will be removed from the table. - The new columns will be named with the name of the input with a + The new columns will be named with the name of the input column with a incrementing number after. Arguments: - - column: The column to tokenize the text of. + - column: The name or index of the column to tokenize the text of. - pattern: The pattern used to find within the text. - case_sensitivity: Specifies if the text values should be compared case sensitively. @@ -1443,7 +1445,7 @@ type Table tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Auto on_problems=Report_Error = _ = [column pattern case_sensitivity column_count on_problems] - Unimplemented.throw "This is an interface only." + Error.throw (Unsupported_Database_Operation.Error "Table.tokenize_to_columns is not implemented yet for the Database backends.") ## Tokenizes a column of text into a set of new rows using a regular expression. @@ -1452,7 +1454,7 @@ type Table The values of other columns are repeated for the new rows. Arguments: - - column: The column to split the text of. + - column: The name or index of the column to tokenize the text of. - pattern: The pattern used to find within the text. - case_sensitivity: Specifies if the text values should be compared case sensitively. @@ -1460,7 +1462,7 @@ type Table tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = _ = [column pattern case_sensitivity on_problems] - Unimplemented.throw "This is an interface only." + Error.throw (Unsupported_Database_Operation.Error "Table.tokenize_to_rows is not implemented yet for the Database backends.") ## PRIVATE UNSTABLE diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index 018b1dd61136..f0e75c09dd24 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -921,11 +921,11 @@ type Table ## Splits a column of text into a set of new columns. The original column will be removed from the table. - The new columns will be named with the name of the input with a + The new columns will be named with the name of the input column with a incrementing number after. Arguments: - - column: The column to split the text of. + - column: The name or index of the column to split the text of. - delimiter: The term or terms used to split the text. - column_count: The number of columns to split to. If `Auto` then columns will be added to fit all data. @@ -940,7 +940,7 @@ type Table The values of other columns are repeated for the new rows. Arguments: - - column: The column to split the text of. + - column: The name or index of the column to split the text of. - delimiter: The term or terms used to split the text. - on_problems: Specifies the behavior when a problem occurs. split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table @@ -952,11 +952,11 @@ type Table If the pattern contains marked groups, the values are concatenated together; otherwise the whole match is returned. The original column will be removed from the table. - The new columns will be named with the name of the input with a + The new columns will be named with the name of the input column with a incrementing number after. Arguments: - - column: The column to tokenize the text of. + - column: The name or index of the column to tokenize the text of. - pattern: The pattern used to find within the text. - case_sensitivity: Specifies if the text values should be compared case sensitively. @@ -976,7 +976,7 @@ type Table The values of other columns are repeated for the new rows. Arguments: - - column: The column to split the text of. + - column: The name or index of the column to tokenize the text of. - pattern: The pattern used to find within the text. - case_sensitivity: Specifies if the text values should be compared case sensitively. From b6db6d81804655e28141f3418dd996c6f67c1c2c Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 11:43:06 -0400 Subject: [PATCH 46/64] Auto to Nothing --- .../Database/0.0.0-dev/src/Data/Table.enso | 12 ++++++------ .../Table/0.0.0-dev/src/Data/Table.enso | 12 ++++++------ .../Standard/Table/0.0.0-dev/src/Errors.enso | 2 +- .../0.0.0-dev/src/Internal/Split_Tokenize.enso | 17 ++++++++--------- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index e72c3ad80974..256d74c5343f 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1401,14 +1401,14 @@ type Table - column: The name or index of the column to split the text of. - delimiter: The term or terms used to split the text. - column_count: The number of columns to split to. - If `Auto` then columns will be added to fit all data. + If `Nothing` then columns will be added to fit all data. - on_problems: Specifies the behavior when a problem occurs. ! Error Conditions If the data exceeds the `column_count`, a `Column_Count_Exceeded` will be reported according to the `on_problems` behavior. - split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table - split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = + split_to_columns : Text | Integer -> Text -> Integer | Nothing -> Problem_Behavior -> Table + split_to_columns self column delimiter="," column_count=Nothing on_problems=Report_Error = _ = [column delimiter column_count on_problems] Error.throw (Unsupported_Database_Operation.Error "Table.split_to_columns is not implemented yet for the Database backends.") @@ -1438,12 +1438,12 @@ type Table - case_sensitivity: Specifies if the text values should be compared case sensitively. - column_count: The number of columns to split to. - If `Auto` then columns will be added to fit all data. + If `Nothing` then columns will be added to fit all data. If the data exceeds the `column_count`, a `Column_Count_Exceeded` error will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. - tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table - tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Auto on_problems=Report_Error = + tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Integer | Nothing -> Problem_Behavior -> Table + tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Nothing on_problems=Report_Error = _ = [column pattern case_sensitivity column_count on_problems] Error.throw (Unsupported_Database_Operation.Error "Table.tokenize_to_columns is not implemented yet for the Database backends.") diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index f0e75c09dd24..d90f9a7179c0 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -928,12 +928,12 @@ type Table - column: The name or index of the column to split the text of. - delimiter: The term or terms used to split the text. - column_count: The number of columns to split to. - If `Auto` then columns will be added to fit all data. + If `Nothing` then columns will be added to fit all data. If the data exceeds the `column_count`, a `Column_Count_Exceeded` error will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. - split_to_columns : Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table - split_to_columns self column delimiter="," column_count=Auto on_problems=Report_Error = + split_to_columns : Text | Integer -> Text -> Integer | Nothing -> Problem_Behavior -> Table + split_to_columns self column delimiter="," column_count=Nothing on_problems=Report_Error = Split_Tokenize.split_to_columns self column delimiter column_count on_problems ## Splits a column of text into a set of new rows. @@ -961,12 +961,12 @@ type Table - case_sensitivity: Specifies if the text values should be compared case sensitively. - column_count: The number of columns to split to. - If `Auto` then columns will be added to fit all data. + If `Nothing` then columns will be added to fit all data. If the data exceeds the `column_count`, a `Column_Count_Exceeded` error will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. - tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table - tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Auto on_problems=Report_Error = + tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Integer | Nothing -> Problem_Behavior -> Table + tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Nothing on_problems=Report_Error = Split_Tokenize.tokenize_to_columns self column pattern case_sensitivity column_count on_problems ## Tokenizes a column of text into a set of new rows using a regular diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso index a7d7ce13135c..e478139156df 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Errors.enso @@ -564,4 +564,4 @@ type Column_Count_Exceeded Create a human-readable version of the error. to_display_text : Text to_display_text self = - "The operation produced more columns than the specified limit. The limit is "+self.limit.to_text+" and the number of new columns was "+self.column_count.to_text+". The limit may be turned off by setting the `limit` option to `Auto`." + "The operation produced more columns than the specified limit. The limit is "+self.limit.to_text+" and the number of new columns was "+self.column_count.to_text+". The limit may be turned off by setting the `limit` option to `Nothing`." diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 1a8944bcb20b..66dbfa1086ed 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -6,7 +6,6 @@ import project.Internal.Java_Exports import project.Internal.Problem_Builder.Problem_Builder import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy -from project.Data.Type.Value_Type import Auto from Standard.Table import Value_Type from Standard.Table.Errors import Column_Count_Exceeded, Duplicate_Output_Column_Names, Invalid_Value_Type, Missing_Input_Columns from project.Internal.Java_Exports import make_string_builder @@ -16,8 +15,8 @@ polyglot java import org.enso.table.data.mask.OrderMask ## PRIVATE Splits a column of text into a set of new columns. See `Table.split_to_columns`. -split_to_columns : Table -> Text | Integer -> Text -> Auto | Integer -> Problem_Behavior -> Table -split_to_columns table input_column_name delimiter="," column_count=Auto on_problems=Report_Error = +split_to_columns : Table -> Text | Integer -> Text -> Integer | Nothing -> Problem_Behavior -> Table +split_to_columns table input_column_name delimiter="," column_count=Nothing on_problems=Report_Error = expect_text_column table input_column_name <| fan_out_to_columns table input_column_name (ignore_nothing (_.split delimiter)) column_count on_problems @@ -34,7 +33,7 @@ split_to_rows table input_column_name delimiter="," on_problems=Report_Error = Tokenizes a column of text into a set of new columns using a regular expression. See `Table.tokenize_to_columns`. -tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Auto | Integer -> Problem_Behavior -> Table +tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Integer | Nothing -> Problem_Behavior -> Table tokenize_to_columns table input_column_name pattern case_sensitivity column_count on_problems = expect_text_column table input_column_name <| fan_out_to_columns table input_column_name (ignore_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems @@ -59,8 +58,8 @@ tokenize_to_rows table input_column_name pattern="." case_sensitivity=Case_Sensi - input_column: The column to transform. - function: A function that transforms a single element of `input_column` to multiple values. -fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Auto | Integer -> Problem_Behavior -> Table | Nothing -fan_out_to_columns table input_column_name function column_count=Auto on_problems=Report_Error = +fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Integer | Nothing -> Problem_Behavior -> Table | Nothing +fan_out_to_columns table input_column_name function column_count=Nothing on_problems=Report_Error = input_column = table.get input_column_name problem_builder = Problem_Builder.new new_columns_unrenamed = map_columns_to_multiple input_column function column_count problem_builder @@ -131,17 +130,17 @@ fan_out_to_rows table input_column_name function = - function: A function that transforms a single element of `input_column` to multiple values. - column_count: The number of columns to split to. - If `Auto` then columns will be added to fit all data. + If `Nothing` then columns will be added to fit all data. If the data exceeds the `column_count`, a `Column_Count_Exceeded` error will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. -map_columns_to_multiple : Column -> (Any -> Vector Any) -> Auto | Integer -> Problem_Builder -> Vector Column +map_columns_to_multiple : Column -> (Any -> Vector Any) -> Integer | Nothing -> Problem_Builder -> Vector Column map_columns_to_multiple input_column function column_count problem_builder = num_rows = input_column.length input_storage = input_column.java_column.getStorage builders = case column_count of - Auto -> + Nothing -> builders = Vector.new_builder 0.up_to num_rows . map i-> From b57666c95b7816a6718710185e0766f4f063ead8 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 11:46:11 -0400 Subject: [PATCH 47/64] review --- .../Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 66dbfa1086ed..6c08c820f228 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -6,8 +6,8 @@ import project.Internal.Java_Exports import project.Internal.Problem_Builder.Problem_Builder import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy -from Standard.Table import Value_Type -from Standard.Table.Errors import Column_Count_Exceeded, Duplicate_Output_Column_Names, Invalid_Value_Type, Missing_Input_Columns +from project import Value_Type +from project.Errors import Column_Count_Exceeded, Duplicate_Output_Column_Names, Invalid_Value_Type, Missing_Input_Columns from project.Internal.Java_Exports import make_string_builder polyglot java import org.enso.table.data.mask.OrderMask From 5c0d66b6b0d4a723af7964c9988a401eda53b6a1 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 11:47:31 -0400 Subject: [PATCH 48/64] review --- .../src/Internal/Split_Tokenize.enso | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 6c08c820f228..5748f6657ad3 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -16,37 +16,37 @@ polyglot java import org.enso.table.data.mask.OrderMask Splits a column of text into a set of new columns. See `Table.split_to_columns`. split_to_columns : Table -> Text | Integer -> Text -> Integer | Nothing -> Problem_Behavior -> Table -split_to_columns table input_column_name delimiter="," column_count=Nothing on_problems=Report_Error = - expect_text_column table input_column_name <| - fan_out_to_columns table input_column_name (ignore_nothing (_.split delimiter)) column_count on_problems +split_to_columns table input_column_id delimiter="," column_count=Nothing on_problems=Report_Error = + expect_text_column table input_column_id <| + fan_out_to_columns table input_column_id (ignore_nothing (_.split delimiter)) column_count on_problems ## PRIVATE Splits a column of text into a set of new rows. See `Table.split_to_rows`. split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table -split_to_rows table input_column_name delimiter="," on_problems=Report_Error = +split_to_rows table input_column_id delimiter="," on_problems=Report_Error = _ = [on_problems] - expect_text_column table input_column_name <| - fan_out_to_rows table input_column_name (ignore_nothing (_.split delimiter)) + expect_text_column table input_column_id <| + fan_out_to_rows table input_column_id (ignore_nothing (_.split delimiter)) ## PRIVATE Tokenizes a column of text into a set of new columns using a regular expression. See `Table.tokenize_to_columns`. tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Integer | Nothing -> Problem_Behavior -> Table -tokenize_to_columns table input_column_name pattern case_sensitivity column_count on_problems = - expect_text_column table input_column_name <| - fan_out_to_columns table input_column_name (ignore_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems +tokenize_to_columns table input_column_id pattern case_sensitivity column_count on_problems = + expect_text_column table input_column_id <| + fan_out_to_columns table input_column_id (ignore_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems ## PRIVATE Tokenizes a column of text into a set of new rows using a regular expression. See `Table.tokenize_to_rows`. tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table -tokenize_to_rows table input_column_name pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = +tokenize_to_rows table input_column_id pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = _ = [on_problems] - expect_text_column table input_column_name <| - fan_out_to_rows table input_column_name (ignore_nothing (_.tokenize pattern case_sensitivity)) + expect_text_column table input_column_id <| + fan_out_to_rows table input_column_id (ignore_nothing (_.tokenize pattern case_sensitivity)) ## PRIVATE Transform a table by transforming a column into a set of columns. Takes a @@ -59,8 +59,8 @@ tokenize_to_rows table input_column_name pattern="." case_sensitivity=Case_Sensi - function: A function that transforms a single element of `input_column` to multiple values. fan_out_to_columns : Table -> Text | Integer -> (Any -> Vector Any) -> Integer | Nothing -> Problem_Behavior -> Table | Nothing -fan_out_to_columns table input_column_name function column_count=Nothing on_problems=Report_Error = - input_column = table.get input_column_name +fan_out_to_columns table input_column_id function column_count=Nothing on_problems=Report_Error = + input_column = table.get input_column_id problem_builder = Problem_Builder.new new_columns_unrenamed = map_columns_to_multiple input_column function column_count problem_builder new_columns = rename_new_columns table new_columns_unrenamed problem_builder @@ -79,8 +79,8 @@ fan_out_to_columns table input_column_name function column_count=Nothing on_prob - function: A function that transforms a single element of `input_column` to multiple values. fan_out_to_rows : Table -> Text | Integer -> (Any -> Vector Any) -> Table -fan_out_to_rows table input_column_name function = - input_column = table.at input_column_name +fan_out_to_rows table input_column_id function = + input_column = table.at input_column_id input_storage = input_column.java_column.getStorage num_input_rows = input_storage.size @@ -99,14 +99,14 @@ fan_out_to_rows table input_column_name function = # Build the output column output_storage = output_column_builder.seal - output_column = Column.from_storage input_column_name output_storage + output_column = Column.from_storage input_column_id output_storage # Build the order mask. order_mask = OrderMask.new (order_mask_positions.to_vector) # Build the other columns, and include the output_column while doing it. new_columns = table.columns.map column-> - case column.name == input_column_name of + case column.name == input_column_id of True -> # Replace the input column with the output column. output_column From 4f66416371b823affc14adc4dd98bc6c379478db Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 11:48:48 -0400 Subject: [PATCH 49/64] max --- .../Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 5748f6657ad3..11b9a300dc18 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -180,7 +180,7 @@ map_columns_to_multiple input_column function column_count problem_builder = output_values.length - max_output_length = output_lengths.fold 0 .max + max_output_length = maximum output_lengths if max_output_length > column_count then problem = Column_Count_Exceeded.Error column_count max_output_length From ab8318cbc6aab87626e917de968065268b1072da Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 11:59:48 -0400 Subject: [PATCH 50/64] should_equal_verbose --- .../src/In_Memory/Split_Tokenize_Spec.enso | 39 +++++++------------ test/Table_Tests/src/Util.enso | 10 +++++ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index 9dea97a6ed04..7920f8c013a7 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -6,18 +6,9 @@ import Standard.Test.Extensions from Standard.Table import Table from Standard.Table.Errors import Invalid_Value_Type, Column_Count_Exceeded, Duplicate_Output_Column_Names, Missing_Input_Columns from Standard.Test import Test, Test_Suite, Problems +from project.Util import all spec = - tables_equal t0 t1 = - same_headers = (t0.columns.map .name) == (t1.columns.map .name) - same_columns = (t0.columns.map .to_vector) == (t1.columns.map .to_vector) - same_headers && same_columns - tables_should_be_equal actual expected = - equal = tables_equal actual expected - if equal.not then - msg = 'Tables differ.\nActual:\n' + actual.display + '\nExpected:\n' + expected.display - Test.fail msg - Test.group 'split' <| Test.specify 'can do split_to_columns' <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] @@ -25,7 +16,7 @@ spec = expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.split_to_columns 'bar' 'b' - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.specify 'can do split_to_rows' <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] @@ -33,7 +24,7 @@ spec = expected_rows = [[0, 'a'], [0, 'c'], [1, 'c'], [1, 'd'], [1, 'ef'], [2, 'gh'], [2, 'ij'], [2, 'u']] expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.split_to_rows 'bar' 'b' - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.group 'tokenize' <| Test.specify 'can do tokenize_to_columns' <| @@ -42,7 +33,7 @@ spec = expected_rows = [[0, '12', '34', '5'], [1, '23', Nothing, Nothing], [2, '2', '4', '55']] expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.tokenize_to_columns 'bar' "\d+" - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.specify 'can do tokenize_to_rows' <| cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] @@ -50,7 +41,7 @@ spec = expected_rows = [[0, '12'], [0, '34'], [0, '5'], [1, '23'], [2, '2'], [2, '4'], [2, '55']] expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.tokenize_to_rows 'bar' "\d+" - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.specify "can do tokenize_to_columns with groups" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] @@ -58,7 +49,7 @@ spec = expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.specify "can do tokenize_to_rows with groups" <| cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] @@ -66,7 +57,7 @@ spec = expected_rows = [[0, 'a1'], [0, 'b12'], [0, 'd50'], [1, 'b10'], [1, 'c20']] expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.specify 'can do tokenize_to_columns case-insensitively' <| cols = [['foo', [0, 1, 2]], ['bar', ['aBqcE', 'qcBr', 'cCb']]] @@ -74,7 +65,7 @@ spec = expected_rows = [[0, 'B', 'c', Nothing], [1, 'c', 'B', Nothing], [2, 'c', 'C', 'b']] expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows t2 = t.tokenize_to_columns 'bar' "[bc]" case_sensitivity=Case_Sensitivity.Insensitive - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.specify 'can do tokenize_to_rows case-insensitively' <| cols = [['foo', [0, 1, 2]], ['bar', ['aBqcE', 'qcBr', 'cCb']]] @@ -82,7 +73,7 @@ spec = expected_rows = [[0, 'B'], [0, 'c'], [1, 'c'], [1, 'B'], [2, 'c'], [2, 'C'], [2, 'b']] expected = Table.from_rows ['foo', 'bar'] expected_rows t2 = t.tokenize_to_rows 'bar' "[bc]" case_sensitivity=Case_Sensitivity.Insensitive - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.group 'column count' <| Test.specify 'should generate extra empty columns if column_count is set' <| @@ -91,7 +82,7 @@ spec = expected_rows = [[0, 'a', 'c', Nothing, Nothing], [1, 'c', 'd', 'ef', Nothing], [2, 'gh', 'ij', 'u', Nothing]] expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2', 'bar_3'] expected_rows t2 = t.split_to_columns 'bar' 'b' column_count=4 - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.specify "split should limit columns and return problems when exceeding the column limit" <| cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] @@ -99,7 +90,7 @@ spec = expected_rows = [[0, 'a', 'c'], [1, 'c', 'd'], [2, 'gh', 'ij']] expected = Table.from_rows ['foo', 'bar_0', 'bar_1'] expected_rows action = t.split_to_columns 'bar' 'b' column_count=2 on_problems=_ - tester = t-> tables_should_be_equal t expected + tester = t-> t.should_equal_verbose expected problems = [Column_Count_Exceeded.Error 2 3] Problems.test_problem_handling action problems tester @@ -109,7 +100,7 @@ spec = expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] expected = Table.from_rows ['foo', 'bar_0', 'bar_1'] expected_rows action = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" column_count=2 on_problems=_ - tester = t-> tables_should_be_equal t expected + tester = t-> t.should_equal_verbose expected problems = [Column_Count_Exceeded.Error 2 3] Problems.test_problem_handling action problems tester @@ -119,7 +110,7 @@ spec = expected_rows = [[0, 'gh', 'ij', 'u', Nothing], [1, 'c', 'd', 'ef', Nothing], [2, 'a', 'c', Nothing, Nothing]] expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2', 'bar_3'] expected_rows t2 = t.split_to_columns 'bar' 'b' column_count=4 - tables_should_be_equal t2 expected + t2.should_equal_verbose expected Test.group 'errors' <| Test.specify "won't work on a non-text column" <| @@ -155,7 +146,7 @@ spec = expected_rows = [[0, 'a', 'c', Nothing, 'a'], [1, 'c', 'd', 'ef', 'b'], [2, 'gh', 'ij', 'u', 'c']] expected = Table.from_rows ['foo', 'bar_0', 'bar_1_1', 'bar_2', 'bar_1'] expected_rows action = t.split_to_columns 'bar' 'b' on_problems=_ - tester = t-> tables_should_be_equal t expected + tester = t-> t.should_equal_verbose expected problems = [Duplicate_Output_Column_Names.Error ['bar_1']] Problems.test_problem_handling action problems tester @@ -166,6 +157,6 @@ spec = expected_rows = [[0, 'a', 'c', Nothing, 1], [1, 'c', 'd', 'ef', 2], [2, 'gh', 'ij', 'u', 3]] expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2', 'baz'] expected_rows t2 = t.split_to_columns 'bar' 'b' - tables_should_be_equal t2 expected + t2.should_equal_verbose expected main = Test_Suite.run_main spec diff --git a/test/Table_Tests/src/Util.enso b/test/Table_Tests/src/Util.enso index 01a9b01458aa..7d6572de90b6 100644 --- a/test/Table_Tests/src/Util.enso +++ b/test/Table_Tests/src/Util.enso @@ -13,6 +13,16 @@ Table.should_equal self expected = self_cols.map .name . should_equal (that_cols.map .name) frames_to_skip=1 self_cols.map .to_vector . should_equal (that_cols.map .to_vector) frames_to_skip=1 +Table.should_equal_verbose self expected = + tables_equal t0 t1 = + same_headers = (t0.columns.map .name) == (t1.columns.map .name) + same_columns = (t0.columns.map .to_vector) == (t1.columns.map .to_vector) + same_headers && same_columns + equal = tables_equal self expected + if equal.not then + msg = 'Tables differ.\nself:\n' + self.display + '\nExpected:\n' + expected.display + Test.fail msg + Column.should_equal self expected = if self.name != expected.name then Test.fail "Expected column name "+expected.name+", but got "+self.name+"." From 851709bbe7cf39f5c9c727c464ea216ec4bd4e43 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 12:08:13 -0400 Subject: [PATCH 51/64] double quotes --- .../src/In_Memory/Split_Tokenize_Spec.enso | 170 +++++++++--------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index 7920f8c013a7..b2f7be1c29e0 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -9,154 +9,154 @@ from Standard.Test import Test, Test_Suite, Problems from project.Util import all spec = - Test.group 'split' <| - Test.specify 'can do split_to_columns' <| - cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + Test.group "split" <| + Test.specify "can do split_to_columns" <| + cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]]] t = Table.new cols - expected_rows = [[0, 'a', 'c', Nothing], [1, 'c', 'd', 'ef'], [2, 'gh', 'ij', 'u']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows - t2 = t.split_to_columns 'bar' 'b' + expected_rows = [[0, "a", "c", Nothing], [1, "c", "d", "ef"], [2, "gh", "ij", "u"]] + expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2"] expected_rows + t2 = t.split_to_columns "bar" "b" t2.should_equal_verbose expected - Test.specify 'can do split_to_rows' <| - cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + Test.specify "can do split_to_rows" <| + cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]]] t = Table.new cols - expected_rows = [[0, 'a'], [0, 'c'], [1, 'c'], [1, 'd'], [1, 'ef'], [2, 'gh'], [2, 'ij'], [2, 'u']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.split_to_rows 'bar' 'b' + expected_rows = [[0, "a"], [0, "c"], [1, "c"], [1, "d"], [1, "ef"], [2, "gh"], [2, "ij"], [2, "u"]] + expected = Table.from_rows ["foo", "bar"] expected_rows + t2 = t.split_to_rows "bar" "b" t2.should_equal_verbose expected - Test.group 'tokenize' <| - Test.specify 'can do tokenize_to_columns' <| - cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] + Test.group "tokenize" <| + Test.specify "can do tokenize_to_columns" <| + cols = [["foo", [0, 1, 2]], ["bar", ["a12b34r5", "23", "2r4r55"]]] t = Table.new cols - expected_rows = [[0, '12', '34', '5'], [1, '23', Nothing, Nothing], [2, '2', '4', '55']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows - t2 = t.tokenize_to_columns 'bar' "\d+" + expected_rows = [[0, "12", "34", "5"], [1, "23", Nothing, Nothing], [2, "2", "4", "55"]] + expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2"] expected_rows + t2 = t.tokenize_to_columns "bar" "\d+" t2.should_equal_verbose expected - Test.specify 'can do tokenize_to_rows' <| - cols = [['foo', [0, 1, 2]], ['bar', ['a12b34r5', '23', '2r4r55']]] + Test.specify "can do tokenize_to_rows" <| + cols = [["foo", [0, 1, 2]], ["bar", ["a12b34r5", "23", "2r4r55"]]] t = Table.new cols - expected_rows = [[0, '12'], [0, '34'], [0, '5'], [1, '23'], [2, '2'], [2, '4'], [2, '55']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.tokenize_to_rows 'bar' "\d+" + expected_rows = [[0, "12"], [0, "34"], [0, "5"], [1, "23"], [2, "2"], [2, "4"], [2, "55"]] + expected = Table.from_rows ["foo", "bar"] expected_rows + t2 = t.tokenize_to_rows "bar" "\d+" t2.should_equal_verbose expected Test.specify "can do tokenize_to_columns with groups" <| - cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols - expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows - t2 = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" + expected_rows = [[0, "a1", "b12", "d50"], [1, "b10", "c20", Nothing]] + expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2"] expected_rows + t2 = t.tokenize_to_columns "bar" "([a-z]).(\d+)" t2.should_equal_verbose expected Test.specify "can do tokenize_to_rows with groups" <| - cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols - expected_rows = [[0, 'a1'], [0, 'b12'], [0, 'd50'], [1, 'b10'], [1, 'c20']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.tokenize_to_rows 'bar' "([a-z]).(\d+)" + expected_rows = [[0, "a1"], [0, "b12"], [0, "d50"], [1, "b10"], [1, "c20"]] + expected = Table.from_rows ["foo", "bar"] expected_rows + t2 = t.tokenize_to_rows "bar" "([a-z]).(\d+)" t2.should_equal_verbose expected - Test.specify 'can do tokenize_to_columns case-insensitively' <| - cols = [['foo', [0, 1, 2]], ['bar', ['aBqcE', 'qcBr', 'cCb']]] + Test.specify "can do tokenize_to_columns case-insensitively" <| + cols = [["foo", [0, 1, 2]], ["bar", ["aBqcE", "qcBr", "cCb"]]] t = Table.new cols - expected_rows = [[0, 'B', 'c', Nothing], [1, 'c', 'B', Nothing], [2, 'c', 'C', 'b']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2'] expected_rows - t2 = t.tokenize_to_columns 'bar' "[bc]" case_sensitivity=Case_Sensitivity.Insensitive + expected_rows = [[0, "B", "c", Nothing], [1, "c", "B", Nothing], [2, "c", "C", "b"]] + expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2"] expected_rows + t2 = t.tokenize_to_columns "bar" "[bc]" case_sensitivity=Case_Sensitivity.Insensitive t2.should_equal_verbose expected - Test.specify 'can do tokenize_to_rows case-insensitively' <| - cols = [['foo', [0, 1, 2]], ['bar', ['aBqcE', 'qcBr', 'cCb']]] + Test.specify "can do tokenize_to_rows case-insensitively" <| + cols = [["foo", [0, 1, 2]], ["bar", ["aBqcE", "qcBr", "cCb"]]] t = Table.new cols - expected_rows = [[0, 'B'], [0, 'c'], [1, 'c'], [1, 'B'], [2, 'c'], [2, 'C'], [2, 'b']] - expected = Table.from_rows ['foo', 'bar'] expected_rows - t2 = t.tokenize_to_rows 'bar' "[bc]" case_sensitivity=Case_Sensitivity.Insensitive + expected_rows = [[0, "B"], [0, "c"], [1, "c"], [1, "B"], [2, "c"], [2, "C"], [2, "b"]] + expected = Table.from_rows ["foo", "bar"] expected_rows + t2 = t.tokenize_to_rows "bar" "[bc]" case_sensitivity=Case_Sensitivity.Insensitive t2.should_equal_verbose expected - Test.group 'column count' <| - Test.specify 'should generate extra empty columns if column_count is set' <| - cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + Test.group "column count" <| + Test.specify "should generate extra empty columns if column_count is set" <| + cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]]] t = Table.new cols - expected_rows = [[0, 'a', 'c', Nothing, Nothing], [1, 'c', 'd', 'ef', Nothing], [2, 'gh', 'ij', 'u', Nothing]] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2', 'bar_3'] expected_rows - t2 = t.split_to_columns 'bar' 'b' column_count=4 + expected_rows = [[0, "a", "c", Nothing, Nothing], [1, "c", "d", "ef", Nothing], [2, "gh", "ij", "u", Nothing]] + expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2", "bar_3"] expected_rows + t2 = t.split_to_columns "bar" "b" column_count=4 t2.should_equal_verbose expected Test.specify "split should limit columns and return problems when exceeding the column limit" <| - cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']]] + cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]]] t = Table.new cols - expected_rows = [[0, 'a', 'c'], [1, 'c', 'd'], [2, 'gh', 'ij']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1'] expected_rows - action = t.split_to_columns 'bar' 'b' column_count=2 on_problems=_ + expected_rows = [[0, "a", "c"], [1, "c", "d"], [2, "gh", "ij"]] + expected = Table.from_rows ["foo", "bar_0", "bar_1"] expected_rows + action = t.split_to_columns "bar" "b" column_count=2 on_problems=_ tester = t-> t.should_equal_verbose expected problems = [Column_Count_Exceeded.Error 2 3] Problems.test_problem_handling action problems tester Test.specify "tokenize should limit columns and return problems when exceeding the column limit" <| - cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols - expected_rows = [[0, 'a1', 'b12', 'd50'], [1, 'b10', 'c20', Nothing]] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1'] expected_rows - action = t.tokenize_to_columns 'bar' "([a-z]).(\d+)" column_count=2 on_problems=_ + expected_rows = [[0, "a1", "b12", "d50"], [1, "b10", "c20", Nothing]] + expected = Table.from_rows ["foo", "bar_0", "bar_1"] expected_rows + action = t.tokenize_to_columns "bar" "([a-z]).(\d+)" column_count=2 on_problems=_ tester = t-> t.should_equal_verbose expected problems = [Column_Count_Exceeded.Error 2 3] Problems.test_problem_handling action problems tester - Test.specify 'should generate extra empty columns if column_count is set (reverse row order)' <| - cols = [['foo', [0, 1, 2]], ['bar', ['ghbijbu', 'cbdbef', 'abc']]] + Test.specify "should generate extra empty columns if column_count is set (reverse row order)" <| + cols = [["foo", [0, 1, 2]], ["bar", ["ghbijbu", "cbdbef", "abc"]]] t = Table.new cols - expected_rows = [[0, 'gh', 'ij', 'u', Nothing], [1, 'c', 'd', 'ef', Nothing], [2, 'a', 'c', Nothing, Nothing]] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2', 'bar_3'] expected_rows - t2 = t.split_to_columns 'bar' 'b' column_count=4 + expected_rows = [[0, "gh", "ij", "u", Nothing], [1, "c", "d", "ef", Nothing], [2, "a", "c", Nothing, Nothing]] + expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2", "bar_3"] expected_rows + t2 = t.split_to_columns "bar" "b" column_count=4 t2.should_equal_verbose expected - Test.group 'errors' <| + Test.group "errors" <| Test.specify "won't work on a non-text column" <| - cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols - t.split_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type - t.split_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type - t.tokenize_to_columns 'foo' "x" . should_fail_with Invalid_Value_Type - t.tokenize_to_rows 'foo' "x" . should_fail_with Invalid_Value_Type + t.split_to_columns "foo" "x" . should_fail_with Invalid_Value_Type + t.split_to_rows "foo" "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_columns "foo" "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_rows "foo" "x" . should_fail_with Invalid_Value_Type Test.specify "won't work on a mixed column" <| - cols = [['foo', [0, 1]], ['bar', [500, 'ab-10:bc-20c']]] + cols = [["foo", [0, 1]], ["bar", [500, "ab-10:bc-20c"]]] t = Table.new cols - t.split_to_columns 'bar' "x" . should_fail_with Invalid_Value_Type - t.split_to_rows 'bar' "x" . should_fail_with Invalid_Value_Type - t.tokenize_to_columns 'bar' "x" . should_fail_with Invalid_Value_Type - t.tokenize_to_rows 'bar' "x" . should_fail_with Invalid_Value_Type + t.split_to_columns "bar" "x" . should_fail_with Invalid_Value_Type + t.split_to_rows "bar" "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_columns "bar" "x" . should_fail_with Invalid_Value_Type + t.tokenize_to_rows "bar" "x" . should_fail_with Invalid_Value_Type Test.specify "*_to_columns handles missing input column" <| - cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols - t.tokenize_to_columns 'invalid_name' "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns + t.tokenize_to_columns "invalid_name" "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns Test.specify "*_to_rows handles missing input column" <| - cols = [['foo', [0, 1]], ['bar', ['r a-1, b-12,qd-50', 'ab-10:bc-20c']]] + cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols - t.tokenize_to_rows 'invalid_name' "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns + t.tokenize_to_rows "invalid_name" "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns Test.group "name conflicts" <| - Test.specify 'will make column names unique' <| - cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']], ['bar_1', ['a', 'b', 'c']]] + Test.specify "will make column names unique" <| + cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]], ["bar_1", ["a", "b", "c"]]] t = Table.new cols - expected_rows = [[0, 'a', 'c', Nothing, 'a'], [1, 'c', 'd', 'ef', 'b'], [2, 'gh', 'ij', 'u', 'c']] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1_1', 'bar_2', 'bar_1'] expected_rows - action = t.split_to_columns 'bar' 'b' on_problems=_ + expected_rows = [[0, "a", "c", Nothing, "a"], [1, "c", "d", "ef", "b"], [2, "gh", "ij", "u", "c"]] + expected = Table.from_rows ["foo", "bar_0", "bar_1_1", "bar_2", "bar_1"] expected_rows + action = t.split_to_columns "bar" "b" on_problems=_ tester = t-> t.should_equal_verbose expected - problems = [Duplicate_Output_Column_Names.Error ['bar_1']] + problems = [Duplicate_Output_Column_Names.Error ["bar_1"]] Problems.test_problem_handling action problems tester Test.group "column order" <| - Test.specify 'preserves column order' <| - cols = [['foo', [0, 1, 2]], ['bar', ['abc', 'cbdbef', 'ghbijbu']], ['baz', [1, 2, 3]]] + Test.specify "preserves column order" <| + cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]], ["baz", [1, 2, 3]]] t = Table.new cols - expected_rows = [[0, 'a', 'c', Nothing, 1], [1, 'c', 'd', 'ef', 2], [2, 'gh', 'ij', 'u', 3]] - expected = Table.from_rows ['foo', 'bar_0', 'bar_1', 'bar_2', 'baz'] expected_rows - t2 = t.split_to_columns 'bar' 'b' + expected_rows = [[0, "a", "c", Nothing, 1], [1, "c", "d", "ef", 2], [2, "gh", "ij", "u", 3]] + expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2", "baz"] expected_rows + t2 = t.split_to_columns "bar" "b" t2.should_equal_verbose expected main = Test_Suite.run_main spec From 4e0abe5457f8ea2620f5cf8771387ab4687c5aea Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 12:21:18 -0400 Subject: [PATCH 52/64] check empty column --- test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index b2f7be1c29e0..8ea5018580e7 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -83,6 +83,7 @@ spec = expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2", "bar_3"] expected_rows t2 = t.split_to_columns "bar" "b" column_count=4 t2.should_equal_verbose expected + t2.at "bar_3" . value_type . is_text . should_be_true Test.specify "split should limit columns and return problems when exceeding the column limit" <| cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]]] @@ -111,6 +112,7 @@ spec = expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2", "bar_3"] expected_rows t2 = t.split_to_columns "bar" "b" column_count=4 t2.should_equal_verbose expected + t2.at "bar_3" . value_type . is_text . should_be_true Test.group "errors" <| Test.specify "won't work on a non-text column" <| From 5540c3475b7b2115a60df7136ef0b17eda9a104d Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 12:23:03 -0400 Subject: [PATCH 53/64] group names --- .../src/In_Memory/Split_Tokenize_Spec.enso | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index 8ea5018580e7..d6886d06ad82 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -9,7 +9,7 @@ from Standard.Test import Test, Test_Suite, Problems from project.Util import all spec = - Test.group "split" <| + Test.group "Table.split" <| Test.specify "can do split_to_columns" <| cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]]] t = Table.new cols @@ -26,7 +26,7 @@ spec = t2 = t.split_to_rows "bar" "b" t2.should_equal_verbose expected - Test.group "tokenize" <| + Test.group "Table.tokenize" <| Test.specify "can do tokenize_to_columns" <| cols = [["foo", [0, 1, 2]], ["bar", ["a12b34r5", "23", "2r4r55"]]] t = Table.new cols @@ -75,7 +75,7 @@ spec = t2 = t.tokenize_to_rows "bar" "[bc]" case_sensitivity=Case_Sensitivity.Insensitive t2.should_equal_verbose expected - Test.group "column count" <| + Test.group "Table.split/tokenize column count" <| Test.specify "should generate extra empty columns if column_count is set" <| cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]]] t = Table.new cols @@ -114,7 +114,7 @@ spec = t2.should_equal_verbose expected t2.at "bar_3" . value_type . is_text . should_be_true - Test.group "errors" <| + Test.group "Table.split/tokenize errors" <| Test.specify "won't work on a non-text column" <| cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols @@ -141,7 +141,7 @@ spec = t = Table.new cols t.tokenize_to_rows "invalid_name" "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns - Test.group "name conflicts" <| + Test.group "Table.split/tokenize name conflicts" <| Test.specify "will make column names unique" <| cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]], ["bar_1", ["a", "b", "c"]]] t = Table.new cols @@ -152,7 +152,7 @@ spec = problems = [Duplicate_Output_Column_Names.Error ["bar_1"]] Problems.test_problem_handling action problems tester - Test.group "column order" <| + Test.group "Table.split/tokenize column order" <| Test.specify "preserves column order" <| cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]], ["baz", [1, 2, 3]]] t = Table.new cols From 0d00b56c7a3807dd39bfca9e50dfbcf5105c72e8 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 12:27:24 -0400 Subject: [PATCH 54/64] tok name conflict test --- .../src/In_Memory/Split_Tokenize_Spec.enso | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index d6886d06ad82..5fc057900ad4 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -142,7 +142,7 @@ spec = t.tokenize_to_rows "invalid_name" "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns Test.group "Table.split/tokenize name conflicts" <| - Test.specify "will make column names unique" <| + Test.specify "split will make column names unique" <| cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]], ["bar_1", ["a", "b", "c"]]] t = Table.new cols expected_rows = [[0, "a", "c", Nothing, "a"], [1, "c", "d", "ef", "b"], [2, "gh", "ij", "u", "c"]] @@ -152,6 +152,16 @@ spec = problems = [Duplicate_Output_Column_Names.Error ["bar_1"]] Problems.test_problem_handling action problems tester + Test.specify "tokenize will make column names unique" <| + cols = [["foo", [0, 1, 2]], ["bar", ["a12b34r5", "23", "2r4r55"]], ["bar_1", ["a", "b", "c"]]] + t = Table.new cols + expected_rows = [[0, "12", "34", "5", "a"], [1, "23", Nothing, Nothing, "b"], [2, "2", "4", "55", "c"]] + expected = Table.from_rows ["foo", "bar_0", "bar_1_1", "bar_2", "bar_1"] expected_rows + action = t.tokenize_to_columns "bar" "\d+" on_problems=_ + tester = t-> t.should_equal_verbose expected + problems = [Duplicate_Output_Column_Names.Error ["bar_1"]] + Problems.test_problem_handling action problems tester + Test.group "Table.split/tokenize column order" <| Test.specify "preserves column order" <| cols = [["foo", [0, 1, 2]], ["bar", ["abc", "cbdbef", "ghbijbu"]], ["baz", [1, 2, 3]]] From aee7c27b6f4e939ed7734d801d9534ca48f3c424 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 12:31:42 -0400 Subject: [PATCH 55/64] unused on_problems --- .../Standard/Database/0.0.0-dev/src/Data/Table.enso | 8 ++++---- .../lib/Standard/Table/0.0.0-dev/src/Data/Table.enso | 12 ++++++------ .../Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 12 +++++------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index 256d74c5343f..1f6ba6919332 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1419,8 +1419,8 @@ type Table - column: The name or index of the column to split the text of. - delimiter: The term or terms used to split the text. - on_problems: Specifies the behavior when a problem occurs. - split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table - split_to_rows self column delimiter="," on_problems=Report_Error = + split_to_rows : Text | Integer -> Text -> Table + split_to_rows self column delimiter="," = _ = [column delimiter on_problems] Error.throw (Unsupported_Database_Operation.Error "Table.split_to_rows is not implemented yet for the Database backends.") @@ -1459,8 +1459,8 @@ type Table - case_sensitivity: Specifies if the text values should be compared case sensitively. - on_problems: Specifies the behavior when a problem occurs. - tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table - tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = + tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Table + tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive = _ = [column pattern case_sensitivity on_problems] Error.throw (Unsupported_Database_Operation.Error "Table.tokenize_to_rows is not implemented yet for the Database backends.") diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index d90f9a7179c0..b70951f0a0c5 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -943,9 +943,9 @@ type Table - column: The name or index of the column to split the text of. - delimiter: The term or terms used to split the text. - on_problems: Specifies the behavior when a problem occurs. - split_to_rows : Text | Integer -> Text -> Problem_Behavior -> Table - split_to_rows self column delimiter="," on_problems=Report_Error = - Split_Tokenize.split_to_rows self column delimiter on_problems + split_to_rows : Text | Integer -> Text -> Table + split_to_rows self column delimiter="," = + Split_Tokenize.split_to_rows self column delimiter ## Tokenizes a column of text into a set of new columns using a regular expression. @@ -981,9 +981,9 @@ type Table - case_sensitivity: Specifies if the text values should be compared case sensitively. - on_problems: Specifies the behavior when a problem occurs. - tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table - tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = - Split_Tokenize.tokenize_to_rows self column pattern case_sensitivity on_problems + tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Table + tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive = + Split_Tokenize.tokenize_to_rows self column pattern case_sensitivity ## ALIAS Filter Rows diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 11b9a300dc18..17b0c2fd439b 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -23,9 +23,8 @@ split_to_columns table input_column_id delimiter="," column_count=Nothing on_pro ## PRIVATE Splits a column of text into a set of new rows. See `Table.split_to_rows`. -split_to_rows : Table -> Text | Integer -> Text -> Problem_Behavior -> Table -split_to_rows table input_column_id delimiter="," on_problems=Report_Error = - _ = [on_problems] +split_to_rows : Table -> Text | Integer -> Text -> Table +split_to_rows table input_column_id delimiter="," = expect_text_column table input_column_id <| fan_out_to_rows table input_column_id (ignore_nothing (_.split delimiter)) @@ -42,9 +41,8 @@ tokenize_to_columns table input_column_id pattern case_sensitivity column_count Tokenizes a column of text into a set of new rows using a regular expression. See `Table.tokenize_to_rows`. -tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Problem_Behavior -> Table -tokenize_to_rows table input_column_id pattern="." case_sensitivity=Case_Sensitivity.Sensitive on_problems=Report_Error = - _ = [on_problems] +tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Table +tokenize_to_rows table input_column_id pattern="." case_sensitivity=Case_Sensitivity.Sensitive = expect_text_column table input_column_id <| fan_out_to_rows table input_column_id (ignore_nothing (_.tokenize pattern case_sensitivity)) @@ -194,7 +192,7 @@ map_columns_to_multiple input_column function column_count problem_builder = Column.from_storage name storage ## PRIVATE - Name a set of column vectors to be unique when added to a table. + Rename a vector of columns to be unique when added to a table. rename_new_columns : Table -> Vector Column -> Problem_Builder -> Vector Column rename_new_columns table columns problem_builder = unique = Unique_Name_Strategy.new From 94057a1577c8bbb7e7980e5a197c048dcc6aa2ff Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 12:32:33 -0400 Subject: [PATCH 56/64] comment --- .../Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 1 + 1 file changed, 1 insertion(+) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 17b0c2fd439b..31d802ea0eaf 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -205,6 +205,7 @@ rename_new_columns table columns problem_builder = ## PRIVATE Replace a single column in a table with new columns. + Does not ensure names are unique; that must be done before calling this. replace_column_with_columns : Table -> Column -> Vector Column -> Table replace_column_with_columns table old_column new_columns = Table.new ((table.columns.map (c-> if c.name == old_column.name then new_columns else [c])).flatten) From 4e75f7c32bb6a0bff482dde1ece6986771f77f25 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 12:40:27 -0400 Subject: [PATCH 57/64] each --- .../0.0.0-dev/src/Internal/Split_Tokenize.enso | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 31d802ea0eaf..702a32c22beb 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -87,11 +87,11 @@ fan_out_to_rows table input_column_id function = # Accumulates repeated position indices for the order mask. order_mask_positions = Vector.new_builder 0 - 0.up_to num_input_rows . map i-> + 0.up_to num_input_rows . each i-> input_value = input_storage.getItemBoxed i output_values = function input_value # Append each value. - output_values.map v-> output_column_builder.append v + output_values.each v-> output_column_builder.append v # Append n copies of the input row position, n = # of output values. repeat output_values.length <| order_mask_positions.append i @@ -141,7 +141,7 @@ map_columns_to_multiple input_column function column_count problem_builder = Nothing -> builders = Vector.new_builder - 0.up_to num_rows . map i-> + 0.up_to num_rows . each i-> input_value = input_storage.getItemBoxed i output_values = function input_value @@ -155,11 +155,11 @@ map_columns_to_multiple input_column function column_count problem_builder = num_nulls_needed = i builder.appendNulls num_nulls_needed - builders.append <| builder + builders.append builder ## Add `output_values` to builders; if there are more builders than `output_values`, pad with null. - 0.up_to builders.length . map i-> + 0.up_to builders.length . each i-> builders.at i . appendNoGrow (output_values.get i Nothing) builders.to_vector @@ -173,7 +173,7 @@ map_columns_to_multiple input_column function column_count problem_builder = ## Add `output_values` to builders; if there are more builders than `output_values`, pad with null. - 0.up_to builders.length . map i-> + 0.up_to builders.length . each i-> builders.at i . appendNoGrow (output_values.get i Nothing) output_values.length @@ -187,7 +187,7 @@ map_columns_to_multiple input_column function column_count problem_builder = builders # Build Columns. - builders.map .seal . map_with_index i->storage-> + builders.map .seal . map_with_index i-> storage-> name = input_column.name + "_" + i.to_text Column.from_storage name storage From e1cbf164afbdbd583496f65517fafc123556e6e3 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 16:02:08 -0400 Subject: [PATCH 58/64] should_equal_verbose text --- test/Table_Tests/src/Util.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Table_Tests/src/Util.enso b/test/Table_Tests/src/Util.enso index 7d6572de90b6..b99a4ab04a7f 100644 --- a/test/Table_Tests/src/Util.enso +++ b/test/Table_Tests/src/Util.enso @@ -20,7 +20,7 @@ Table.should_equal_verbose self expected = same_headers && same_columns equal = tables_equal self expected if equal.not then - msg = 'Tables differ.\nself:\n' + self.display + '\nExpected:\n' + expected.display + msg = 'Tables differ.\nactual:\n' + self.display + '\nExpected:\n' + expected.display Test.fail msg Column.should_equal self expected = From ba3dba99571c679be3be7e722cbf63080dd600df Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 16:09:02 -0400 Subject: [PATCH 59/64] wip --- .../Table/0.0.0-dev/src/Internal/Split_Tokenize.enso | 6 ++++-- test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 702a32c22beb..32839a043050 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -82,10 +82,12 @@ fan_out_to_rows table input_column_id function = input_storage = input_column.java_column.getStorage num_input_rows = input_storage.size + # Guess that most of the time, we'll get at least one value for each input. + initial_size = input_column.length # Accumulates the output of the output column values. - output_column_builder = make_string_builder 0 + output_column_builder = make_string_builder initial_size # Accumulates repeated position indices for the order mask. - order_mask_positions = Vector.new_builder 0 + order_mask_positions = Vector.new_builder initial_size 0.up_to num_input_rows . each i-> input_value = input_storage.getItemBoxed i diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index 5fc057900ad4..9b8f8b4ffbad 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -105,7 +105,7 @@ spec = problems = [Column_Count_Exceeded.Error 2 3] Problems.test_problem_handling action problems tester - Test.specify "should generate extra empty columns if column_count is set (reverse row order)" <| + Test.specify "should generate extra empty columns if column_count is set (with rows in a different order)" <| cols = [["foo", [0, 1, 2]], ["bar", ["ghbijbu", "cbdbef", "abc"]]] t = Table.new cols expected_rows = [[0, "gh", "ij", "u", Nothing], [1, "c", "d", "ef", Nothing], [2, "a", "c", Nothing, Nothing]] From 4701650b9fa92a6e00f4eeb1d52c549abdcd4a3e Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 16:11:02 -0400 Subject: [PATCH 60/64] tokenize spec with no matches --- test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index 9b8f8b4ffbad..d6fe82c5adc9 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -43,6 +43,14 @@ spec = t2 = t.tokenize_to_rows "bar" "\d+" t2.should_equal_verbose expected + Test.specify "can do tokenize_to_rows with some rows that have no matches" <| + cols = [["foo", [0, 1, 2, 3]], ["bar", ["a12b34r5", "23", "q", "2r4r55"]]] + t = Table.new cols + expected_rows = [[0, "12"], [0, "34"], [0, "5"], [1, "23"], [3, "2"], [3, "4"], [3, "55"]] + expected = Table.from_rows ["foo", "bar"] expected_rows + t2 = t.tokenize_to_rows "bar" "\d+" + t2.should_equal_verbose expected + Test.specify "can do tokenize_to_columns with groups" <| cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols From 2ab67774202477d9ab035d8b9fc2f0238be63598 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Apr 2023 16:18:55 -0400 Subject: [PATCH 61/64] tests for nothing --- .../src/Internal/Split_Tokenize.enso | 16 +++++----- .../src/In_Memory/Split_Tokenize_Spec.enso | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index 32839a043050..b16cb6f35325 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -18,7 +18,7 @@ polyglot java import org.enso.table.data.mask.OrderMask split_to_columns : Table -> Text | Integer -> Text -> Integer | Nothing -> Problem_Behavior -> Table split_to_columns table input_column_id delimiter="," column_count=Nothing on_problems=Report_Error = expect_text_column table input_column_id <| - fan_out_to_columns table input_column_id (ignore_nothing (_.split delimiter)) column_count on_problems + fan_out_to_columns table input_column_id (handle_nothing (_.split delimiter)) column_count on_problems ## PRIVATE Splits a column of text into a set of new rows. @@ -26,7 +26,7 @@ split_to_columns table input_column_id delimiter="," column_count=Nothing on_pro split_to_rows : Table -> Text | Integer -> Text -> Table split_to_rows table input_column_id delimiter="," = expect_text_column table input_column_id <| - fan_out_to_rows table input_column_id (ignore_nothing (_.split delimiter)) + fan_out_to_rows table input_column_id (handle_nothing (_.split delimiter)) ## PRIVATE Tokenizes a column of text into a set of new columns using a regular @@ -35,7 +35,7 @@ split_to_rows table input_column_id delimiter="," = tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Integer | Nothing -> Problem_Behavior -> Table tokenize_to_columns table input_column_id pattern case_sensitivity column_count on_problems = expect_text_column table input_column_id <| - fan_out_to_columns table input_column_id (ignore_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems + fan_out_to_columns table input_column_id (handle_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems ## PRIVATE Tokenizes a column of text into a set of new rows using a regular @@ -44,7 +44,7 @@ tokenize_to_columns table input_column_id pattern case_sensitivity column_count tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Table tokenize_to_rows table input_column_id pattern="." case_sensitivity=Case_Sensitivity.Sensitive = expect_text_column table input_column_id <| - fan_out_to_rows table input_column_id (ignore_nothing (_.tokenize pattern case_sensitivity)) + fan_out_to_rows table input_column_id (handle_nothing (_.tokenize pattern case_sensitivity)) ## PRIVATE Transform a table by transforming a column into a set of columns. Takes a @@ -220,10 +220,10 @@ maximum vec = if vec.is_empty then Nothing else vec.reduce (a-> b-> a.max b) ## PRIVATE - Wrap a function so that it returns Nothing if passed Nothing -ignore_nothing : (Any -> Any) -> (Any -> Any) -ignore_nothing function = x-> case x of - _ : Nothing -> Nothing + Wrap a function so that it returns [] if passed Nothing +handle_nothing : (Any -> Any) -> (Any -> Any) +handle_nothing function = x-> case x of + _ : Nothing -> [] _ -> function x ## PRIVATE diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index d6fe82c5adc9..cb25fd2f37f3 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -26,6 +26,22 @@ spec = t2 = t.split_to_rows "bar" "b" t2.should_equal_verbose expected + Test.specify "can do split_to_columns with some Nothings" <| + cols = [["foo", [0, 1, 2, 3]], ["bar", ["abc", "cbdbef", Nothing, "ghbijbu"]]] + t = Table.new cols + expected_rows = [[0, "a", "c", Nothing], [1, "c", "d", "ef"], [2, Nothing, Nothing, Nothing], [3, "gh", "ij", "u"]] + expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2"] expected_rows + t2 = t.split_to_columns "bar" "b" + t2.should_equal_verbose expected + + Test.specify "can do split_to_rows with some Nothings" <| + cols = [["foo", [0, 1, 2, 3]], ["bar", ["abc", "cbdbef", Nothing, "ghbijbu"]]] + t = Table.new cols + expected_rows = [[0, "a"], [0, "c"], [1, "c"], [1, "d"], [1, "ef"], [3, "gh"], [3, "ij"], [3, "u"]] + expected = Table.from_rows ["foo", "bar"] expected_rows + t2 = t.split_to_rows "bar" "b" + t2.should_equal_verbose expected + Test.group "Table.tokenize" <| Test.specify "can do tokenize_to_columns" <| cols = [["foo", [0, 1, 2]], ["bar", ["a12b34r5", "23", "2r4r55"]]] @@ -43,6 +59,22 @@ spec = t2 = t.tokenize_to_rows "bar" "\d+" t2.should_equal_verbose expected + Test.specify "can do tokenize_to_columns with some nothings" <| + cols = [["foo", [0, 1, 2, 3]], ["bar", ["a12b34r5", Nothing, "23", "2r4r55"]]] + t = Table.new cols + expected_rows = [[0, "12", "34", "5"], [1, Nothing, Nothing, Nothing], [2, "23", Nothing, Nothing], [3, "2", "4", "55"]] + expected = Table.from_rows ["foo", "bar_0", "bar_1", "bar_2"] expected_rows + t2 = t.tokenize_to_columns "bar" "\d+" + t2.should_equal_verbose expected + + Test.specify "can do tokenize_to_rows with some Nothings" <| + cols = [["foo", [0, 1, 2, 3]], ["bar", ["a12b34r5", Nothing, "23", "2r4r55"]]] + t = Table.new cols + expected_rows = [[0, "12"], [0, "34"], [0, "5"], [2, "23"], [3, "2"], [3, "4"], [3, "55"]] + expected = Table.from_rows ["foo", "bar"] expected_rows + t2 = t.tokenize_to_rows "bar" "\d+" + t2.should_equal_verbose expected + Test.specify "can do tokenize_to_rows with some rows that have no matches" <| cols = [["foo", [0, 1, 2, 3]], ["bar", ["a12b34r5", "23", "q", "2r4r55"]]] t = Table.new cols From de16f7c32d1cac6474da83febb068ee5e2045606 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 14 Apr 2023 09:22:04 -0400 Subject: [PATCH 62/64] expect_text --- .../src/Internal/Split_Tokenize.enso | 34 +++++++------------ .../src/In_Memory/Split_Tokenize_Spec.enso | 6 ++-- test/Table_Tests/src/Util.enso | 2 +- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso index b16cb6f35325..364b4c70ab2f 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Split_Tokenize.enso @@ -2,6 +2,7 @@ from Standard.Base import all import project.Data.Column.Column import project.Data.Table.Table +import project.Data.Type.Value_Type.Value_Type import project.Internal.Java_Exports import project.Internal.Problem_Builder.Problem_Builder import project.Internal.Unique_Name_Strategy.Unique_Name_Strategy @@ -17,7 +18,8 @@ polyglot java import org.enso.table.data.mask.OrderMask See `Table.split_to_columns`. split_to_columns : Table -> Text | Integer -> Text -> Integer | Nothing -> Problem_Behavior -> Table split_to_columns table input_column_id delimiter="," column_count=Nothing on_problems=Report_Error = - expect_text_column table input_column_id <| + column = table.at input_column_id + Value_Type.expect_text (column.value_type) related_column=column <| fan_out_to_columns table input_column_id (handle_nothing (_.split delimiter)) column_count on_problems ## PRIVATE @@ -25,7 +27,8 @@ split_to_columns table input_column_id delimiter="," column_count=Nothing on_pro See `Table.split_to_rows`. split_to_rows : Table -> Text | Integer -> Text -> Table split_to_rows table input_column_id delimiter="," = - expect_text_column table input_column_id <| + column = table.at input_column_id + Value_Type.expect_text (column.value_type) related_column=column <| fan_out_to_rows table input_column_id (handle_nothing (_.split delimiter)) ## PRIVATE @@ -34,7 +37,8 @@ split_to_rows table input_column_id delimiter="," = See `Table.tokenize_to_columns`. tokenize_to_columns : Table -> Text | Integer -> Text -> Case_Sensitivity -> Integer | Nothing -> Problem_Behavior -> Table tokenize_to_columns table input_column_id pattern case_sensitivity column_count on_problems = - expect_text_column table input_column_id <| + column = table.at input_column_id + Value_Type.expect_text (column.value_type) related_column=column <| fan_out_to_columns table input_column_id (handle_nothing (_.tokenize pattern case_sensitivity)) column_count on_problems ## PRIVATE @@ -43,7 +47,8 @@ tokenize_to_columns table input_column_id pattern case_sensitivity column_count See `Table.tokenize_to_rows`. tokenize_to_rows : Table -> Text | Integer -> Text -> Case_Sensitivity -> Table tokenize_to_rows table input_column_id pattern="." case_sensitivity=Case_Sensitivity.Sensitive = - expect_text_column table input_column_id <| + column = table.at input_column_id + Value_Type.expect_text (column.value_type) related_column=column <| fan_out_to_rows table input_column_id (handle_nothing (_.tokenize pattern case_sensitivity)) ## PRIVATE @@ -95,7 +100,7 @@ fan_out_to_rows table input_column_id function = # Append each value. output_values.each v-> output_column_builder.append v # Append n copies of the input row position, n = # of output values. - repeat output_values.length <| order_mask_positions.append i + repeat_each output_values.length <| order_mask_positions.append i # Build the output column output_storage = output_column_builder.seal @@ -150,7 +155,7 @@ map_columns_to_multiple input_column function column_count problem_builder = # Add more builders if necessary to accommodate `output_values`. if output_values.length > builders.length then num_builders_needed = output_values.length - builders.length - repeat num_builders_needed <| + repeat_each num_builders_needed <| builder = make_string_builder num_rows # Pad the new builder with nulls @@ -226,21 +231,6 @@ handle_nothing function = x-> case x of _ : Nothing -> [] _ -> function x -## PRIVATE - Asserts that a column exists in the table and is a text column. -expect_text_column : Table -> Text | Integer -> Problem_Behavior -> Any -> Any -expect_text_column table column_id ~action = - column = table.get column_id - case column of - Nothing -> - Error.throw <| Missing_Input_Columns.Error [column_id] - _ : Column -> - case Value_Type.is_text column.value_type of - False -> - Error.throw <| Invalid_Value_Type.Error Value_Type.Char column.value_type column_id - True -> - action - ## PRIVATE Repeat a computation n times. -repeat n ~action = 0.up_to n . map _-> action +repeat_each n ~action = 0.up_to n . each _-> action diff --git a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso index cb25fd2f37f3..ca63fb4010d6 100644 --- a/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Split_Tokenize_Spec.enso @@ -4,7 +4,7 @@ import Standard.Base.Data.Text.Case_Sensitivity.Case_Sensitivity import Standard.Test.Extensions from Standard.Table import Table -from Standard.Table.Errors import Invalid_Value_Type, Column_Count_Exceeded, Duplicate_Output_Column_Names, Missing_Input_Columns +from Standard.Table.Errors import Invalid_Value_Type, Column_Count_Exceeded, Duplicate_Output_Column_Names, No_Such_Column from Standard.Test import Test, Test_Suite, Problems from project.Util import all @@ -174,12 +174,12 @@ spec = Test.specify "*_to_columns handles missing input column" <| cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols - t.tokenize_to_columns "invalid_name" "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns + t.tokenize_to_columns "invalid_name" "([a-z]).(\d+)" . should_fail_with No_Such_Column Test.specify "*_to_rows handles missing input column" <| cols = [["foo", [0, 1]], ["bar", ["r a-1, b-12,qd-50", "ab-10:bc-20c"]]] t = Table.new cols - t.tokenize_to_rows "invalid_name" "([a-z]).(\d+)" . should_fail_with Missing_Input_Columns + t.tokenize_to_rows "invalid_name" "([a-z]).(\d+)" . should_fail_with No_Such_Column Test.group "Table.split/tokenize name conflicts" <| Test.specify "split will make column names unique" <| diff --git a/test/Table_Tests/src/Util.enso b/test/Table_Tests/src/Util.enso index b99a4ab04a7f..048d59261506 100644 --- a/test/Table_Tests/src/Util.enso +++ b/test/Table_Tests/src/Util.enso @@ -20,7 +20,7 @@ Table.should_equal_verbose self expected = same_headers && same_columns equal = tables_equal self expected if equal.not then - msg = 'Tables differ.\nactual:\n' + self.display + '\nExpected:\n' + expected.display + msg = 'Tables differ.\nActual:\n' + self.display + '\nExpected:\n' + expected.display Test.fail msg Column.should_equal self expected = From 451714e84ce5f5f4d44fcb7b983988bd19c22ba4 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 14 Apr 2023 09:43:23 -0400 Subject: [PATCH 63/64] docs consistent --- .../Standard/Database/0.0.0-dev/src/Data/Table.enso | 6 ++++-- .../lib/Standard/Table/0.0.0-dev/src/Data/Table.enso | 12 ++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index 1f6ba6919332..5a07fc69f775 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1439,9 +1439,11 @@ type Table sensitively. - column_count: The number of columns to split to. If `Nothing` then columns will be added to fit all data. - If the data exceeds the `column_count`, a `Column_Count_Exceeded` error - will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. + + ! Error Conditions + If the data exceeds the `column_count`, a `Column_Count_Exceeded` will + be reported according to the `on_problems` behavior. tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Integer | Nothing -> Problem_Behavior -> Table tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Nothing on_problems=Report_Error = _ = [column pattern case_sensitivity column_count on_problems] diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso index b70951f0a0c5..c36b1b818bf8 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Data/Table.enso @@ -929,9 +929,11 @@ type Table - delimiter: The term or terms used to split the text. - column_count: The number of columns to split to. If `Nothing` then columns will be added to fit all data. - If the data exceeds the `column_count`, a `Column_Count_Exceeded` error - will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. + + ! Error Conditions + If the data exceeds the `column_count`, a `Column_Count_Exceeded` will + be reported according to the `on_problems` behavior. split_to_columns : Text | Integer -> Text -> Integer | Nothing -> Problem_Behavior -> Table split_to_columns self column delimiter="," column_count=Nothing on_problems=Report_Error = Split_Tokenize.split_to_columns self column delimiter column_count on_problems @@ -962,9 +964,11 @@ type Table sensitively. - column_count: The number of columns to split to. If `Nothing` then columns will be added to fit all data. - If the data exceeds the `column_count`, a `Column_Count_Exceeded` error - will follow the `on_problems` behavior. - on_problems: Specifies the behavior when a problem occurs. + + ! Error Conditions + If the data exceeds the `column_count`, a `Column_Count_Exceeded` will + be reported according to the `on_problems` behavior. tokenize_to_columns : Text | Integer -> Text -> Case_Sensitivity -> Integer | Nothing -> Problem_Behavior -> Table tokenize_to_columns self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive column_count=Nothing on_problems=Report_Error = Split_Tokenize.tokenize_to_columns self column pattern case_sensitivity column_count on_problems From 150149b38ffced67698d5e79e283c60cb7630e99 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 14 Apr 2023 11:09:37 -0400 Subject: [PATCH 64/64] removed param --- .../lib/Standard/Database/0.0.0-dev/src/Data/Table.enso | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index 5a07fc69f775..f603843fc79a 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1421,7 +1421,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. split_to_rows : Text | Integer -> Text -> Table split_to_rows self column delimiter="," = - _ = [column delimiter on_problems] + _ = [column delimiter] Error.throw (Unsupported_Database_Operation.Error "Table.split_to_rows is not implemented yet for the Database backends.") ## Tokenizes a column of text into a set of new columns using a regular @@ -1463,7 +1463,7 @@ type Table - on_problems: Specifies the behavior when a problem occurs. tokenize_to_rows : Text | Integer -> Text -> Case_Sensitivity -> Table tokenize_to_rows self column pattern="." case_sensitivity=Case_Sensitivity.Sensitive = - _ = [column pattern case_sensitivity on_problems] + _ = [column pattern case_sensitivity] Error.throw (Unsupported_Database_Operation.Error "Table.tokenize_to_rows is not implemented yet for the Database backends.") ## PRIVATE