From 2e1d79216059ac24abc8f27b3f232657b8eb659e Mon Sep 17 00:00:00 2001 From: jianfengmao Date: Sun, 11 Sep 2022 11:47:44 -0600 Subject: [PATCH 1/2] Merge py integration tests into the main suite --- Integrations/build.gradle | 6 --- .../integration-tests/null_integers_test.py | 23 ---------- .../numbers_from_list_and_func_test.py | 42 ------------------- py/server/integration-tests/perfmon_test.py | 24 ----------- .../integration-tests/pylong_conv_test.py | 24 ----------- .../pyobj_field_access_test.py | 34 --------------- py/server/integration-tests/run-tests.sh | 14 ------- py/server/tests/test_column.py | 35 +++++++++++++++- py/server/tests/test_constants.py | 14 +++++++ .../test_numba_vectorized_column.py} | 3 -- .../test_numba_vectorized_filter.py} | 3 -- py/server/tests/test_perfmon.py | 12 +++++- py/server/tests/test_table.py | 33 +++++++++++++-- 13 files changed, 88 insertions(+), 179 deletions(-) delete mode 100644 py/server/integration-tests/null_integers_test.py delete mode 100644 py/server/integration-tests/numbers_from_list_and_func_test.py delete mode 100644 py/server/integration-tests/perfmon_test.py delete mode 100644 py/server/integration-tests/pylong_conv_test.py delete mode 100644 py/server/integration-tests/pyobj_field_access_test.py delete mode 100755 py/server/integration-tests/run-tests.sh rename py/server/{integration-tests/numba_vectorized_column_test.py => tests/test_numba_vectorized_column.py} (95%) rename py/server/{integration-tests/numba_vectorized_filter_test.py => tests/test_numba_vectorized_filter.py} (95%) diff --git a/Integrations/build.gradle b/Integrations/build.gradle index ac4ec50aa37..8055b8ad11b 100644 --- a/Integrations/build.gradle +++ b/Integrations/build.gradle @@ -102,14 +102,8 @@ Closure composeConfig = { task -> } def pyTest = runInDocker('test-py-deephaven', '../py/server', ['python3', '-m', 'xmlrunner', 'discover', '-s', 'tests', '-t', '.', '-v', '-o', '/out/report'], composeConfig) -def pyTestIntegration = runInDocker('test-py-deephaven-integration', '../py/server', ['/bin/bash', '-c', 'PYTHONPATH=/python integration-tests/run-tests.sh'], composeConfig) pyTest.configure({ onlyIf { TestTools.shouldRunTests(project) } }) tasks.getByName('check').dependsOn(pyTest) - -pyTestIntegration.configure({ - onlyIf { TestTools.shouldRunTests(project) } -}) -tasks.getByName('check').dependsOn(pyTestIntegration) diff --git a/py/server/integration-tests/null_integers_test.py b/py/server/integration-tests/null_integers_test.py deleted file mode 100644 index 034e13ea8c8..00000000000 --- a/py/server/integration-tests/null_integers_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending -# - -from test_helper import start_jvm -start_jvm() - -from deephaven import empty_table -from deephaven.constants import NULL_LONG, NULL_SHORT, NULL_INT, NULL_BYTE - -null_byte = NULL_BYTE -null_short = NULL_SHORT -null_int = NULL_INT -null_long = NULL_LONG - - -def return_null_long(): - return NULL_LONG - - -t = empty_table(9).update( - ["X = null_byte", "Y = null_short", "YY = null_int", "Z = null_long", "ZZ = (long)return_null_long()"]) -assert t.to_string().count("null") == 45 diff --git a/py/server/integration-tests/numbers_from_list_and_func_test.py b/py/server/integration-tests/numbers_from_list_and_func_test.py deleted file mode 100644 index 72408f8f127..00000000000 --- a/py/server/integration-tests/numbers_from_list_and_func_test.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending -# - -from test_helper import start_jvm -start_jvm() - -from deephaven import dtypes -from deephaven import empty_table -from deephaven.constants import MAX_BYTE, MAX_SHORT, MAX_INT, MAX_LONG - -x = [MAX_BYTE, MAX_SHORT, MAX_INT, MAX_LONG, 0.98888, 999999.888888] -n = len(x) - - -def get_x(i): - return x[i] - - -t_list = empty_table(n).update(["X = x[i]"]) -t_func = empty_table(n).update(["X = get_x(i)"]) -# We want to test that casting on both PyObject and JObject works as expected. -assert t_list.columns[0].data_type == dtypes.PyObject -assert t_func.columns[0].data_type == dtypes.JObject - -t_list_integers = t_list.update( - ["A = (byte)X", "B = (short)X", "C = (int)X", "D = (long)X", "E = (float)X", "F = (double)X"]) -assert t_list_integers.columns[1].data_type == dtypes.byte -assert t_list_integers.columns[2].data_type == dtypes.short -assert t_list_integers.columns[3].data_type == dtypes.int32 -assert t_list_integers.columns[4].data_type == dtypes.long -assert t_list_integers.columns[5].data_type == dtypes.float32 -assert t_list_integers.columns[6].data_type == dtypes.double - -t_func_integers = t_list.update( - ["A = (byte)X", "B = (short)X", "C = (int)X", "D = (long)X", "E = (float)X", "F = (double)X"]) -assert t_func_integers.columns[1].data_type == dtypes.byte -assert t_func_integers.columns[2].data_type == dtypes.short -assert t_func_integers.columns[3].data_type == dtypes.int32 -assert t_func_integers.columns[4].data_type == dtypes.long -assert t_list_integers.columns[5].data_type == dtypes.float32 -assert t_list_integers.columns[6].data_type == dtypes.float64 diff --git a/py/server/integration-tests/perfmon_test.py b/py/server/integration-tests/perfmon_test.py deleted file mode 100644 index df9266caeea..00000000000 --- a/py/server/integration-tests/perfmon_test.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending -# - -import unittest - -from test_helper import start_jvm -start_jvm() - -from deephaven.perfmon import query_update_performance, query_performance, query_operation_performance, server_state - -class TestClass(unittest.TestCase): - def test_performance_queries(self): - q = query_performance(1) - self.assertTrue(q.to_string()) - q = query_operation_performance(1) - self.assertTrue(q.to_string()) - q = server_state() - self.assertTrue(q.to_string()) - q = query_update_performance(1) - self.assertTrue(q.to_string()) - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/py/server/integration-tests/pylong_conv_test.py b/py/server/integration-tests/pylong_conv_test.py deleted file mode 100644 index 8b585c6912f..00000000000 --- a/py/server/integration-tests/pylong_conv_test.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending -# - -import unittest - -from test_helper import start_jvm -start_jvm() - -from deephaven import empty_table - -long_value = 2 ** 32 + 5 - - -class TestClass(unittest.TestCase): - - def test_long_number_conversion(self): - t = empty_table(1) - result = t.update("X = long_value").to_string(1) - self.assertEqual(long_value, int(result.split()[2])) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/py/server/integration-tests/pyobj_field_access_test.py b/py/server/integration-tests/pyobj_field_access_test.py deleted file mode 100644 index 96a9a02c7f6..00000000000 --- a/py/server/integration-tests/pyobj_field_access_test.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending -# - -import unittest - -from test_helper import start_jvm -start_jvm() - -from deephaven import empty_table -from deephaven.html import to_html - - -class EmptyCls: - ... - - -pyobj = EmptyCls() -pyobj.name = "GOOG" -pyobj.price = 1000 - - -class TestClass(unittest.TestCase): - - def test_pyobj_field_access(self): - t = empty_table(10) - t2 = t.update(formulas=["SYM = `AAPL-` + (String)pyobj.name", "PRICE = i * 1000"]).where("PRICE > (int)pyobj.price + 100") - html_output = to_html(t2) - self.assertIn("AAPL-GOOG", html_output) - self.assertIn("2000", html_output) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/py/server/integration-tests/run-tests.sh b/py/server/integration-tests/run-tests.sh deleted file mode 100755 index 823a32cc996..00000000000 --- a/py/server/integration-tests/run-tests.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -eux - -# -# Run each test py file in its own interpreter so they don't share globals -# - -DIR=$(dirname "$0") - -for test_file_name in $DIR/*test.py; -do - echo $test_file_name >> /out/report/scripting-test_results 2>&1 - python3 $test_file_name >> /out/report/scripting-test_results 2>&1 -done diff --git a/py/server/tests/test_column.py b/py/server/tests/test_column.py index ce930ebb5d6..97e1989b01c 100644 --- a/py/server/tests/test_column.py +++ b/py/server/tests/test_column.py @@ -7,9 +7,11 @@ from dataclasses import dataclass from deephaven import DHError, dtypes, new_table -from deephaven.jcompat import j_array_list +from deephaven import empty_table from deephaven.column import byte_col, char_col, short_col, bool_col, int_col, long_col, float_col, double_col, \ string_col, datetime_col, jobj_col, ColumnType +from deephaven.constants import MAX_BYTE, MAX_SHORT, MAX_INT, MAX_LONG +from deephaven.jcompat import j_array_list from tests.testbase import BaseTestCase @@ -70,6 +72,37 @@ def test_array_column(self): self.assertIsNone(test_table.columns[0].component_type) self.assertEqual(test_table.columns[1].component_type, dtypes.double) + def test_numeric_columns(self): + x = [MAX_BYTE, MAX_SHORT, MAX_INT, MAX_LONG, 0.98888, 999999.888888] + n = len(x) + + def get_x(i): + return x[i] + + t_list = empty_table(n).update(["X = x[i]"]) + t_func = empty_table(n).update(["X = get_x(i)"]) + # We want to test that casting on both PyObject and JObject works as expected. + self.assertEqual(t_list.columns[0].data_type, dtypes.PyObject) + self.assertEqual(t_func.columns[0].data_type, dtypes.JObject) + + t_list_integers = t_list.update( + ["A = (byte)X", "B = (short)X", "C = (int)X", "D = (long)X", "E = (float)X", "F = (double)X"]) + self.assertEqual(t_list_integers.columns[1].data_type, dtypes.byte) + self.assertEqual(t_list_integers.columns[2].data_type, dtypes.short) + self.assertEqual(t_list_integers.columns[3].data_type, dtypes.int32) + self.assertEqual(t_list_integers.columns[4].data_type, dtypes.long) + self.assertEqual(t_list_integers.columns[5].data_type, dtypes.float32) + self.assertEqual(t_list_integers.columns[6].data_type, dtypes.double) + + t_func_integers = t_func.update( + ["A = (byte)X", "B = (short)X", "C = (int)X", "D = (long)X", "E = (float)X", "F = (double)X"]) + self.assertEqual(t_func_integers.columns[1].data_type, dtypes.byte) + self.assertEqual(t_func_integers.columns[2].data_type, dtypes.short) + self.assertEqual(t_func_integers.columns[3].data_type, dtypes.int32) + self.assertEqual(t_func_integers.columns[4].data_type, dtypes.long) + self.assertEqual(t_list_integers.columns[5].data_type, dtypes.float32) + self.assertEqual(t_list_integers.columns[6].data_type, dtypes.float64) + @dataclass class CustomClass: diff --git a/py/server/tests/test_constants.py b/py/server/tests/test_constants.py index a713ad12f24..13c8a77d4a0 100644 --- a/py/server/tests/test_constants.py +++ b/py/server/tests/test_constants.py @@ -2,6 +2,7 @@ # Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending # +from deephaven import empty_table from deephaven.constants import * from tests.testbase import BaseTestCase @@ -43,3 +44,16 @@ def test_constants_values(self): self.assertEqual(NULL_SHORT, -32768) self.assertEqual(POS_INFINITY_DOUBLE, float('inf')) self.assertEqual(POS_INFINITY_FLOAT, float('inf')) + + def test_return_null_long(self): + null_byte = NULL_BYTE + null_short = NULL_SHORT + null_int = NULL_INT + null_long = NULL_LONG + + def return_null_long(): + return NULL_LONG + + t = empty_table(9).update( + ["X = null_byte", "Y = null_short", "YY = null_int", "Z = null_long", "ZZ = (long)return_null_long()"]) + self.assertEqual(t.to_string().count("null"), 45) diff --git a/py/server/integration-tests/numba_vectorized_column_test.py b/py/server/tests/test_numba_vectorized_column.py similarity index 95% rename from py/server/integration-tests/numba_vectorized_column_test.py rename to py/server/tests/test_numba_vectorized_column.py index a7fcc380caf..f1252b8b3ab 100644 --- a/py/server/integration-tests/numba_vectorized_column_test.py +++ b/py/server/tests/test_numba_vectorized_column.py @@ -6,9 +6,6 @@ from numba import vectorize, int64 -from test_helper import start_jvm -start_jvm() - from deephaven import empty_table from deephaven.html import to_html diff --git a/py/server/integration-tests/numba_vectorized_filter_test.py b/py/server/tests/test_numba_vectorized_filter.py similarity index 95% rename from py/server/integration-tests/numba_vectorized_filter_test.py rename to py/server/tests/test_numba_vectorized_filter.py index c15b1a22a27..ea296abc207 100644 --- a/py/server/integration-tests/numba_vectorized_filter_test.py +++ b/py/server/tests/test_numba_vectorized_filter.py @@ -6,9 +6,6 @@ from numba import vectorize, int64, boolean, short -from test_helper import start_jvm -start_jvm() - from deephaven import empty_table from deephaven.html import to_html diff --git a/py/server/tests/test_perfmon.py b/py/server/tests/test_perfmon.py index 673e2393d7c..45f2392b33a 100644 --- a/py/server/tests/test_perfmon.py +++ b/py/server/tests/test_perfmon.py @@ -5,10 +5,10 @@ import unittest from deephaven import empty_table - from deephaven.perfmon import process_info_log, process_metrics_log, server_state_log, \ query_operation_performance_log, query_performance_log, update_performance_log, metrics_get_counters, \ metrics_reset_counters +from deephaven.perfmon import query_update_performance, query_performance, query_operation_performance, server_state from tests.testbase import BaseTestCase @@ -54,6 +54,16 @@ def test_query_logs(self): log_table = update_performance_log() self.assertTrue(log_table.to_string()) + def test_performance_queries(self): + q = query_performance(1) + self.assertTrue(q.to_string()) + q = query_operation_performance(1) + self.assertTrue(q.to_string()) + q = server_state() + self.assertTrue(q.to_string()) + q = query_update_performance(1) + self.assertTrue(q.to_string()) + if __name__ == '__main__': unittest.main() diff --git a/py/server/tests/test_table.py b/py/server/tests/test_table.py index dd650fb7f36..0d1526fba22 100644 --- a/py/server/tests/test_table.py +++ b/py/server/tests/test_table.py @@ -10,6 +10,7 @@ from deephaven import DHError, read_csv, empty_table, SortDirection, AsOfMatchRule, time_table, ugp from deephaven.agg import sum_, weighted_avg, avg, pct, group, count_, first, last, max_, median, min_, std, abs_sum, \ var, formula, partition +from deephaven.html import to_html from deephaven.pandas import to_pandas from deephaven.table import Table from tests.testbase import BaseTestCase @@ -625,10 +626,11 @@ def closure_fn() -> str: def test_nested_scopes(self): _JExecutionContext = jpy.get_type("io.deephaven.engine.context.ExecutionContext") context = _JExecutionContext.newBuilder() \ - .captureQueryCompiler() \ - .captureQueryLibrary() \ - .captureQueryScope() \ - .build() + .captureQueryCompiler() \ + .captureQueryLibrary() \ + .captureQueryScope() \ + .build() + def inner_func(p) -> str: openContext = context.open() t = empty_table(1).update("X = p * 10") @@ -724,6 +726,20 @@ def test_ticking_table_scope(self): self.wait_ticking_table_update(rt, row_count=rt.size + 1, timeout=5) self.verify_table_data(rt, list(range(1, 5))) + def test_long_number_conversion(self): + long_value = 2 ** 32 + 5 + t = empty_table(1) + result = t.update("X = long_value").to_string(1) + self.assertEqual(long_value, int(result.split()[2])) + + def test_python_field_access(self): + t = empty_table(10) + t2 = t.update(formulas=["SYM = `AAPL-` + (String)foo.name", "PRICE = i * 1000"]).where( + "PRICE > (int)foo.price + 100") + html_output = to_html(t2) + self.assertIn("AAPL-GOOG", html_output) + self.assertIn("2000", html_output) + def global_fn() -> str: return "global str" @@ -732,5 +748,14 @@ def global_fn() -> str: global_int = 1001 a_number = 10001 + +class EmptyCls: + ... + + +foo = EmptyCls() +foo.name = "GOOG" +foo.price = 1000 + if __name__ == "__main__": unittest.main() From 887fb01b90e226b3b71c5a89dc38306a0cdd3a80 Mon Sep 17 00:00:00 2001 From: jianfengmao Date: Sun, 11 Sep 2022 11:52:35 -0600 Subject: [PATCH 2/2] Small formatting change --- py/server/tests/test_table.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/py/server/tests/test_table.py b/py/server/tests/test_table.py index 0d1526fba22..59d1e54f7d2 100644 --- a/py/server/tests/test_table.py +++ b/py/server/tests/test_table.py @@ -625,11 +625,11 @@ def closure_fn() -> str: def test_nested_scopes(self): _JExecutionContext = jpy.get_type("io.deephaven.engine.context.ExecutionContext") - context = _JExecutionContext.newBuilder() \ - .captureQueryCompiler() \ - .captureQueryLibrary() \ - .captureQueryScope() \ - .build() + context = (_JExecutionContext.newBuilder() + .captureQueryCompiler() + .captureQueryLibrary() + .captureQueryScope() + .build()) def inner_func(p) -> str: openContext = context.open()