feat: add diagnostic tool (#2299)

4paradigm · Aug 10, 2022 · 01a7a67 · 01a7a67
1 parent d4c2099
commit 01a7a67
Show file tree

Hide file tree

Showing 55 changed files with 1,715 additions and 21 deletions.
diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml
@@ -273,21 +273,22 @@ jobs:
         with:
           name: linux-ut-result-python-${{ github.sha }}
           path: |
-            python/openmldb/test/pytest.xml
+            python/openmldb_sdk/openmldb/test/pytest.xml
 
       - name: upload python coverage to codecov
         uses: codecov/codecov-action@v2
         with:
           name: coverage
-          files: python/test/coverage.xml
+          files: python/openmldb_sdk/test/coverage.xml
           fail_ci_if_error: true
           verbose: true
 
       - name: upload to pypi
         if: >
           github.repository == '4paradigm/OpenMLDB' && startsWith(github.ref, 'refs/tags/v')
         run: |
-          cp python/dist/openmldb*.whl .
+          cp python/openmldb_sdk/dist/openmldb*.whl .
+          cp python/openmldb_tool/dist/openmldb*.whl .
           twine upload openmldb-*.whl
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
@@ -329,13 +330,14 @@ jobs:
         with:
           name: mac-ut-result-python-${{ github.sha }}
           path: |
-            python/openmldb/test/pytest.xml
+            python/openmldb_sdk/openmldb/test/pytest.xml
 
       - name: upload to pypi
         if: >
           github.repository == '4paradigm/OpenMLDB' && startsWith(github.ref, 'refs/tags/v')
         run: |
-          cp python/dist/openmldb*.whl .
+          cp python/openmldb_sdk/dist/openmldb*.whl .
+          cp python/openmldb_tool/dist/openmldb*.whl .
           twine upload openmldb-*.whl
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}

diff --git a/.gitignore b/.gitignore
@@ -99,9 +99,11 @@ java/hybridse-proto/src
 logs
 
 # python builds
-/python/dist/
-/python/*.egg-info/
-/python/openmldb/native/**
-!/python/openmldb/native/__init__.pyt
-/python/test/*.xml
+/python/openmldb_sdk/dist/
+/python/openmldb_sdk/*.egg-info/
+/python/openmldb_sdk/openmldb/native/**
+!/python/openmldb_sdk/openmldb/native/__init__.pyt
+/python/openmldb_sdk/test/*.xml
+/python/openmldb_tool/dist/
+/python/openmldb_tool/openmldb.egg-info/
 
diff --git a/python/openmldb/__init__.py → python/openmldb_sdk/openmldb/__init__.py b/python/openmldb/__init__.py → python/openmldb_sdk/openmldb/__init__.py
diff --git a/python/openmldb/dbapi/__init__.py → ...n/openmldb_sdk/openmldb/dbapi/__init__.py b/python/openmldb/dbapi/__init__.py → ...n/openmldb_sdk/openmldb/dbapi/__init__.py
diff --git a/python/openmldb/dbapi/dbapi.py → python/openmldb_sdk/openmldb/dbapi/dbapi.py b/python/openmldb/dbapi/dbapi.py → python/openmldb_sdk/openmldb/dbapi/dbapi.py
diff --git a/python/openmldb/native/__init__.py → .../openmldb_sdk/openmldb/native/__init__.py b/python/openmldb/native/__init__.py → .../openmldb_sdk/openmldb/native/__init__.py
diff --git a/python/openmldb/sdk/__init__.py → python/openmldb_sdk/openmldb/sdk/__init__.py b/python/openmldb/sdk/__init__.py → python/openmldb_sdk/openmldb/sdk/__init__.py
diff --git a/python/openmldb/sdk/sdk.py → python/openmldb_sdk/openmldb/sdk/sdk.py b/python/openmldb/sdk/sdk.py → python/openmldb_sdk/openmldb/sdk/sdk.py
diff --git a/python/openmldb/sql_magic/__init__.py → ...enmldb_sdk/openmldb/sql_magic/__init__.py b/python/openmldb/sql_magic/__init__.py → ...enmldb_sdk/openmldb/sql_magic/__init__.py
diff --git a/python/openmldb/sql_magic/sql_magic.py → ...nmldb_sdk/openmldb/sql_magic/sql_magic.py b/python/openmldb/sql_magic/sql_magic.py → ...nmldb_sdk/openmldb/sql_magic/sql_magic.py
diff --git a/.../openmldb/sqlalchemy_openmldb/__init__.py → .../openmldb/sqlalchemy_openmldb/__init__.py b/.../openmldb/sqlalchemy_openmldb/__init__.py → .../openmldb/sqlalchemy_openmldb/__init__.py
diff --git a/...b/sqlalchemy_openmldb/openmldb_dialect.py → ...b/sqlalchemy_openmldb/openmldb_dialect.py b/...b/sqlalchemy_openmldb/openmldb_dialect.py → ...b/sqlalchemy_openmldb/openmldb_dialect.py
diff --git a/...nmldb/sqlalchemy_openmldb/requirements.py → ...nmldb/sqlalchemy_openmldb/requirements.py b/...nmldb/sqlalchemy_openmldb/requirements.py → ...nmldb/sqlalchemy_openmldb/requirements.py
diff --git a/python/openmldb/sqlalchemy_test_config → ...nmldb_sdk/openmldb/sqlalchemy_test_config b/python/openmldb/sqlalchemy_test_config → ...nmldb_sdk/openmldb/sqlalchemy_test_config
diff --git a/python/openmldb/test/conftest.py → ...on/openmldb_sdk/openmldb/test/conftest.py b/python/openmldb/test/conftest.py → ...on/openmldb_sdk/openmldb/test/conftest.py
diff --git a/...b/test/sqlalchemy_standardtest_example.py → ...b/test/sqlalchemy_standardtest_example.py b/...b/test/sqlalchemy_standardtest_example.py → ...b/test/sqlalchemy_standardtest_example.py
diff --git a/python/setup.py → python/openmldb_sdk/setup.py b/python/setup.py → python/openmldb_sdk/setup.py
diff --git a/python/test/case_conf.py → python/openmldb_sdk/test/case_conf.py b/python/test/case_conf.py → python/openmldb_sdk/test/case_conf.py
diff --git a/python/test/dbapi_test.py → python/openmldb_sdk/test/dbapi_test.py b/python/test/dbapi_test.py → python/openmldb_sdk/test/dbapi_test.py
diff --git a/python/test/openmldb_client_test.py → ...openmldb_sdk/test/openmldb_client_test.py b/python/test/openmldb_client_test.py → ...openmldb_sdk/test/openmldb_client_test.py
diff --git a/python/test/sdk_smoke_test.py → python/openmldb_sdk/test/sdk_smoke_test.py b/python/test/sdk_smoke_test.py → python/openmldb_sdk/test/sdk_smoke_test.py
diff --git a/python/test/sql_magic_test.py → python/openmldb_sdk/test/sql_magic_test.py b/python/test/sql_magic_test.py → python/openmldb_sdk/test/sql_magic_test.py
diff --git a/python/test/sqlalchemy_api_test.py → .../openmldb_sdk/test/sqlalchemy_api_test.py b/python/test/sqlalchemy_api_test.py → .../openmldb_sdk/test/sqlalchemy_api_test.py
diff --git a/python/openmldb_tool/diagnostic_tool/README.md b/python/openmldb_tool/diagnostic_tool/README.md
@@ -0,0 +1,57 @@
+# Diag Tool
+main: diagnose.py
+
+ssh/scp by connections.py
+
+read distribution yaml by dist_conf.py
+
+## Collector
+collector.py collects config, log and version
+
+TODO: `<cluster-name>-conf` is better than custom dest name?
+
+### config
+```
+<dest>/
+  <ip:port>-nameserver/
+    nameserver.flags
+  <ip:port>-tablet/
+    tablet.flags
+  <ip:port>-tablet/
+    tablet.flags
+  <ip:port>-taskmanager/
+    taskmanager.properties
+```
+
+### log
+Find log path in remote config file.
+
+Get last 2 files.
+
+```
+<dest>/
+  <ip:port>-nameserver/
+    nameserver.info.log.1
+    nameserver.info.log.2
+    ...
+  <ip:port>-tablet/
+    ...
+  <ip:port>-taskmanager/
+    taskmanager.log.1
+    job_1_error.log
+    ...
+```
+
+### version
+
+exec openmldb
+
+run jar taskmanager and batch
+
+#### find batch jar
+find spark home from remote taskmanager config file.
+
+## analysis
+
+log_analysis.py read logs from local path `<dest>`. 
+NOTE: if diag local cluster/standalone, directory structure is different.
diff --git a/python/openmldb_tool/diagnostic_tool/__init__.py b/python/openmldb_tool/diagnostic_tool/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2021 4Paradigm
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.