Merge pull request #3 from hugegraph/ci_test

Update deploy_start.py test
hugegraph · Nov 20, 2023 · 7c16511 · 7c16511
2 parents 89e5b34 + f90e9f5
commit 7c16511
Show file tree

Hide file tree

Showing 47 changed files with 1,228 additions and 471 deletions.
diff --git a/.github/workflows/code_stst.yml b/.github/workflows/code_stst.yml
@@ -0,0 +1,125 @@
+name: hugegraph code statistics
+on:
+  pull_request:
+    branches:
+      - master
+  # 每月的1号在8点执行任务
+  schedule:
+    - cron: '0 0 1 * *'
+  # 手动触发
+  workflow_dispatch:
+
+jobs:
+  code_statistics:
+    runs-on: ubuntu-16.04
+    steps:
+     - id: stat_cmd
+       run: |
+         d_new=`date +%Y-%m-%d`
+         ts_new=`date +%s`
+         ts_old=`expr $ts_new - 3600 \* 24 \* 31`
+         d_old=`date -d@$ts_old +%Y-%m-%d`
+         echo "::set-output name=cmd::git log  --all --pretty=tformat: --since=$d_old --until=$d_new  --numstat | awk '{ add += \$1; subs += \$2} END { printf \"%s,%s\\n\", add, subs }' >> ../stat.txt"
+         
+     - id: server
+       run: |
+         git clone https://github.com/hugegraph/hugegraph.git
+         cd hugegraph
+         ${{ steps.stat_cmd.outputs.cmd }}
+ 
+     - id: loader
+       run: |
+         git clone https://github.com/hugegraph/hugegraph-loader.git
+         cd hugegraph-loader
+         ${{ steps.stat_cmd.outputs.cmd }}
+     
+     - id: tools
+       run: |
+         git clone https://github.com/hugegraph/hugegraph-tools.git
+         cd hugegraph-tools
+         ${{ steps.stat_cmd.outputs.cmd }}
+     
+     - id: hubbble
+       run: |
+         git clone https://github.com/hugegraph/hugegraph-hubble.git
+         cd hugegraph-hubble
+         ${{ steps.stat_cmd.outputs.cmd }}
+             
+     - id: common
+       run: |
+         git clone https://github.com/hugegraph/hugegraph-common.git
+         cd hugegraph-common
+         ${{ steps.stat_cmd.outputs.cmd }}     
+         
+     - id: client
+       run: |
+         git clone https://github.com/hugegraph/hugegraph-client.git
+         cd hugegraph-client
+         ${{ steps.stat_cmd.outputs.cmd }}      
+         
+     - id: computer
+       run: |
+         git clone https://github.com/hugegraph/hugegraph-computer.git
+         cd hugegraph-computer
+         ${{ steps.stat_cmd.outputs.cmd }}        
+
+     - name: code-stat
+       run: |
+         cat stat.txt
+         awk -F"," '{ add += $1; subs += $2; loc += $1 + $2} END { printf "add diff lines:%s, subtract diff lines:%s, total diff lines:%s\n", add, subs, loc}' stat.txt
+
+
+# jobs:
+#  code_statistics:
+#    runs-on: ubuntu-16.04
+#    steps:
+#       - uses: actions/checkout@v2
+#         with:
+#           repository: hugegraph/hugegraph
+#           path: hugegraph
+
+#       - uses: actions/checkout@v2
+#         with:
+#           repository: hugegraph/hugegraph-loader
+#           path: loader
+
+#       - uses: actions/checkout@v2
+#         with:
+#           repository: hugegraph/hugegraph-tools
+#           path: tools
+
+#       - uses: actions/checkout@v2
+#         with:
+#           repository: hugegraph/hugegraph-hubble
+#           path: hubble
+
+#       - uses: actions/checkout@v2
+#         with:
+#           repository: hugegraph/hugegraph-common
+#           path: common
+
+#       - uses: actions/checkout@v2
+#         with:
+#           repository: hugegraph/hugegraph-client
+#           path: client
+
+#       - uses: actions/checkout@v2
+#         with:
+#           repository: hugegraph/hugegraph-computer
+#           path: computer
+
+#       - id: code_stat
+#         run: |
+#           ls
+#           pwd
+#           cd hugegraph && git log  --all --pretty=tformat: --since=2021-06-01 --until=2021-07-01  --numstat | awk '{ add += $1; subs += $2} END { printf "%s,%s\n", add, subs }' >> ../stat.txt
+#           cd ../loader && git log  --all --pretty=tformat: --since=2021-06-01 --until=2021-07-01  --numstat | awk '{ add += $1; subs += $2} END { printf "%s,%s\n", add, subs }' >> ../stat.txt
+#           cd ../tools && git log --all --pretty=tformat: --since=2021-06-01 --until=2021-07-01  --numstat | awk '{ add += $1; subs += $2} END { printf "%s,%s\n", add, subs }' >> ../stat.txt
+#           cd ../hubble && git log  --all --pretty=tformat: --since=2021-06-01 --until=2021-07-01  --numstat | awk '{ add += $1; subs += $2} END { printf "%s,%s\n", add, subs }' >> ../stat.txt
+#           cd ../common && git log  --all --pretty=tformat: --since=2021-06-01 --until=2021-07-01  --numstat | awk '{ add += $1; subs += $2} END { printf "%s,%s\n", add, subs }' >> ../stat.txt
+#           cd ../client && git log  --all --pretty=tformat: --since=2021-06-01 --until=2021-07-01  --numstat | awk '{ add += $1; subs += $2} END { printf "%s,%s\n", add, subs }' >> ../stat.txt
+#           cd ../computer && git log  --all --pretty=tformat: --since=2021-06-01 --until=2021-07-01  --numstat | awk '{ add += $1; subs += $2} END { printf "%s,%s\n", add, subs }' >> ../stat.txt
+#           cat ../stat.txt
+
+
+
diff --git a/.github/workflows/rocksdb.yml b/.github/workflows/rocksdb.yml
@@ -51,75 +51,90 @@ jobs:
       - name: checkout python scripts
         uses: actions/checkout@v2
         with:
-          ref: master
           fetch-depth: 2
 
       - name: install python dependencies
-        if: steps.cache-pip.outputs.cache-hit != 'true'
         run: |
-          pwd
-          ls
-          tree
           python -m pip install --upgrade pip
           pip install -r ./requirements.txt
 
-      - name: edit config and install hugegraph components
+      - name: deploy hugegraph
         run: |
-          python src/deploy_start.py
+          python src/deploy_start.py all
+
+      - name: decompress dataset.zip
+        run: |
+          unzip src/config/dataset.zip
+          mv dataset src/config/
+          
+      - name: run test cases
+        run: |
+          pytest --html=test.html --capture=tee-sys
+
+
+  # job: openSource and exist auth
+  openSource_exist_auth:
+    name: openSource and exist auth
+    runs-on: ubuntu-16.04
+    steps:
+      - name: install JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
+      - name: cache maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: install python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+          architecture: 'x64'
+
+      - name: cache python dependencies
+        uses: actions/cache@v2
+        id: cache-pip
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: checkout python scripts
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: install python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ./requirements.txt
+
+      - name: deploy hugegraph
+        run: |
+          sed -i -e "s/^is_auth.*$/is_auth = True/g" src/config/basic_config.py
+          sed -i -e "s/^admin_password.*$/admin_password = {'admin': '123456'}/g" src/config/basic_config.py
+          sed -i -e "s/^test_password.*$/test_password = {'tester': '123456'}/g" src/config/basic_config.py
+          python src/deploy_start.py all
+
+
+      - name: decompress dataset.zip
+        run: |
+          unzip src/config/dataset.zip
+          mv dataset src/config/
 
       - name: run test cases
-        run: pytest src/test_start.py
-
-
-#  # job: openSource and exist auth
-#  openSource_exist_auth:
-#    name: openSource and exist auth
-#    runs-on: ubuntu-16.04
-#    steps:
-#      - name: install JDK 8
-#        uses: actions/setup-java@v2
-#        with:
-#          java-version: '8'
-#          distribution: 'adopt'
-#
-#      - name: cache maven repository
-#        uses: actions/cache@v2
-#        with:
-#          path: ~/.m2/repository
-#          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-#          restore-keys: |
-#            ${{ runner.os }}-maven-
-#
-#      - name: install python 3.7
-#        uses: actions/setup-python@v2
-#        with:
-#          python-version: '3.7'
-#          architecture: 'x64'
-#
-#      - name: cache python dependencies
-#        uses: actions/cache@v2
-#        id: cache-pip
-#        with:
-#          path: ~/.cache/pip
-#          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-#          restore-keys: |
-#            ${{ runner.os }}-pip-
-#
-#      - name: checkout python scripts
-#        uses: actions/checkout@v2
-#        with:
-#          ref: master
-#          fetch-depth: 2
-#
-#      - name: install python dependencies
-#        if: steps.cache-pip.outputs.cache-hit != 'true'
-#        run: |
-#            pip install -r requirements.txt
-#
-#      - name: edit config and install hugegraph components
-#        run: |
-#          sed -i 's/^is_auth.*$/is_auth = True/g' src/config/basic_config.py
-#          python src/deploy_start.py
-#
-#      - name: run test cases
-#        run: pytest src/test_start.py
+        run: |
+          pytest --html=test.html --capture=tee-sys
+
+        
+      
+
+
+
diff --git a/src/graph_deploy/deploy_graph.py → src/common/deploy_graph.py b/src/graph_deploy/deploy_graph.py → src/common/deploy_graph.py
@@ -22,41 +22,40 @@ def get_code(pwd, git_obj, code_dir):
     :param git_obj: git 配置
     :param code_dir:
     """
+    branch = git_obj['branch']
+    url = git_obj['url']
     if not is_match_re(pwd, code_dir):
-        branch = git_obj['branch']
-        url = git_obj['url']
-        print('cd %s && git clone -b %s %s' % (pwd, branch, url))
-        os.system('cd %s && git clone -b %s %s' % (pwd, branch, url))
+        clone_cmd = 'cd %s && git clone %s && cd %s && git checkout %s' % (pwd, url, code_dir, branch)
+        print('clone code: ' + clone_cmd)
+        os.system(clone_cmd)
     else:
-        os.system('cd %s/%s && git pull' % (pwd, code_dir))
+        pull_cmd = 'cd %s/%s && git checkout %s && git pull' % (pwd, code_dir, branch)
+        print('pull code: ' + pull_cmd)
+        os.system(pull_cmd)
 
 
-def compile_package(mvn_path, dir_code_path):
+def compile_package(dir_code_path):
     """
     编译包
-    :param mvn_path: 添加mvn_path
     :param dir_code_path: 本地代码库路径
     :return:
     """
     g_name = dir_code_path.split('/')[-1]
     if g_name == 'hugegraph-loader':
-        os.system(
-            'cd %s && '
-            '%smvn install:install-file '
-            '-Dfile=./assembly/static/lib/ojdbc8-12.2.0.1.jar '
-            '-DgroupId=com.oracle '
-            '-DartifactId=ojdbc8 '
-            '-Dversion=12.2.0.1 '
-            '-Dpackaging=jar && '
-            '%smvn clean package -Dmaven.test.skip=true | '
-            'grep -v \"Downloading\|Downloaded\"' % (dir_code_path, mvn_path, mvn_path)
-        )
+        cmd = 'cd %s && ' \
+              'mvn install:install-file ' \
+              '-Dfile=./assembly/static/lib/ojdbc8-12.2.0.1.jar ' \
+              '-DgroupId=com.oracle ' \
+              '-DartifactId=ojdbc8 ' \
+              '-Dversion=12.2.0.1 ' \
+              '-Dpackaging=jar | grep -v \"Downloading\|Downloaded\" && ' \
+              'mvn clean package -Dmaven.test.skip=true -q | grep \"tar.gz\"' % dir_code_path
+        print(cmd)
+        os.system(cmd)
     else:
-        os.system(
-            'cd %s && '
-            '%smvn clean package -Dmaven.test.skip=true | '
-            'grep -v \"Downloading\|Downloaded\"' % (dir_code_path, mvn_path)
-        )
+        cmd = 'cd %s && mvn clean package -Dmaven.test.skip=true -q | grep \"tar.gz\"' % dir_code_path
+        print(cmd)
+        os.system(cmd)
 
 
 def set_server_properties(package_dir_path, host, server_port, gremlin_port):
@@ -120,7 +119,6 @@ def __init__(self, obj):
         self.gremlin_port = obj.gremlin_port
         self.hubble_host = obj.hubble_host
         self.hubble_port = obj.hubble_port
-        self.mvn_path = obj.mvn_path
         self.code_path = obj.code_path
         self.server_git = obj.server_git
         self.loader_git = obj.loader_git
@@ -138,7 +136,7 @@ def server(self):
 
         is_exists_path(self.code_path)
         get_code(self.code_path, self.server_git, code_dir)
-        compile_package(self.mvn_path, code_dir_path)
+        compile_package(code_dir_path)
         #  start graph_server
         package_dir_name = is_match_re(code_dir_path, re_dir)
         package_dir_path = code_dir_path + '/' + package_dir_name
@@ -153,15 +151,15 @@ def hubble(self):
         code_dir = 'hugegraph-hubble'
         code_dir_path = self.code_path + '/' + code_dir
         re_dir = '^%s-(\d).(\d{1,2}).(\d)$' % code_dir
-#         # get code && compile
-#         is_exists_path(self.code_path)
-#         get_code(self.code_path, self.hubble_git, code_dir)
-#         compile_package(self.mvn_path, code_dir_path)
+        # # get code && compile
+        # is_exists_path(self.code_path)
+        # get_code(self.code_path, self.hubble_git, code_dir)
+        # compile_package(code_dir_path)
         # wget tar
         is_exists_path(code_dir_path)
         os.system(
             'cd %s && '
-            'wget https://github.com/hugegraph/hugegraph-hubble/releases/download/v1.5.0/hugegraph-hubble-1.5.0.tar.gz '
+            'wget https://github.com/hugegraph/hugegraph-hubble/releases/download/v1.5.0/hugegraph-hubble-1.5.0.tar.gz -q'
             '&& tar xzvf hugegraph-hubble-1.5.0.tar.gz' % code_dir_path
         )
         # set properties && start hubble
@@ -179,7 +177,7 @@ def loader(self):
         code_dir_path = self.code_path + '/' + code_dir
         is_exists_path(self.code_path)
         get_code(self.code_path, self.loader_git, code_dir)
-        compile_package(self.mvn_path, code_dir_path)
+        compile_package(code_dir_path)
 
     @staticmethod
     def tools(self):
@@ -190,7 +188,7 @@ def tools(self):
         code_dir_path = self.code_path + '/' + code_dir
         is_exists_path(self.code_path)
         get_code(self.code_path, self.tools_git, code_dir)
-        compile_package(self.mvn_path, code_dir_path)
+        compile_package(code_dir_path)
 
 
 if __name__ == "__main__":