From 2cb914f1d746dfec3ef77afc1ebdd5630ddf2721 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 29 Oct 2024 09:28:42 +0000 Subject: [PATCH 1/9] Update for to_json Signed-off-by: GBBBAS --- .../queries/time_series/_time_series_query_builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py b/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py index 085941859..d0d0771a0 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py @@ -1117,10 +1117,13 @@ def _query_builder(parameters_dict: dict, query_type: str) -> str: + " " + parameters_dict["time_interval_unit"][0] ) + to_json_flag = parameters_dict.get("to_json", False) + parameters_dict["to_json"] = False sample_prepared_query, sample_query, sample_parameters = _sample_query( parameters_dict ) sample_parameters["is_resample"] = False + sample_parameters["to_json"] = to_json_flag return _interpolation_query(parameters_dict, sample_query, sample_parameters) if query_type == "time_weighted_average": From bb4105aea68a8d663ab94f49c0d7d168f760611a Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 29 Oct 2024 09:45:35 +0000 Subject: [PATCH 2/9] Update package environment Signed-off-by: GBBBAS --- environment.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/environment.yml b/environment.yml index 645307d64..65eb76712 100644 --- a/environment.yml +++ b/environment.yml @@ -62,7 +62,7 @@ dependencies: - twine==4.0.2 - delta-sharing-python>=1.0.0,<2.0.0 - polars>=0.18.8,<1.0.0 - - moto[s3]>=4.1.14 + - moto[s3]>=4.1.14,<5.0.0 - xarray>=2023.1.0,<2023.8.0 - ecmwf-api-client>=1.6.3,<2.0.0 - netCDF4>=1.6.4,<2.0.0 @@ -70,18 +70,18 @@ dependencies: - joblib==1.3.2,<2.0.0 - great-expectations>=0.18.8,<1.0.0 - pip: - - databricks-sdk>=0.20.0,<1.0.0 - - dependency-injector>=4.41.0,<5.0.0 - - azure-functions>=1.15.0,<2.0.0 - - azure-mgmt-eventgrid>=10.2.0 - - hvac>=1.1.1 - - langchain>=0.2.0,<0.3.0 - - langchain-community>=0.2.0,<0.3.0 - - build==0.10.0 - - deltalake>=0.10.1,<1.0.0 - - trio>=0.22.1 - - sqlparams>=5.1.0,<6.0.0 - - entsoe-py>=0.5.10,<1.0.0 - - web3>=6.18.0,<7.0.0 - - eth-typing>=4.2.3,<5.0.0 - - pandas>=1.5.2,<2.2.0 + - databricks-sdk>=0.20.0,<1.0.0 + - dependency-injector>=4.41.0,<5.0.0 + - azure-functions>=1.15.0,<2.0.0 + - azure-mgmt-eventgrid>=10.2.0 + - hvac>=1.1.1 + - langchain>=0.2.0,<0.3.0 + - langchain-community>=0.2.0,<0.3.0 + - build==0.10.0 + - deltalake>=0.10.1,<1.0.0 + - trio>=0.22.1 + - sqlparams>=5.1.0,<6.0.0 + - entsoe-py>=0.5.10,<1.0.0 + - web3>=6.18.0,<7.0.0 + - eth-typing>=4.2.3,<5.0.0 + - pandas>=1.5.2,<2.2.0 From e851809475bbae509d64ad3abacf2bae84fca704 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 29 Oct 2024 09:51:11 +0000 Subject: [PATCH 3/9] Updates to moto package Signed-off-by: GBBBAS --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 65eb76712..9e81c6071 100644 --- a/environment.yml +++ b/environment.yml @@ -62,7 +62,7 @@ dependencies: - twine==4.0.2 - delta-sharing-python>=1.0.0,<2.0.0 - polars>=0.18.8,<1.0.0 - - moto[s3]>=4.1.14,<5.0.0 + - moto[s3]>=5.0.16,<6.0.0 - xarray>=2023.1.0,<2023.8.0 - ecmwf-api-client>=1.6.3,<2.0.0 - netCDF4>=1.6.4,<2.0.0 From da019b86b3c40960f2d0f91d945158b8008f6ac9 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 29 Oct 2024 09:54:43 +0000 Subject: [PATCH 4/9] Update to moto Signed-off-by: GBBBAS --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 9e81c6071..5e8d1eb2f 100644 --- a/environment.yml +++ b/environment.yml @@ -62,7 +62,6 @@ dependencies: - twine==4.0.2 - delta-sharing-python>=1.0.0,<2.0.0 - polars>=0.18.8,<1.0.0 - - moto[s3]>=5.0.16,<6.0.0 - xarray>=2023.1.0,<2023.8.0 - ecmwf-api-client>=1.6.3,<2.0.0 - netCDF4>=1.6.4,<2.0.0 @@ -85,3 +84,4 @@ dependencies: - web3>=6.18.0,<7.0.0 - eth-typing>=4.2.3,<5.0.0 - pandas>=1.5.2,<2.2.0 + - moto[s3]>=5.0.16,<6.0.0 From 665ab8744b26e7c73c31c3fc9239bdf1a53c8697 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 29 Oct 2024 10:13:49 +0000 Subject: [PATCH 5/9] Update actions packages Signed-off-by: GBBBAS --- .github/workflows/develop.yml | 4 +- .github/workflows/release.yml | 10 +- .github/workflows/sonarcloud.yml | 94 ++++++------- .github/workflows/sonarcloud_reusable.yml | 27 ++-- .github/workflows/test.yml | 160 +++++++++++----------- 5 files changed, 147 insertions(+), 148 deletions(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 30c8f2190..3476ae2f6 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -44,9 +44,9 @@ jobs: packages: write contents: read steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: 3.11 - name: Install dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be941be8d..815f58056 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,9 +23,9 @@ jobs: job_build_python_whl: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: 3.11 - name: Install dependencies @@ -53,7 +53,7 @@ jobs: contents: read steps: - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Log in to Docker Hub uses: docker/login-action@v2 @@ -96,10 +96,10 @@ jobs: env: PYTHONPATH: home/runner/work/core/ steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: 3.11 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 9182c1934..d0803c87c 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Sonarcloud Scan' +name: "Sonarcloud Scan" run-name: ${{ github.event.workflow_run.display_title }} @@ -27,53 +27,53 @@ on: jobs: job_download_pr_artifact: - outputs: - pr_info: ${{ steps.pr.outputs.result }} - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: 'Download Artifact' - uses: actions/github-script@v6 - with: - script: | - let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.payload.workflow_run.id, - }); - let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == "pr_number" - })[0]; - let download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_number.zip`, Buffer.from(download.data)); + outputs: + pr_info: ${{ steps.pr.outputs.result }} + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - name: "Download Artifact" + uses: actions/github-script@v7 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "pr_number" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_number.zip`, Buffer.from(download.data)); - - name: 'Unzip Artifact' - run: unzip pr_number.zip + - name: "Unzip Artifact" + run: unzip pr_number.zip - - name: 'Read Artifact' - id: pr - uses: actions/github-script@v6 - with: - script: | - let fs = require('fs'); - return fs.readFileSync('./pr_number'); - result-encoding: string + - name: "Read Artifact" + id: pr + uses: actions/github-script@v7 + with: + script: | + let fs = require('fs'); + return fs.readFileSync('./pr_number'); + result-encoding: string job_run_unit_tests_and_sonarqube: - needs: job_download_pr_artifact - uses: rtdip/core/.github/workflows/sonarcloud_reusable.yml@develop - with: - REPO_NAME: ${{ github.event.workflow_run.head_repository.full_name }} - HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - PR_NUMBER: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_number }} - PR_HEAD_REF: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_head_ref }} - PR_BASE_REF: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_base_ref }} - secrets: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + needs: job_download_pr_artifact + uses: rtdip/core/.github/workflows/sonarcloud_reusable.yml@develop + with: + REPO_NAME: ${{ github.event.workflow_run.head_repository.full_name }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + PR_NUMBER: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_number }} + PR_HEAD_REF: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_head_ref }} + PR_BASE_REF: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_base_ref }} + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/sonarcloud_reusable.yml b/.github/workflows/sonarcloud_reusable.yml index f3a3fd021..884fe15d4 100644 --- a/.github/workflows/sonarcloud_reusable.yml +++ b/.github/workflows/sonarcloud_reusable.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Reusable Sonarcloud Scan' +name: "Reusable Sonarcloud Scan" on: workflow_call: @@ -25,20 +25,20 @@ on: type: string HEAD_SHA: required: true - type: string + type: string PR_NUMBER: required: true - type: string + type: string PR_HEAD_REF: required: true type: string PR_BASE_REF: required: true - type: string + type: string secrets: SONAR_TOKEN: required: true - + jobs: job_test_python_pyspark_latest_version: defaults: @@ -50,19 +50,19 @@ jobs: python-version: ["3.11"] pyspark: ["3.5.1"] delta-spark: ["3.0.0"] - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: ${{ inputs.REPO_NAME }} ref: ${{ inputs.HEAD_BRANCH }} fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@v4 + - name: Setup Python + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - + - name: Install Boost run: | sudo apt update @@ -80,7 +80,7 @@ jobs: create-args: >- python=${{ matrix.python-version }} pyspark=${{ matrix.pyspark }} - delta-spark=${{ matrix.delta-spark }} + delta-spark=${{ matrix.delta-spark }} cache-environment: true - name: Test @@ -93,12 +93,11 @@ jobs: coverage xml --omit "venv/**,maintenance/**,xunit-reports/**" -i -o coverage-reports/coverage-unittests.xml echo Coverage `coverage report --omit "venv/**" | grep TOTAL | tr -s ' ' | cut -d" " -f4` - - name: Mkdocs Test run: | mkdocs build --strict - - name: Override Coverage Source Path for Sonar + - name: Override Coverage Source Path for Sonar run: | sed -i "s/\/home\/runner\/work\/core\/core<\/source>/\/github\/workspace<\/source>/g" /home/runner/work/core/core/coverage-reports/coverage-unittests.xml @@ -115,5 +114,5 @@ jobs: -Dsonar.pullrequest.branch=${{ inputs.PR_HEAD_REF }} -Dsonar.pullrequest.base=${{ inputs.PR_BASE_REF }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 192747f41..36333108e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,88 +12,88 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Reusable Test Workflow' +name: "Reusable Test Workflow" on: workflow_call: jobs: job_test_python_pyspark_versions: - defaults: - run: - shell: bash -l {0} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python-version: ["3.9", "3.10", "3.11"] - pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0"] - exclude: - - pyspark: "3.5.0" - python-version: "3.9" - - pyspark: "3.5.0" - python-version: "3.10" - - pyspark: "3.4.1" - python-version: "3.9" - - pyspark: "3.4.1" - python-version: "3.10" - - pyspark: "3.4.0" - python-version: "3.9" - - pyspark: "3.4.0" - python-version: "3.10" - - pyspark: "3.3.2" - python-version: "3.11" - - pyspark: "3.3.1" - python-version: "3.11" - - pyspark: "3.3.0" - python-version: "3.11" - include: - - pyspark: "3.3.0" - delta-spark: "2.2.0" - - pyspark: "3.3.1" - delta-spark: "2.3.0" - - pyspark: "3.3.2" - delta-spark: "2.3.0" - - pyspark: "3.4.0" - delta-spark: "2.4.0" - - pyspark: "3.4.1" - delta-spark: "2.4.0" - - pyspark: "3.5.0" - delta-spark: "3.0.0" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.9", "3.10", "3.11"] + pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0"] + exclude: + - pyspark: "3.5.0" + python-version: "3.9" + - pyspark: "3.5.0" + python-version: "3.10" + - pyspark: "3.4.1" + python-version: "3.9" + - pyspark: "3.4.1" + python-version: "3.10" + - pyspark: "3.4.0" + python-version: "3.9" + - pyspark: "3.4.0" + python-version: "3.10" + - pyspark: "3.3.2" + python-version: "3.11" + - pyspark: "3.3.1" + python-version: "3.11" + - pyspark: "3.3.0" + python-version: "3.11" + include: + - pyspark: "3.3.0" + delta-spark: "2.2.0" + - pyspark: "3.3.1" + delta-spark: "2.3.0" + - pyspark: "3.3.2" + delta-spark: "2.3.0" + - pyspark: "3.4.0" + delta-spark: "2.4.0" + - pyspark: "3.4.1" + delta-spark: "2.4.0" + - pyspark: "3.5.0" + delta-spark: "3.0.0" + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Boost - run: | - sudo apt update - sudo apt install -y libboost-all-dev + - name: Install Boost + run: | + sudo apt update + sudo apt install -y libboost-all-dev - - name: Add conda to system path - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH - - name: Install Conda environment with Micromamba - uses: mamba-org/setup-micromamba@main - with: - environment-file: environment.yml - create-args: >- - python=${{ matrix.python-version }} - pyspark=${{ matrix.pyspark }} - delta-spark=${{ matrix.delta-spark }} - cache-environment: true + - name: Install Conda environment with Micromamba + uses: mamba-org/setup-micromamba@main + with: + environment-file: environment.yml + create-args: >- + python=${{ matrix.python-version }} + pyspark=${{ matrix.pyspark }} + delta-spark=${{ matrix.delta-spark }} + cache-environment: true - - name: Test - run: | - coverage run -m pytest --junitxml=xunit-reports/xunit-result-unitttests.xml tests + - name: Test + run: | + coverage run -m pytest --junitxml=xunit-reports/xunit-result-unitttests.xml tests job_test_mkdocs: defaults: @@ -105,19 +105,19 @@ jobs: python-version: ["3.11"] pyspark: ["3.5.0"] delta-spark: ["3.0.0"] - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: ${{ inputs.REPO_NAME }} ref: ${{ inputs.HEAD_BRANCH }} fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@v4 + - name: Setup Python + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - + - name: Install Boost run: | sudo apt update @@ -135,7 +135,7 @@ jobs: create-args: >- python=${{ matrix.python-version }} pyspark=${{ matrix.pyspark }} - delta-spark=${{ matrix.delta-spark }} + delta-spark=${{ matrix.delta-spark }} cache-environment: true - name: Mkdocs Test @@ -145,5 +145,5 @@ jobs: job_lint_python_black: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: psf/black@stable \ No newline at end of file + - uses: actions/checkout@v4 + - uses: psf/black@stable From 7cfea055f03074fd7889d772e04eaa736bff58eb Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 29 Oct 2024 15:58:37 +0000 Subject: [PATCH 6/9] Updates for Packages and interpolation query Signed-off-by: GBBBAS --- environment.yml | 1 + setup.py | 1 + .../queries/time_series/_time_series_query_builder.py | 8 ++++---- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index 5e8d1eb2f..d9d65e0a8 100644 --- a/environment.yml +++ b/environment.yml @@ -85,3 +85,4 @@ dependencies: - eth-typing>=4.2.3,<5.0.0 - pandas>=1.5.2,<2.2.0 - moto[s3]>=5.0.16,<6.0.0 + - pyarrow>=14.0.1,<17.0.0 diff --git a/setup.py b/setup.py index 8d340d1d0..d1e0c5d90 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ INSTALL_REQUIRES = [ "databricks-sql-connector>=3.1.0,<4.0.0", + "pyarrow>=14.0.1,<17.0.0", "azure-identity>=1.12.0,<2.0.0", "pandas>=1.5.2,<2.2.0", "jinja2>=3.1.2,<4.0.0", diff --git a/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py b/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py index d0d0771a0..e89c7b07c 100644 --- a/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/time_series/_time_series_query_builder.py @@ -171,10 +171,10 @@ def _sample_query(parameters_dict: dict) -> tuple: "'{{ tag_names[i] }}' AS `{{ tag_names[i] }}`{% if not loop.last %}, {% endif %}" "{% endfor %}" "{% endif %}" - '))) SELECT {% if to_json is defined and to_json == true %}to_json(struct(*), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}*{% endif %} FROM pivot ORDER BY `{{ timestamp_column }}` ' + '))) SELECT {% if to_json_resample is defined and to_json_resample == true %}to_json(struct(*), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}*{% endif %} FROM pivot ORDER BY `{{ timestamp_column }}` ' "{% else %}" "{% if display_uom is defined and display_uom == true %}" - 'SELECT {% if to_json is defined and to_json == true %}to_json(struct(p.`EventTime`, p.`TagName`, p.`Value`, m.`UoM`), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}p.`EventTime`, p.`TagName`, p.`Value`, m.`UoM`{% endif %} FROM project p ' + 'SELECT {% if to_json_resample is defined and to_json_resample == true %}to_json(struct(p.`EventTime`, p.`TagName`, p.`Value`, m.`UoM`), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}p.`EventTime`, p.`TagName`, p.`Value`, m.`UoM`{% endif %} FROM project p ' "LEFT OUTER JOIN " "{% if metadata_source is defined and metadata_source is not none %}" "`{{ metadata_source|lower }}` m ON p.`{{ tagname_column }}` = m.`{{ metadata_tagname_column }}` " @@ -182,7 +182,7 @@ def _sample_query(parameters_dict: dict) -> tuple: "`{{ business_unit|lower }}`.`sensors`.`{{ asset|lower }}_{{ data_security_level|lower }}_metadata` m ON p.`{{ tagname_column }}` = m.`{{ tagname_column }}` " "{% endif %}" "{% else %}" - 'SELECT {% if to_json is defined and to_json == true %}to_json(struct(*), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}*{% endif %} FROM project ' + 'SELECT {% if to_json_resample is defined and to_json_resample == true %}to_json(struct(*), map("timestampFormat", "yyyy-MM-dd\'T\'HH:mm:ss.SSSSSSSSSXXX")) as Value{% else %}*{% endif %} FROM project ' "{% endif %}" "{% endif %}" "{% if is_resample is defined and is_resample == true and limit is defined and limit is not none %}" @@ -237,7 +237,7 @@ def _sample_query(parameters_dict: dict) -> tuple: "metadata_tagname_column", "TagName" ), "metadata_uom_column": parameters_dict.get("metadata_uom_column", "UoM"), - "to_json": parameters_dict.get("to_json", False), + "to_json_resample": parameters_dict.get("to_json", False), } sql_template = Template(sample_query) From b5d5339884bf02eec01835bcc38b733191f476d2 Mon Sep 17 00:00:00 2001 From: GBBBAS Date: Tue, 29 Oct 2024 16:12:57 +0000 Subject: [PATCH 7/9] Update pyspark versions Signed-off-by: GBBBAS --- .github/workflows/sonarcloud_reusable.yml | 4 ++-- .github/workflows/test.yml | 12 +++++++++--- setup.py | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sonarcloud_reusable.yml b/.github/workflows/sonarcloud_reusable.yml index 884fe15d4..0ea070121 100644 --- a/.github/workflows/sonarcloud_reusable.yml +++ b/.github/workflows/sonarcloud_reusable.yml @@ -48,8 +48,8 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.11"] - pyspark: ["3.5.1"] - delta-spark: ["3.0.0"] + pyspark: ["3.5.2"] + delta-spark: ["3.2.1"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36333108e..2ff1d9582 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,12 +27,16 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.9", "3.10", "3.11"] - pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0"] + pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0", "3.5.1"] exclude: - pyspark: "3.5.0" python-version: "3.9" - pyspark: "3.5.0" python-version: "3.10" + - pyspark: "3.5.1" + python-version: "3.9" + - pyspark: "3.5.1" + python-version: "3.10" - pyspark: "3.4.1" python-version: "3.9" - pyspark: "3.4.1" @@ -60,6 +64,8 @@ jobs: delta-spark: "2.4.0" - pyspark: "3.5.0" delta-spark: "3.0.0" + - pyspark: "3.5.1" + delta-spark: "3.1.0" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -103,8 +109,8 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.11"] - pyspark: ["3.5.0"] - delta-spark: ["3.0.0"] + pyspark: ["3.5.2"] + delta-spark: ["3.2.1"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/setup.py b/setup.py index d1e0c5d90..b991d88b5 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ PYSPARK_PACKAGES = [ "pyspark>=3.3.0,<3.6.0", - "delta-spark>=2.2.0,<3.2.0", + "delta-spark>=2.2.0,<3.2.1", ] PIPELINE_PACKAGES = [ From b3f674e3dd03475a4714c649abbb3e4447ef5fd8 Mon Sep 17 00:00:00 2001 From: GBBBAS <42962356+GBBBAS@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:20:00 +0000 Subject: [PATCH 8/9] Update pyspark versions (#835) Signed-off-by: GBBBAS --- .github/workflows/sonarcloud_reusable.yml | 4 ++-- .github/workflows/test.yml | 28 ++++++++++++++++++----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sonarcloud_reusable.yml b/.github/workflows/sonarcloud_reusable.yml index 0ea070121..3c358efd1 100644 --- a/.github/workflows/sonarcloud_reusable.yml +++ b/.github/workflows/sonarcloud_reusable.yml @@ -48,8 +48,8 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.11"] - pyspark: ["3.5.2"] - delta-spark: ["3.2.1"] + pyspark: ["3.5.3"] + delta-spark: ["3.0.0"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ff1d9582..99bd9856d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,16 +27,30 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.9", "3.10", "3.11"] - pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0", "3.5.1"] + pyspark: + [ + "3.3.0", + "3.3.1", + "3.3.2", + "3.4.0", + "3.4.1", + "3.5.0", + "3.5.1", + "3.5.2", + ] exclude: - - pyspark: "3.5.0" + - pyspark: "3.5.2" python-version: "3.9" - - pyspark: "3.5.0" + - pyspark: "3.5.2" python-version: "3.10" - pyspark: "3.5.1" python-version: "3.9" - pyspark: "3.5.1" python-version: "3.10" + - pyspark: "3.5.0" + python-version: "3.9" + - pyspark: "3.5.0" + python-version: "3.10" - pyspark: "3.4.1" python-version: "3.9" - pyspark: "3.4.1" @@ -65,7 +79,9 @@ jobs: - pyspark: "3.5.0" delta-spark: "3.0.0" - pyspark: "3.5.1" - delta-spark: "3.1.0" + delta-spark: "3.0.0" + - pyspark: "3.5.2" + delta-spark: "3.0.0" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -109,8 +125,8 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.11"] - pyspark: ["3.5.2"] - delta-spark: ["3.2.1"] + pyspark: ["3.5.3"] + delta-spark: ["3.0.0"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From e67d6fdab109e867de5ca42ba0561c40077cc004 Mon Sep 17 00:00:00 2001 From: GBBBAS <42962356+GBBBAS@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:33:35 +0000 Subject: [PATCH 9/9] Remove 3.5.2 from test list (#836) Signed-off-by: GBBBAS --- .github/workflows/test.yml | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99bd9856d..53af11eeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,22 +27,8 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.9", "3.10", "3.11"] - pyspark: - [ - "3.3.0", - "3.3.1", - "3.3.2", - "3.4.0", - "3.4.1", - "3.5.0", - "3.5.1", - "3.5.2", - ] + pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0", "3.5.1"] exclude: - - pyspark: "3.5.2" - python-version: "3.9" - - pyspark: "3.5.2" - python-version: "3.10" - pyspark: "3.5.1" python-version: "3.9" - pyspark: "3.5.1" @@ -80,8 +66,6 @@ jobs: delta-spark: "3.0.0" - pyspark: "3.5.1" delta-spark: "3.0.0" - - pyspark: "3.5.2" - delta-spark: "3.0.0" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4