From 006a5baf08cb72da130cd03a962cf216e35837d8 Mon Sep 17 00:00:00 2001 From: Daniele Briggi Date: Wed, 22 May 2024 13:22:03 +0000 Subject: [PATCH 1/5] Code linting Installed isort and flake8 Setup flake8 --- .flake8 | 6 ++++++ pyproject.toml | 6 ++++++ requirements-dev.txt | 11 +++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 .flake8 create mode 100644 pyproject.toml diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..1253914 --- /dev/null +++ b/.flake8 @@ -0,0 +1,6 @@ +[flake8] +# E203: it's a parse limit of flake8 +# W503: https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#w503 +ignore = E203, W503, E501,E701 +# align to black choice +max-line-length = 88 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..54087a8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +# Configuration file to store build system requirements for Python projects +# https://www.python.org/dev/peps/pep-0518/ + +[tool.isort] +# make the tools compatible to each other +profile = "black" \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index c3c2518..73cb201 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,12 @@ -pylint==2.13.9 -mypy==0.971 typing-extensions==4.1.1 -bump2version==1.0.1 pytest==7.0.1 pytest-mock==3.6.1 -black==22.8.0 python-dotenv==0.20.0 lz4==3.1.10 +bump2version==1.0.1 +black==22.8.0 +pylint==2.13.9 +mypy==0.971 +flake8==5.0.4 +isort==5.10.1 +autoflake==1.4 \ No newline at end of file From fdc45c5b236d17dc5ac8d9f18cf6528f7a3302d9 Mon Sep 17 00:00:00 2001 From: Daniele Briggi Date: Wed, 22 May 2024 13:52:49 +0000 Subject: [PATCH 2/5] Apply black, isort, autoflake Test with autoflake --- .flake8 | 4 +-- src/setup.py | 44 +++++++++++++------------- src/sqlitecloud/client.py | 10 +++--- src/sqlitecloud/download.py | 2 +- src/sqlitecloud/driver.py | 8 +++-- src/sqlitecloud/pubsub.py | 3 +- src/sqlitecloud/types.py | 4 +-- src/sqlitecloud/upload.py | 10 +++--- src/tests/conftest.py | 5 ++- src/tests/integration/test_client.py | 25 +++++++-------- src/tests/integration/test_download.py | 9 +++--- src/tests/integration/test_driver.py | 16 ++++++++-- src/tests/integration/test_pubsub.py | 17 +++++++--- src/tests/integration/test_upload.py | 5 ++- src/tests/unit/test_client.py | 29 ++++++++--------- src/tests/unit/test_driver.py | 14 +------- src/tests/unit/test_resultset.py | 5 +-- 17 files changed, 108 insertions(+), 102 deletions(-) diff --git a/.flake8 b/.flake8 index 1253914..53a4e5b 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ [flake8] # E203: it's a parse limit of flake8 -# W503: https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#w503 -ignore = E203, W503, E501,E701 +# W503,E501,E701: https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#w503 +ignore = E203, W503, E501, E701 # align to black choice max-line-length = 88 \ No newline at end of file diff --git a/src/setup.py b/src/setup.py index 99522dd..f179d4b 100644 --- a/src/setup.py +++ b/src/setup.py @@ -1,44 +1,44 @@ import io import os -from setuptools import setup, find_packages +from setuptools import find_packages, setup def read_file(filename): base_path = os.path.abspath(os.path.dirname(__file__)) full_path = os.path.join(base_path, filename) try: - with io.open(full_path, encoding='utf-8') as file: + with io.open(full_path, encoding="utf-8") as file: return file.read() except FileNotFoundError: - full_path = os.path.join(base_path, f'../{filename}') - with io.open(full_path, encoding='utf-8') as file: + full_path = os.path.join(base_path, f"../{filename}") + with io.open(full_path, encoding="utf-8") as file: return file.read() setup( - name='SqliteCloud', - version='0.0.75', - author='sqlitecloud.io', - description='A Python package for working with SQLite databases in the cloud.', - long_description=read_file('README-PYPI.md'), - long_description_content_type='text/markdown', + name="SqliteCloud", + version="0.0.75", + author="sqlitecloud.io", + description="A Python package for working with SQLite databases in the cloud.", + long_description=read_file("README-PYPI.md"), + long_description_content_type="text/markdown", url="https://github.com/sqlitecloud/python", packages=find_packages(), install_requires=[ - 'lz4 == 3.1.10', + "lz4 == 3.1.10", ], classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], - license='MIT', + license="MIT", ) diff --git a/src/sqlitecloud/client.py b/src/sqlitecloud/client.py index 647e6c5..ba201dd 100644 --- a/src/sqlitecloud/client.py +++ b/src/sqlitecloud/client.py @@ -29,7 +29,8 @@ def __init__( """Initializes a new instance of the class with connection information. Args: - cloud_account (SqliteCloudAccount): The account information for the SQlite Cloud database. + cloud_account (SqliteCloudAccount): The account information for the + SQlite Cloud database. connection_str (str): The connection string for the SQlite Cloud database. Eg: sqlitecloud://user:pass@host.com:port/dbname?timeout=10&apikey=abcd123 @@ -49,7 +50,8 @@ def open_connection(self) -> SQCloudConnect: """Opens a connection to the SQCloud server. Returns: - SQCloudConnect: An instance of the SQCloudConnect class representing the connection to the SQCloud server. + SQCloudConnect: An instance of the SQCloudConnect class representing + the connection to the SQCloud server. Raises: SQCloudException: If an error occurs while opening the connection. @@ -75,9 +77,7 @@ def is_connected(self, conn: SQCloudConnect) -> bool: """ return self._driver.is_connected(conn) - def exec_query( - self, query: str, conn: SQCloudConnect - ) -> SqliteCloudResultSet: + def exec_query(self, query: str, conn: SQCloudConnect) -> SqliteCloudResultSet: """Executes a SQL query on the SQLite Cloud database. Args: diff --git a/src/sqlitecloud/download.py b/src/sqlitecloud/download.py index 92bbf92..21ac3dd 100644 --- a/src/sqlitecloud/download.py +++ b/src/sqlitecloud/download.py @@ -1,5 +1,5 @@ -from io import BufferedWriter import logging +from io import BufferedWriter from sqlitecloud.driver import Driver from sqlitecloud.types import SQCloudConnect diff --git a/src/sqlitecloud/driver.py b/src/sqlitecloud/driver.py index aeb7767..efa207c 100644 --- a/src/sqlitecloud/driver.py +++ b/src/sqlitecloud/driver.py @@ -1,10 +1,13 @@ -from io import BufferedReader, BufferedWriter import logging import select +import socket import ssl import threading +from io import BufferedReader, BufferedWriter from typing import Callable, Optional, Union + import lz4.block + from sqlitecloud.resultset import SQCloudResult, SqliteCloudResultSet from sqlitecloud.types import ( SQCLOUD_CMD, @@ -19,7 +22,6 @@ SQCloudRowsetSignature, SQCloudValue, ) -import socket class Driver: @@ -131,7 +133,7 @@ def _internal_connect( try: sock.connect((hostname, port)) except Exception as e: - errmsg = f"An error occurred while initializing the socket." + errmsg = "An error occurred while initializing the socket." raise SQCloudException(errmsg) from e return sock diff --git a/src/sqlitecloud/pubsub.py b/src/sqlitecloud/pubsub.py index 148a7c2..39277cf 100644 --- a/src/sqlitecloud/pubsub.py +++ b/src/sqlitecloud/pubsub.py @@ -1,6 +1,5 @@ -import socket -from sqlite3 import connect from typing import Callable, Optional + from sqlitecloud.driver import Driver from sqlitecloud.resultset import SqliteCloudResultSet from sqlitecloud.types import SQCLOUD_PUBSUB_SUBJECT, SQCloudConnect diff --git a/src/sqlitecloud/types.py b/src/sqlitecloud/types.py index 6fb1aec..468bcfc 100644 --- a/src/sqlitecloud/types.py +++ b/src/sqlitecloud/types.py @@ -1,9 +1,7 @@ +import types from asyncio import AbstractEventLoop from enum import Enum -from threading import Thread -import types from typing import Callable, Optional -from enum import Enum class SQCLOUD_DEFAULT(Enum): diff --git a/src/sqlitecloud/upload.py b/src/sqlitecloud/upload.py index 9f90a7e..bea6578 100644 --- a/src/sqlitecloud/upload.py +++ b/src/sqlitecloud/upload.py @@ -1,9 +1,11 @@ -from io import BufferedReader +import logging import os +from io import BufferedReader from typing import Optional + from sqlitecloud.driver import Driver from sqlitecloud.types import SQCloudConnect -import logging + def xCallback(fd: BufferedReader, blen: int, ntot: int, nprogress: int) -> bytes: """ @@ -45,11 +47,11 @@ def upload_db( SQCloudException: If an error occurs while uploading the database. """ - + # Create a driver object driver = Driver() - with open(filename, 'rb') as fd: + with open(filename, "rb") as fd: dbsize = os.path.getsize(filename) driver.upload_database( diff --git a/src/tests/conftest.py b/src/tests/conftest.py index cf8b266..e7c5bff 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -1,14 +1,17 @@ import os + import pytest from dotenv import load_dotenv from sqlitecloud.client import SqliteCloudClient from sqlitecloud.types import SQCloudConnect, SqliteCloudAccount + @pytest.fixture(autouse=True) def load_env_vars(): load_dotenv(".env") + @pytest.fixture() def sqlitecloud_connection(): account = SqliteCloudAccount() @@ -26,4 +29,4 @@ def sqlitecloud_connection(): yield (connection, client) - client.disconnect(connection) \ No newline at end of file + client.disconnect(connection) diff --git a/src/tests/integration/test_client.py b/src/tests/integration/test_client.py index 1ae2622..4e143da 100644 --- a/src/tests/integration/test_client.py +++ b/src/tests/integration/test_client.py @@ -1,11 +1,9 @@ import json -from multiprocessing import connection import os -import sqlite3 -import tempfile import time import pytest + from sqlitecloud.client import SqliteCloudClient from sqlitecloud.types import ( SQCLOUD_ERRCODE, @@ -90,10 +88,10 @@ def test_is_connected(self): client = SqliteCloudClient(cloud_account=account) conn = client.open_connection() - assert client.is_connected(conn) == True + assert client.is_connected(conn) client.disconnect(conn) - assert client.is_connected(conn) == False + assert not client.is_connected(conn) def test_disconnect(self): account = SqliteCloudAccount() @@ -104,10 +102,10 @@ def test_disconnect(self): client = SqliteCloudClient(cloud_account=account) conn = client.open_connection() - assert client.is_connected(conn) == True + assert client.is_connected(conn) client.disconnect(conn) - assert client.is_connected(conn) == False + assert not client.is_connected(conn) assert conn.socket is None assert conn.pubsub_socket is None @@ -119,7 +117,7 @@ def test_select(self, sqlitecloud_connection): result = client.exec_query("SELECT 'Hello'", connection) - assert False == result.is_result + assert not result.is_result assert 1 == result.nrows assert 1 == result.ncols assert "Hello" == result.get_value(0, 0) @@ -239,7 +237,7 @@ def test_json(self, sqlitecloud_connection): connection, client = sqlitecloud_connection result = client.exec_query("TEST JSON", connection) - assert SQCLOUD_RESULT_TYPE.RESULT_JSON == result.tag + assert SQCLOUD_RESULT_TYPE.RESULT_JSON == result.tag assert { "msg-from": {"class": "soldier", "name": "Wixilav"}, "msg-to": {"class": "supreme-commander", "name": "[Redacted]"}, @@ -293,7 +291,7 @@ def test_array(self, sqlitecloud_connection): result = client.exec_query("TEST ARRAY", connection) result_array = result.get_result() - + assert SQCLOUD_RESULT_TYPE.RESULT_ARRAY == result.tag assert isinstance(result_array, list) assert len(result_array) == 5 @@ -332,7 +330,6 @@ def test_max_rows_option(self): # just expect everything is ok assert rowset.nrows > 100 - def test_max_rowset_option_to_fail_when_rowset_is_bigger(self): account = SqliteCloudAccount() account.hostname = os.getenv("SQLITE_HOST") @@ -352,7 +349,6 @@ def test_max_rowset_option_to_fail_when_rowset_is_bigger(self): assert SQCLOUD_ERRCODE.INTERNAL.value == e.value.errcode assert "RowSet too big to be sent (limit set to 1024 bytes)." == e.value.errmsg - def test_max_rowset_option_to_succeed_when_rowset_is_lighter(self): account = SqliteCloudAccount() account.hostname = os.getenv("SQLITE_HOST") @@ -439,7 +435,7 @@ def test_query_timeout(self): LIMIT 10000000 ) SELECT i FROM r WHERE i = 1;""", - connection + connection, ) client.disconnect(connection) @@ -645,7 +641,8 @@ def test_compression_multiple_columns(self): # min compression size for rowset set by default to 20400 bytes rowset = client.exec_query( - "SELECT * from albums inner join albums a2 on albums.AlbumId = a2.AlbumId", connection + "SELECT * from albums inner join albums a2 on albums.AlbumId = a2.AlbumId", + connection, ) client.disconnect(connection) diff --git a/src/tests/integration/test_download.py b/src/tests/integration/test_download.py index 513eaaa..2f0d0a7 100644 --- a/src/tests/integration/test_download.py +++ b/src/tests/integration/test_download.py @@ -1,12 +1,13 @@ -import os import sqlite3 import tempfile import pytest from sqlitecloud import download -from sqlitecloud.client import SqliteCloudClient -from sqlitecloud.types import SQCLOUD_ERRCODE, SQCloudConnect, SQCloudException, SqliteCloudAccount +from sqlitecloud.types import ( + SQCLOUD_ERRCODE, + SQCloudException, +) class TestDownload: @@ -31,4 +32,4 @@ def test_download_missing_database(self, sqlitecloud_connection): download.download_db(connection, "missing.sqlite", temp_file) assert e.value.errcode == SQCLOUD_ERRCODE.COMMAND.value - assert e.value.errmsg == "Database missing.sqlite does not exist." \ No newline at end of file + assert e.value.errmsg == "Database missing.sqlite does not exist." diff --git a/src/tests/integration/test_driver.py b/src/tests/integration/test_driver.py index ac1376e..c6b7a7d 100644 --- a/src/tests/integration/test_driver.py +++ b/src/tests/integration/test_driver.py @@ -1,10 +1,14 @@ import tempfile + from sqlitecloud.driver import Driver + class TestDriver: - def test_download_missing_database_without_error_when_expected(self, sqlitecloud_connection): + def test_download_missing_database_without_error_when_expected( + self, sqlitecloud_connection + ): driver = Driver() - + connection, _ = sqlitecloud_connection temp_file = tempfile.mkstemp(prefix="missing")[1] @@ -12,4 +16,10 @@ def test_download_missing_database_without_error_when_expected(self, sqlitecloud if_exists = True with open(temp_file, "wb") as fd: - driver.download_database(connection, "missing.sqlite", fd, lambda x, y, z, k: None, if_exists=if_exists) \ No newline at end of file + driver.download_database( + connection, + "missing.sqlite", + fd, + lambda x, y, z, k: None, + if_exists=if_exists, + ) diff --git a/src/tests/integration/test_pubsub.py b/src/tests/integration/test_pubsub.py index 122df55..d1ed3c8 100644 --- a/src/tests/integration/test_pubsub.py +++ b/src/tests/integration/test_pubsub.py @@ -1,5 +1,5 @@ -from time import sleep import time +from time import sleep import pytest @@ -88,7 +88,12 @@ def test_is_connected(self, sqlitecloud_connection): assert not pubsub.is_connected(connection) pubsub.create_channel(connection, channel_name, if_not_exists=True) - pubsub.listen(connection, SQCLOUD_PUBSUB_SUBJECT.CHANNEL, channel_name, lambda conn, result, data: None) + pubsub.listen( + connection, + SQCLOUD_PUBSUB_SUBJECT.CHANNEL, + channel_name, + lambda conn, result, data: None, + ) assert pubsub.is_connected(connection) @@ -143,13 +148,15 @@ def assert_callback(conn, result, data): pubsub = SqliteCloudPubSub() type = SQCLOUD_PUBSUB_SUBJECT.TABLE - new_name = "Rock"+ str(int(time.time())) + new_name = "Rock" + str(int(time.time())) pubsub.listen(connection, type, "genres", assert_callback, ["somedata"]) - client.exec_query(f"UPDATE genres SET Name = '{new_name}' WHERE GenreId = 1;", connection) + client.exec_query( + f"UPDATE genres SET Name = '{new_name}' WHERE GenreId = 1;", connection + ) # wait for callback to be called sleep(1) - assert callback_called \ No newline at end of file + assert callback_called diff --git a/src/tests/integration/test_upload.py b/src/tests/integration/test_upload.py index eef127f..e01960b 100644 --- a/src/tests/integration/test_upload.py +++ b/src/tests/integration/test_upload.py @@ -1,8 +1,7 @@ import os import uuid -import pytest -from sqlitecloud.client import SqliteCloudClient -from sqlitecloud.types import SQCloudConnect, SqliteCloudAccount + + from sqlitecloud.upload import upload_db diff --git a/src/tests/unit/test_client.py b/src/tests/unit/test_client.py index a45a652..844a456 100644 --- a/src/tests/unit/test_client.py +++ b/src/tests/unit/test_client.py @@ -1,7 +1,7 @@ from sqlitecloud.client import SqliteCloudClient -class TestClient: +class TestClient: def test_parse_connection_string_with_apikey(self): connection_string = "sqlitecloud://user:pass@host.com:8860/dbname?apikey=abc123&timeout=10&compression=true" client = SqliteCloudClient(connection_str=connection_string) @@ -13,7 +13,7 @@ def test_parse_connection_string_with_apikey(self): assert "dbname" == client.config.account.dbname assert "abc123" == client.config.account.apikey assert 10 == client.config.timeout - assert True == client.config.compression + assert client.config.compression def test_parse_connection_string_with_credentials(self): connection_string = "sqlitecloud://user:pass@host.com:8860" @@ -24,7 +24,7 @@ def test_parse_connection_string_with_credentials(self): assert "host.com" == client.config.account.hostname assert 8860 == client.config.account.port assert not client.config.account.dbname - + def test_parse_connection_string_without_credentials(self): connection_string = "sqlitecloud://host.com" client = SqliteCloudClient(connection_str=connection_string) @@ -33,27 +33,26 @@ def test_parse_connection_string_without_credentials(self): assert not client.config.account.password assert "host.com" == client.config.account.hostname - def test_parse_connection_string_with_all_parameters(self): connection_string = "sqlitecloud://host.com:8860/dbname?apikey=abc123&compression=true&zerotext=true&memory=true&create=true&non_linearizable=true&insecure=true&no_verify_certificate=true&root_certificate=rootcert&certificate=cert&certificate_key=certkey&noblob=true&maxdata=10&maxrows=11&maxrowset=12" - + client = SqliteCloudClient(connection_str=connection_string) - + assert "host.com" == client.config.account.hostname assert 8860 == client.config.account.port assert "dbname" == client.config.account.dbname assert "abc123" == client.config.account.apikey - assert True == client.config.compression - assert True == client.config.zerotext - assert True == client.config.memory - assert True == client.config.create - assert True == client.config.non_linearizable - assert True == client.config.insecure - assert True == client.config.no_verify_certificate + assert client.config.compression + assert client.config.zerotext + assert client.config.memory + assert client.config.create + assert client.config.non_linearizable + assert client.config.insecure + assert client.config.no_verify_certificate assert "rootcert" == client.config.root_certificate assert "cert" == client.config.certificate assert "certkey" == client.config.certificate_key - assert True == client.config.noblob + assert client.config.noblob assert 10 == client.config.maxdata assert 11 == client.config.maxrows - assert 12 == client.config.maxrowset \ No newline at end of file + assert 12 == client.config.maxrowset diff --git a/src/tests/unit/test_driver.py b/src/tests/unit/test_driver.py index 750b529..8e6d819 100644 --- a/src/tests/unit/test_driver.py +++ b/src/tests/unit/test_driver.py @@ -1,4 +1,5 @@ import pytest + from sqlitecloud.driver import Driver @@ -88,16 +89,3 @@ def test_parse_rowset_signature(self): assert 1 == result.version assert 1 == result.nrows assert 2 == result.ncols - - def test_parse_rowset_signature(self): - driver = Driver() - buffer = b"*35 0:1 1 2 +2 42+7 'hello':42 +5 hello" - - result = driver._internal_parse_rowset_signature(buffer) - - assert 12 == result.start - assert 35 == result.len - assert 0 == result.idx - assert 1 == result.version - assert 1 == result.nrows - assert 2 == result.ncols diff --git a/src/tests/unit/test_resultset.py b/src/tests/unit/test_resultset.py index b865e43..28b1264 100644 --- a/src/tests/unit/test_resultset.py +++ b/src/tests/unit/test_resultset.py @@ -1,4 +1,5 @@ import pytest + from sqlitecloud.resultset import SQCloudResult, SqliteCloudResultSet from sqlitecloud.types import SQCLOUD_RESULT_TYPE @@ -73,7 +74,7 @@ def test_get_value_with_rowset(self): assert "John" == result_set.get_value(0, 0) assert 24 == result_set.get_value(1, 1) - assert None == result_set.get_value(2, 2) + assert result_set.get_value(2, 2) is None def test_get_value_array(self): result = SQCloudResult(SQCLOUD_RESULT_TYPE.RESULT_ARRAY, result=[1, 2, 3]) @@ -89,7 +90,7 @@ def test_get_colname(self): assert "name" == result_set.get_name(0) assert "age" == result_set.get_name(1) - assert None == result_set.get_name(2) + assert result_set.get_name(2) is None def test_get_result_with_single_value(self): result = SQCloudResult(SQCLOUD_RESULT_TYPE.RESULT_INTEGER, result=42) From 897e7bb655f267bcb241f46e7fa3ee84824a2c99 Mon Sep 17 00:00:00 2001 From: Daniele Briggi Date: Wed, 22 May 2024 15:00:16 +0000 Subject: [PATCH 3/5] Setup pre-commit checks --- .flake8 | 2 +- .github/workflows/pypi-release.yaml | 2 +- .pre-commit-config.yaml | 32 ++++++++++++++++++++++++++ README.md | 2 +- pyproject.toml | 2 +- requirements-dev.txt | 3 ++- src/README-PYPI.md | 2 +- src/tests/integration/test_download.py | 5 +--- src/tests/integration/test_upload.py | 1 - 9 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.flake8 b/.flake8 index 53a4e5b..c0ad647 100644 --- a/.flake8 +++ b/.flake8 @@ -3,4 +3,4 @@ # W503,E501,E701: https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#w503 ignore = E203, W503, E501, E701 # align to black choice -max-line-length = 88 \ No newline at end of file +max-line-length = 88 diff --git a/.github/workflows/pypi-release.yaml b/.github/workflows/pypi-release.yaml index 0676454..60efb44 100644 --- a/.github/workflows/pypi-release.yaml +++ b/.github/workflows/pypi-release.yaml @@ -38,4 +38,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: src/dist/ - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..241304f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: detect-private-key + - id: check-merge-conflict + # Using this mirror lets us use mypyc-compiled black, which is about 2x faster + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 22.8.0 + hooks: + - id: black + # It is recommended to specify the latest version of Python + # supported by your project here, or alternatively use + # pre-commit's default_language_version, see + # https://pre-commit.com/#top_level-default_language_version + language_version: python3.6 + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + name: isort + - repo: https://github.com/PyCQA/autoflake + rev: v1.4 + hooks: + - id: autoflake + - repo: https://github.com/pycqa/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + stages: [push] diff --git a/README.md b/README.md index 7687637..393c60a 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,6 @@ for row in result: ### _Close connection_ -```python +```python client.disconnect(conn) ``` diff --git a/pyproject.toml b/pyproject.toml index 54087a8..0f87497 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,4 +3,4 @@ [tool.isort] # make the tools compatible to each other -profile = "black" \ No newline at end of file +profile = "black" diff --git a/requirements-dev.txt b/requirements-dev.txt index 73cb201..0ebd5a6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,4 +9,5 @@ pylint==2.13.9 mypy==0.971 flake8==5.0.4 isort==5.10.1 -autoflake==1.4 \ No newline at end of file +autoflake==1.4 +pre-commit==2.17.0 diff --git a/src/README-PYPI.md b/src/README-PYPI.md index 825c4e0..ea6338f 100644 --- a/src/README-PYPI.md +++ b/src/README-PYPI.md @@ -57,6 +57,6 @@ for row in result: ### _Close connection_ -```python +```python client.disconnect(conn) ``` diff --git a/src/tests/integration/test_download.py b/src/tests/integration/test_download.py index 2f0d0a7..e40825c 100644 --- a/src/tests/integration/test_download.py +++ b/src/tests/integration/test_download.py @@ -4,10 +4,7 @@ import pytest from sqlitecloud import download -from sqlitecloud.types import ( - SQCLOUD_ERRCODE, - SQCloudException, -) +from sqlitecloud.types import SQCLOUD_ERRCODE, SQCloudException class TestDownload: diff --git a/src/tests/integration/test_upload.py b/src/tests/integration/test_upload.py index e01960b..08a0bf8 100644 --- a/src/tests/integration/test_upload.py +++ b/src/tests/integration/test_upload.py @@ -1,7 +1,6 @@ import os import uuid - from sqlitecloud.upload import upload_db From 22724e4f592bf4115af0a8b3574be3dca0d7139d Mon Sep 17 00:00:00 2001 From: Daniele Briggi Date: Thu, 23 May 2024 12:55:55 +0000 Subject: [PATCH 4/5] Fix ci/cd workflow Use local installation for pre-commit Add Bandit tool for security checks --- .github/workflows/deploy.yaml | 11 +- .pre-commit-config.yaml | 36 ++- README.md | 1 + bandit-baseline.json | 338 +++++++++++++++++++++++++++ bandit.yaml | 2 + requirements-dev.txt | 3 +- src/README-PYPI.md | 1 + src/sqlitecloud/client.py | 5 +- src/sqlitecloud/conn_info.py | 7 - src/sqlitecloud/driver.py | 2 - src/sqlitecloud/types.py | 4 +- src/tests/conftest.py | 2 +- src/tests/integration/test_client.py | 10 +- src/tests/integration/test_pubsub.py | 28 ++- 14 files changed, 406 insertions(+), 44 deletions(-) create mode 100644 bandit-baseline.json create mode 100644 bandit.yaml delete mode 100644 src/sqlitecloud/conn_info.py diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 1e234d0..b499093 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -10,7 +10,7 @@ on: jobs: tests: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: @@ -32,16 +32,15 @@ jobs: isort --diff --check-only src - name: Tests env: - SQLITE_CONNECTION_STRING: ${{ secrets.SQLITE_CONNECTION_STRING }} + SQLITE_CONNECTION_STRING: ${{ vars.SQLITE_CONNECTION_STRING }} SQLITE_USER: ${{ secrets.SQLITE_USER }} SQLITE_PASSWORD: ${{ secrets.SQLITE_PASSWORD }} SQLITE_API_KEY: ${{ secrets.SQLITE_API_KEY }} - SQLITE_HOST: ${{ secrets.SQLITE_HOST }} - SQLITE_DB: ${{ secrets.SQLITE_DB }} - SQLITE_PORT: ${{ secrets.SQLITE_PORT }} + SQLITE_HOST: ${{ vars.SQLITE_HOST }} + SQLITE_DB: ${{ vars.SQLITE_DB }} + SQLITE_PORT: ${{ vars.SQLITE_PORT }} run: | pytest -v src/tests - release: if: ${{ github.ref == 'refs/heads/main' }} needs: tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 241304f..d6f5c64 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,6 @@ +# +# Keep these versions requirements.txt versions aligned. +# repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 @@ -11,10 +14,10 @@ repos: rev: 22.8.0 hooks: - id: black - # It is recommended to specify the latest version of Python - # supported by your project here, or alternatively use - # pre-commit's default_language_version, see - # https://pre-commit.com/#top_level-default_language_version + # It is recommended to specify the latest version of Python + # supported by your project here, or alternatively use + # pre-commit's default_language_version, see + # https://pre-commit.com/#top_level-default_language_version language_version: python3.6 - repo: https://github.com/pycqa/isort rev: 5.10.1 @@ -24,9 +27,30 @@ repos: - repo: https://github.com/PyCQA/autoflake rev: v1.4 hooks: - - id: autoflake + - id: autoflake + name: autoflake + args: + - "--in-place" + - "--expand-star-imports" + - "--remove-duplicate-keys" + - "--remove-unused-variables" + - "--remove-unused-variables" - repo: https://github.com/pycqa/flake8 rev: 5.0.4 hooks: - id: flake8 - stages: [push] + name: flake8 + # stages: [push] + - repo: https://github.com/PyCQA/bandit + rev: 1.7.1 + hooks: + - id: bandit + name: bandit + entry: bandit + args: + - "-b" + - "bandit-baseline.json" + - "--configfile" + - "bandit.yaml" + - "-r" + - "src" diff --git a/README.md b/README.md index 393c60a..fd37b11 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ $ pip install SqliteCloud ```python from sqlitecloud.client import SqliteCloudClient from sqlitecloud.types import SqliteCloudAccount + ``` ### _Init a connection_ diff --git a/bandit-baseline.json b/bandit-baseline.json new file mode 100644 index 0000000..b53ebef --- /dev/null +++ b/bandit-baseline.json @@ -0,0 +1,338 @@ +{ + "errors": [], + "generated_at": "2024-05-23T09:59:11Z", + "metrics": { + "_totals": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 2.0, + "CONFIDENCE.MEDIUM": 1.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 1.0, + "SEVERITY.MEDIUM": 2.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 2096, + "nosec": 0 + }, + "src/setup.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 39, + "nosec": 0 + }, + "src/sqlitecloud/__init__.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 1, + "nosec": 0 + }, + "src/sqlitecloud/client.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 119, + "nosec": 0 + }, + "src/sqlitecloud/conn_info.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 6, + "nosec": 0 + }, + "src/sqlitecloud/download.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 32, + "nosec": 0 + }, + "src/sqlitecloud/driver.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 727, + "nosec": 0 + }, + "src/sqlitecloud/pubsub.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 54, + "nosec": 0 + }, + "src/sqlitecloud/resultset.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 62, + "nosec": 0 + }, + "src/sqlitecloud/types.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 1.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 1.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 143, + "nosec": 0 + }, + "src/sqlitecloud/upload.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 51, + "nosec": 0 + }, + "src/tests/__init__.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 0, + "nosec": 0 + }, + "src/tests/conftest.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 22, + "nosec": 0 + }, + "src/tests/integration/__init__.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 0, + "nosec": 0 + }, + "src/tests/integration/test_client.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 471, + "nosec": 0 + }, + "src/tests/integration/test_download.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 21, + "nosec": 0 + }, + "src/tests/integration/test_driver.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 18, + "nosec": 0 + }, + "src/tests/integration/test_pubsub.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 1.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 1.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 109, + "nosec": 0 + }, + "src/tests/integration/test_upload.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 1.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 1.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 20, + "nosec": 0 + }, + "src/tests/unit/test_client.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 48, + "nosec": 0 + }, + "src/tests/unit/test_driver.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 76, + "nosec": 0 + }, + "src/tests/unit/test_resultset.py": { + "CONFIDENCE.HIGH": 0.0, + "CONFIDENCE.LOW": 0.0, + "CONFIDENCE.MEDIUM": 0.0, + "CONFIDENCE.UNDEFINED": 0.0, + "SEVERITY.HIGH": 0.0, + "SEVERITY.LOW": 0.0, + "SEVERITY.MEDIUM": 0.0, + "SEVERITY.UNDEFINED": 0.0, + "loc": 77, + "nosec": 0 + } + }, + "results": [ + { + "code": "95 class SqliteCloudAccount:\n96 def __init__(\n97 self,\n98 username: Optional[str] = \"\",\n99 password: Optional[str] = \"\",\n100 hostname: Optional[str] = \"\",\n101 dbname: Optional[str] = \"\",\n102 port: Optional[int] = SQCLOUD_DEFAULT.PORT.value,\n103 apikey: Optional[str] = \"\",\n104 ) -> None:\n105 # User name is required unless connectionstring is provided\n106 self.username = username\n107 # Password is required unless connection string is provided\n108 self.password = password\n109 # Password is hashed\n110 self.password_hashed = False\n111 # API key instead of username and password\n112 self.apikey = apikey\n113 # Name of database to open\n114 self.dbname = dbname\n115 # Like mynode.sqlitecloud.io\n116 self.hostname = hostname\n117 self.port = port\n118 \n", + "col_offset": 4, + "filename": "src/sqlitecloud/types.py", + "issue_confidence": "MEDIUM", + "issue_severity": "LOW", + "issue_text": "Possible hardcoded password: ''", + "line_number": 96, + "line_range": [ + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b107_hardcoded_password_default.html", + "test_id": "B107", + "test_name": "hardcoded_password_default" + }, + { + "code": "155 client.exec_query(\n156 f\"UPDATE genres SET Name = '{new_name}' WHERE GenreId = 1;\", connection\n157 )\n", + "col_offset": 12, + "filename": "src/tests/integration/test_pubsub.py", + "issue_confidence": "LOW", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 156, + "line_range": [ + 156 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + }, + { + "code": "18 rowset = client.exec_query(\n19 f\"USE DATABASE {dbname}; SELECT * FROM contacts\", connection\n20 )\n", + "col_offset": 16, + "filename": "src/tests/integration/test_upload.py", + "issue_confidence": "LOW", + "issue_severity": "MEDIUM", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "line_number": 19, + "line_range": [ + 19 + ], + "more_info": "https://bandit.readthedocs.io/en/latest/plugins/b608_hardcoded_sql_expressions.html", + "test_id": "B608", + "test_name": "hardcoded_sql_expressions" + } + ] +} diff --git a/bandit.yaml b/bandit.yaml new file mode 100644 index 0000000..b99641c --- /dev/null +++ b/bandit.yaml @@ -0,0 +1,2 @@ +# https://bndit.readthedocs.io/en/latest/config.html +skips: ['B101'] diff --git a/requirements-dev.txt b/requirements-dev.txt index 0ebd5a6..7139e51 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ +# Latest versions that supports Py 3.6 typing-extensions==4.1.1 pytest==7.0.1 pytest-mock==3.6.1 @@ -6,8 +7,8 @@ lz4==3.1.10 bump2version==1.0.1 black==22.8.0 pylint==2.13.9 -mypy==0.971 flake8==5.0.4 isort==5.10.1 autoflake==1.4 pre-commit==2.17.0 +bandit==1.7.1 diff --git a/src/README-PYPI.md b/src/README-PYPI.md index ea6338f..0ed4bd4 100644 --- a/src/README-PYPI.md +++ b/src/README-PYPI.md @@ -19,6 +19,7 @@ $ pip install SqliteCloud ```python from sqlitecloud.client import SqliteCloudClient from sqlitecloud.types import SqliteCloudAccount + ``` ### _Init a connection_ diff --git a/src/sqlitecloud/client.py b/src/sqlitecloud/client.py index ba201dd..cff4824 100644 --- a/src/sqlitecloud/client.py +++ b/src/sqlitecloud/client.py @@ -7,6 +7,7 @@ from sqlitecloud.driver import Driver from sqlitecloud.resultset import SqliteCloudResultSet from sqlitecloud.types import ( + SQCLOUD_DEFAULT, SQCloudConfig, SQCloudConnect, SQCloudException, @@ -19,8 +20,6 @@ class SqliteCloudClient: Client to interact with Sqlite Cloud """ - SQLITE_DEFAULT_PORT = 8860 - def __init__( self, cloud_account: Optional[SqliteCloudAccount] = None, @@ -148,7 +147,7 @@ def _parse_connection_string(self, connection_string) -> SQCloudConfig: config.account.hostname = params.hostname config.account.port = ( - int(params.port) if params.port else self.SQLITE_DEFAULT_PORT + int(params.port) if params.port else SQCLOUD_DEFAULT.PORT.value ) return config diff --git a/src/sqlitecloud/conn_info.py b/src/sqlitecloud/conn_info.py deleted file mode 100644 index 4c67138..0000000 --- a/src/sqlitecloud/conn_info.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -user = os.environ.get("SQLITE_USER", "ADMIN") -password = os.environ.get("SQLITE_PASSWORD", "WGf646dXUk") -host = os.environ.get("SQLITE_HOST", "wpkvc7n-m.sqlite.cloud") -db_name = os.environ.get("SQLITE_DB", "people") -port = os.environ.get("SQLITE_PORT", 8860) diff --git a/src/sqlitecloud/driver.py b/src/sqlitecloud/driver.py index efa207c..15ae0b8 100644 --- a/src/sqlitecloud/driver.py +++ b/src/sqlitecloud/driver.py @@ -67,8 +67,6 @@ def disconnect(self, conn: SQCloudConnect, only_main_socket: bool = False) -> No conn.socket.close() if not only_main_socket and conn.pubsub_socket: conn.pubsub_socket.close() - except Exception: - pass finally: conn.socket = None if not only_main_socket: diff --git a/src/sqlitecloud/types.py b/src/sqlitecloud/types.py index 468bcfc..242d085 100644 --- a/src/sqlitecloud/types.py +++ b/src/sqlitecloud/types.py @@ -97,9 +97,9 @@ def __init__( self, username: Optional[str] = "", password: Optional[str] = "", - hostname: Optional[str] = "", + hostname: str = "", dbname: Optional[str] = "", - port: Optional[int] = SQCLOUD_DEFAULT.PORT.value, + port: int = SQCLOUD_DEFAULT.PORT.value, apikey: Optional[str] = "", ) -> None: # User name is required unless connectionstring is provided diff --git a/src/tests/conftest.py b/src/tests/conftest.py index e7c5bff..da4f982 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -19,7 +19,7 @@ def sqlitecloud_connection(): account.password = os.getenv("SQLITE_PASSWORD") account.dbname = os.getenv("SQLITE_DB") account.hostname = os.getenv("SQLITE_HOST") - account.port = 8860 + account.port = int(os.getenv("SQLITE_PORT")) client = SqliteCloudClient(cloud_account=account) diff --git a/src/tests/integration/test_client.py b/src/tests/integration/test_client.py index 4e143da..273bb19 100644 --- a/src/tests/integration/test_client.py +++ b/src/tests/integration/test_client.py @@ -28,7 +28,7 @@ def test_connection_with_credentials(self): account.password = os.getenv("SQLITE_PASSWORD") account.dbname = os.getenv("SQLITE_DB") account.hostname = os.getenv("SQLITE_HOST") - account.port = 8860 + account.port = int(os.getenv("SQLITE_PORT")) client = SqliteCloudClient(cloud_account=account) conn = client.open_connection() @@ -40,7 +40,7 @@ def test_connection_with_apikey(self): account = SqliteCloudAccount() account.username = os.getenv("SQLITE_API_KEY") account.hostname = os.getenv("SQLITE_HOST") - account.port = 8860 + account.port = int(os.getenv("SQLITE_PORT")) client = SqliteCloudClient(cloud_account=account) conn = client.open_connection() @@ -52,7 +52,7 @@ def test_connection_without_credentials_and_apikey(self): account = SqliteCloudAccount() account.dbname = os.getenv("SQLITE_DB") account.hostname = os.getenv("SQLITE_HOST") - account.port = 8860 + account.port = int(os.getenv("SQLITE_PORT")) client = SqliteCloudClient(cloud_account=account) @@ -83,7 +83,7 @@ def test_is_connected(self): account = SqliteCloudAccount() account.username = os.getenv("SQLITE_API_KEY") account.hostname = os.getenv("SQLITE_HOST") - account.port = 8860 + account.port = int(os.getenv("SQLITE_PORT")) client = SqliteCloudClient(cloud_account=account) @@ -97,7 +97,7 @@ def test_disconnect(self): account = SqliteCloudAccount() account.username = os.getenv("SQLITE_API_KEY") account.hostname = os.getenv("SQLITE_HOST") - account.port = 8860 + account.port = int(os.getenv("SQLITE_PORT")) client = SqliteCloudClient(cloud_account=account) diff --git a/src/tests/integration/test_pubsub.py b/src/tests/integration/test_pubsub.py index d1ed3c8..4b6cba5 100644 --- a/src/tests/integration/test_pubsub.py +++ b/src/tests/integration/test_pubsub.py @@ -1,5 +1,5 @@ -import time -from time import sleep +import threading +import uuid import pytest @@ -18,6 +18,7 @@ def test_listen_channel_and_notify(self, sqlitecloud_connection): connection, _ = sqlitecloud_connection callback_called = False + flag = threading.Event() def assert_callback(conn, result, data): nonlocal callback_called @@ -26,10 +27,11 @@ def assert_callback(conn, result, data): assert result.tag == SQCLOUD_RESULT_TYPE.RESULT_JSON assert data == ["somedata"] callback_called = True + flag.set() pubsub = SqliteCloudPubSub() type = SQCLOUD_PUBSUB_SUBJECT.CHANNEL - channel = "channel" + str(int(time.time())) + channel = "channel" + str(uuid.uuid4()) pubsub.create_channel(connection, channel) pubsub.listen(connection, type, channel, assert_callback, ["somedata"]) @@ -37,7 +39,7 @@ def assert_callback(conn, result, data): pubsub.notify_channel(connection, channel, "somedata2") # wait for callback to be called - sleep(1) + flag.wait(30) assert callback_called @@ -46,7 +48,7 @@ def test_unlisten_channel(self, sqlitecloud_connection): pubsub = SqliteCloudPubSub() type = SQCLOUD_PUBSUB_SUBJECT.CHANNEL - channel_name = "channel" + str(int(time.time())) + channel_name = "channel" + str(uuid.uuid4()) pubsub.create_channel(connection, channel_name) pubsub.listen(connection, type, channel_name, lambda conn, result, data: None) @@ -66,7 +68,7 @@ def test_create_channel_to_fail_if_exists(self, sqlitecloud_connection): connection, _ = sqlitecloud_connection pubsub = SqliteCloudPubSub() - channel_name = "channel" + str(int(time.time())) + channel_name = "channel" + str(uuid.uuid4()) pubsub.create_channel(connection, channel_name, if_not_exists=True) @@ -83,7 +85,7 @@ def test_is_connected(self, sqlitecloud_connection): connection, _ = sqlitecloud_connection pubsub = SqliteCloudPubSub() - channel_name = "channel" + str(int(time.time())) + channel_name = "channel" + str(uuid.uuid4()) assert not pubsub.is_connected(connection) @@ -101,6 +103,7 @@ def test_set_pubsub_only(self, sqlitecloud_connection): connection, client = sqlitecloud_connection callback_called = False + flag = threading.Event() def assert_callback(conn, result, data): nonlocal callback_called @@ -108,10 +111,11 @@ def assert_callback(conn, result, data): if isinstance(result, SqliteCloudResultSet): assert result.get_result() is not None callback_called = True + flag.set() pubsub = SqliteCloudPubSub() type = SQCLOUD_PUBSUB_SUBJECT.CHANNEL - channel = "channel" + str(int(time.time())) + channel = "channel" + str(uuid.uuid4()) pubsub.create_channel(connection, channel, if_not_exists=True) pubsub.listen(connection, type, channel, assert_callback) @@ -126,7 +130,7 @@ def assert_callback(conn, result, data): pubsub2.notify_channel(connection2, channel, "message-in-a-bottle") # wait for callback to be called - sleep(2) + flag.wait(30) assert callback_called @@ -136,6 +140,7 @@ def test_listen_table_for_update(self, sqlitecloud_connection): connection, client = sqlitecloud_connection callback_called = False + flag = threading.Event() def assert_callback(conn, result, data): nonlocal callback_called @@ -145,10 +150,11 @@ def assert_callback(conn, result, data): assert new_name in result.get_result() assert data == ["somedata"] callback_called = True + flag.set() pubsub = SqliteCloudPubSub() type = SQCLOUD_PUBSUB_SUBJECT.TABLE - new_name = "Rock" + str(int(time.time())) + new_name = "Rock" + str(uuid.uuid4()) pubsub.listen(connection, type, "genres", assert_callback, ["somedata"]) @@ -157,6 +163,6 @@ def assert_callback(conn, result, data): ) # wait for callback to be called - sleep(1) + flag.wait(30) assert callback_called From b392c49d2c0da6aee6477cdab03c4ebe633e2481 Mon Sep 17 00:00:00 2001 From: Daniele Briggi Date: Fri, 24 May 2024 10:46:35 +0000 Subject: [PATCH 5/5] Workflow test, release, badges and cleanup --- .github/workflows/deploy.yaml | 7 +++++-- .gitignore | 1 + README.md | 31 +++++++++++++++++++++++----- requirements-dev.txt | 2 ++ samples.ipynb | 3 +-- src/sqlitecloud/driver.py | 6 +++--- src/tests/integration/test_pubsub.py | 7 +++++-- 7 files changed, 43 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index b499093..59622dd 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -5,7 +5,6 @@ name: Test and Release on: push: - pull_request: workflow_dispatch: jobs: @@ -40,7 +39,11 @@ jobs: SQLITE_DB: ${{ vars.SQLITE_DB }} SQLITE_PORT: ${{ vars.SQLITE_PORT }} run: | - pytest -v src/tests + pytest --cov -v src/tests + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} release: if: ${{ github.ref == 'refs/heads/main' }} needs: tests diff --git a/.gitignore b/.gitignore index fde26c3..9ee7dce 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ main.dSYM/ .env *.pyc *.pyo +.coverage .DS_Store diff --git a/README.md b/README.md index fd37b11..609995c 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,28 @@ # Python SDK for SQLite Cloud -![Build Status](https://github.com/sqlitecloud/python/actions/workflows/deploy.yaml/badge.svg "Build Status") ![Jupyter Notebook](https://img.shields.io/badge/jupyter-%23FA0F00.svg?style=plastic&logo=jupyter&logoColor=white) +

+ SQLite Cloud logo +

-SQLiteCloud is a powerful Python package that allows you to interact with the SQLite Cloud backend server seamlessly. It provides methods for various database operations. This package is designed to simplify database operations in Python applications, making it easier than ever to work with SQLite Cloud. +![Build Status](https://github.com/sqlitecloud/python/actions/workflows/deploy.yaml/badge.svg "Build Status") +[![codecov](https://codecov.io/github/sqlitecloud/python/graph/badge.svg?token=38G6FGOWKP)](https://codecov.io/github/sqlitecloud/python) +![PyPI - Version](https://img.shields.io/pypi/v/sqlitecloud?link=https%3A%2F%2Fpypi.org%2Fproject%2FSqliteCloud%2F) +![PyPI - Downloads](https://img.shields.io/pypi/dm/sqlitecloud?link=https%3A%2F%2Fpypi.org%2Fproject%2FSqliteCloud%2F) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlitecloud?link=https%3A%2F%2Fpypi.org%2Fproject%2FSqliteCloud%2F) +[SQLiteCloud](https://sqlitecloud.io) is a powerful Python package that allows you to interact with the SQLite Cloud backend server seamlessly. It provides methods for various database operations. This package is designed to simplify database operations in Python applications, making it easier than ever to work with SQLite Cloud. + +- Site: [https://sqlitecloud.io](https://sqlitecloud.io/developers) +- Documentation: https://..._coming!_ +- Source: [https://github.com/sqlitecloud/python](https://github.com/sqlitecloud/python) + ## Installation You can install SqliteCloud Package using Python Package Index (PYPI): ```bash -$ pip install SqliteCloud +$ pip install sqlitecloud ``` ## Usage @@ -19,7 +31,6 @@ $ pip install SqliteCloud ```python from sqlitecloud.client import SqliteCloudClient from sqlitecloud.types import SqliteCloudAccount - ``` ### _Init a connection_ @@ -35,7 +46,7 @@ conn = client.open_connection() #### _Using string configuration_ ```python -account = SqliteCloudAccount("sqlitecloud://user:pass@host.com:port/dbname?timeout=10&key2=value2&key3=value3") +account = SqliteCloudAccount("sqlitecloud://user:pass@host.com:port/dbname?apikey=myapikey") client = SqliteCloudClient(cloud_account=account) conn = client.open_connection() ``` @@ -56,6 +67,16 @@ for row in result: print(row) ``` +### _Specific value_ +```python +result.get_value(0, 0) +``` + +### _Column name_ +```python +result.get_name(0) +``` + ### _Close connection_ ```python diff --git a/requirements-dev.txt b/requirements-dev.txt index 7139e51..1f30ce7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,6 +2,8 @@ typing-extensions==4.1.1 pytest==7.0.1 pytest-mock==3.6.1 +pytest-cov==4.0.0 +coverage==6.2 python-dotenv==0.20.0 lz4==3.1.10 bump2version==1.0.1 diff --git a/samples.ipynb b/samples.ipynb index 17ce3da..7f9d9e7 100644 --- a/samples.ipynb +++ b/samples.ipynb @@ -21,7 +21,6 @@ "\n", "sys.path.append('/workspaces/python/src')\n", "\n", - "from sqlitecloud.conn_info import user, password, host, db_name, port\n", "from sqlitecloud.client import SqliteCloudClient\n", "from sqlitecloud.types import SqliteCloudAccount" ] @@ -41,7 +40,7 @@ "metadata": {}, "outputs": [], "source": [ - "account = SqliteCloudAccount(user, password, host, db_name, int(port))\n", + "account = SqliteCloudAccount(user, password, host, db_name, 8860)\n", "client = SqliteCloudClient(cloud_account=account)\n", "conn = client.open_connection()" ] diff --git a/src/sqlitecloud/driver.py b/src/sqlitecloud/driver.py index 15ae0b8..c1843dd 100644 --- a/src/sqlitecloud/driver.py +++ b/src/sqlitecloud/driver.py @@ -186,12 +186,12 @@ def _internal_pubsub_thread(self, connection: SQCloudConnect) -> None: ready_to_read, _, errors = select.select( [connection.pubsub_socket], [], [] ) - # eg, no data to read - if len(ready_to_read) == 0: - continue # eg, if the socket is closed if len(errors) > 0: break + # eg, no data to read + if len(ready_to_read) == 0: + continue data = connection.pubsub_socket.recv(blen) if not data: diff --git a/src/tests/integration/test_pubsub.py b/src/tests/integration/test_pubsub.py index 4b6cba5..2b037ba 100644 --- a/src/tests/integration/test_pubsub.py +++ b/src/tests/integration/test_pubsub.py @@ -22,6 +22,7 @@ def test_listen_channel_and_notify(self, sqlitecloud_connection): def assert_callback(conn, result, data): nonlocal callback_called + nonlocal flag if isinstance(result, SqliteCloudResultSet): assert result.tag == SQCLOUD_RESULT_TYPE.RESULT_JSON @@ -107,6 +108,7 @@ def test_set_pubsub_only(self, sqlitecloud_connection): def assert_callback(conn, result, data): nonlocal callback_called + nonlocal flag if isinstance(result, SqliteCloudResultSet): assert result.get_result() is not None @@ -129,13 +131,13 @@ def assert_callback(conn, result, data): pubsub2 = SqliteCloudPubSub() pubsub2.notify_channel(connection2, channel, "message-in-a-bottle") + client.disconnect(connection2) + # wait for callback to be called flag.wait(30) assert callback_called - client.disconnect(connection2) - def test_listen_table_for_update(self, sqlitecloud_connection): connection, client = sqlitecloud_connection @@ -144,6 +146,7 @@ def test_listen_table_for_update(self, sqlitecloud_connection): def assert_callback(conn, result, data): nonlocal callback_called + nonlocal flag if isinstance(result, SqliteCloudResultSet): assert result.tag == SQCLOUD_RESULT_TYPE.RESULT_JSON