diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml new file mode 100644 index 0000000..a797f88 --- /dev/null +++ b/.github/workflows/python-tests.yml @@ -0,0 +1,34 @@ +name: Python Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ['3.10'] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Run tests + run: | + pytest test/fission_tests.py diff --git a/backend/harvesters/Mastodon/mharvester.py b/backend/harvesters/Mastodon/mharvester.py index 38fa6b7..c62c3f7 100644 --- a/backend/harvesters/Mastodon/mharvester.py +++ b/backend/harvesters/Mastodon/mharvester.py @@ -8,7 +8,7 @@ def parse_json(msg, analyzer): new_msg = {} - new_msg['id'] = msg['id'] + new_msg['id'] = msg.get('id') raw_date = msg.get('created_at') if raw_date: @@ -20,10 +20,10 @@ def parse_json(msg, analyzer): else: new_msg['created_at'] = None - new_msg['content'] = msg['content'] + new_msg['content'] = msg.get('content') try: - new_msg['sentiment'] = analyzer.polarity_scores(new_msg['content']).get('compound') + new_msg['sentiment'] = analyzer.polarity_scores(new_msg.get('content')).get('compound') except: new_msg['sentiment'] = 0 diff --git a/frontend/__pycache__/api.cpython-311.pyc b/frontend/__pycache__/api.cpython-311.pyc deleted file mode 100644 index a3165a7..0000000 Binary files a/frontend/__pycache__/api.cpython-311.pyc and /dev/null differ diff --git a/requirements.txt b/requirements.txt index 6e2ab18..c3d2eb0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,166 @@ tomli==2.0.1 urllib3==2.2.1 Werkzeug==3.0.3 zipp==3.18.1 +anyio==4.3.0 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asttokens==2.4.1 +async-lru==2.0.4 +attrs==23.2.0 +autopage==0.5.2 +Babel==2.15.0 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blinker==1.8.2 +blurhash==1.1.4 +branca==0.7.2 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cliff==4.6.0 +cmd2==2.4.3 +comm==0.2.2 +contourpy==1.2.1 +cryptography==42.0.5 +cycler==0.12.1 +debtcollector==3.0.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dogpile.cache==1.3.2 +elastic-transport==8.13.0 +elasticsearch8==8.13.1 +exceptiongroup==1.2.1 +executing==2.0.1 +fastjsonschema==2.19.1 +Flask==3.0.3 +folium==0.16.0 +fonttools==4.51.0 +fqdn==1.5.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +idna==3.7 +importlib_metadata==7.1.0 +iniconfig==2.0.0 +ipykernel==6.29.4 +ipython==8.24.0 +ipywidgets==8.1.2 +iso8601==2.1.0 +isoduration==20.11.0 +itsdangerous==2.2.0 +jedi==0.19.1 +Jinja2==3.1.4 +jmespath==1.0.1 +json5==0.9.25 +jsonpatch==1.33 +jsonpointer==2.4 +jsonschema==4.22.0 +jsonschema-specifications==2023.12.1 +jupyter==1.0.0 +jupyter-console==6.6.3 +jupyter-events==0.10.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.1 +jupyter_core==5.7.2 +jupyter_server==2.14.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.1.8 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.1 +jupyterlab_widgets==3.0.10 +keystoneauth1==5.6.0 +kiwisolver==1.4.5 +MarkupSafe==2.1.5 +Mastodon.py==1.8.1 +matplotlib==3.8.4 +matplotlib-inline==0.1.7 +mistune==3.0.2 +msgpack==1.0.8 +nbclient==0.10.0 +nbconvert==7.16.4 +nbformat==5.10.4 +nest-asyncio==1.6.0 +netaddr==1.2.1 +netifaces==0.11.0 +notebook==7.1.3 +notebook_shim==0.2.4 +numpy==1.26.4 +openstacksdk==3.1.0 +os-service-types==1.7.0 +osc-lib==3.0.1 +oslo.config==9.4.0 +oslo.i18n==6.3.0 +oslo.serialization==5.4.0 +oslo.utils==7.1.0 +overrides==7.7.0 +packaging==24.0 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.4 +pbr==6.0.0 +pexpect==4.9.0 +pillow==10.3.0 +platformdirs==4.2.0 +pluggy==1.5.0 +prettytable==3.10.0 +prometheus_client==0.20.0 +prompt-toolkit==3.0.43 +psutil==5.9.8 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pycparser==2.22 +Pygments==2.18.0 +pyparsing==3.1.2 +pyperclip==1.8.2 +pytest==8.2.0 +pytest-mock==3.14.0 +python-cinderclient==9.5.0 +python-dateutil==2.9.0.post0 +python-json-logger==2.0.7 +python-keystoneclient==5.4.0 +python-magic==0.4.27 +python-novaclient==18.6.0 +python-openstackclient==6.6.0 +pytz==2024.1 +PyYAML==6.0.1 +pyzmq==26.0.3 +qtconsole==5.5.2 +QtPy==2.4.1 +referencing==0.35.1 +requests==2.31.0 +requestsexceptions==1.4.0 +rfc3339-validator==0.1.4 +rfc3986==2.0.0 +rfc3986-validator==0.1.1 +rpds-py==0.18.1 +seaborn==0.13.2 +Send2Trash==1.8.3 +simplejson==3.19.2 +six==1.16.0 +sniffio==1.3.1 +soupsieve==2.5 +stack-data==0.6.3 +stevedore==5.2.0 +terminado==0.18.1 +tinycss2==1.3.0 +tomli==2.0.1 +tornado==6.4 +traitlets==5.14.3 +types-python-dateutil==2.9.0.20240316 +typing_extensions==4.11.0 +tzdata==2024.1 +uri-template==1.3.0 +urllib3==2.2.1 +vaderSentiment==3.3.2 +wcwidth==0.2.13 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.0.3 +widgetsnbextension==4.0.10 +wrapt==1.16.0 +xyzservices==2024.4.0 +zipp==3.18.1 diff --git a/test/fission_tests.py b/test/fission_tests.py index 630d165..63cec2f 100644 --- a/test/fission_tests.py +++ b/test/fission_tests.py @@ -1,74 +1,106 @@ import pytest import requests +from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer + from backend.elastic_client_provider import get_client from backend.harvesters.BOM.addobservations import call_bom from backend.harvesters.Mastodon.mharvester import parse_json, generate_docs +from flask import Flask +@pytest.fixture +def app(): + app = Flask(__name__) + return app -def test_call_bom_success(mocker): - # Setting up the mock for requests.get to return a successful response - mock_response = mocker.MagicMock(status_code=200) - mock_response.json.return_value = {"data": "example"} - mocker.patch('requests.get', return_value=mock_response) +def test_call_bom_success(app,mocker): + with app.app_context(): + # Setting up the mock for requests.get to return a successful response + mock_response = mocker.MagicMock(status_code=200) + mock_response.json.return_value = {"data": "example"} + mocker.patch('requests.get', return_value=mock_response) + mocker.patch('flask.current_app.logger.info', return_value=None) + # Calling the function + station_name, result = call_bom("TestStation", "http://example.com") - # Calling the function - station_name, result = call_bom("TestStation", "http://example.com") + # Assertions to check the expected output + assert station_name == "TestStation" + assert result == {"data": "example"}, "JSON response should be returned on success" - # Assertions to check the expected output - assert station_name == "TestStation" - assert result == {"data": "example"}, "JSON response should be returned on success" +def test_call_bom_failure(app, mocker): + with app.app_context(): + # Setting up the mock for requests.get to return a non-successful response + mock_response = mocker.MagicMock(status_code=404) # Example of a failed status code + mocker.patch('requests.get', return_value=mock_response) + mocker.patch('flask.current_app.logger.info', return_value=None) + + # Calling the function + station_name, result = call_bom("TestStation", "http://example.com") -def test_call_bom_failure(mocker): - # Setting up the mock for requests.get to return a non-successful response - mock_response = mocker.MagicMock(status_code=404) # Example of a failed status code - mocker.patch('requests.get', return_value=mock_response) + # Assertions to check the expected output + assert station_name == "TestStation" + assert result is None, "None should be returned if the response is not 200" - # Calling the function - station_name, result = call_bom("TestStation", "http://example.com") - # Assertions to check the expected output - assert station_name == "TestStation" - assert result is None, "None should be returned if the response is not 200" +# test elastic search client +def test_get_client(app, mocker): + with app.app_context(): + mocker.patch('flask.current_app.logger.info', return_value=None) + assert str(type(get_client())) == "" -# test elastic search client -def test_get_client(): - assert str(type(get_client())) == "" - - -def test_parse_json_normal_input(): - # Input with a typical list of message dictionaries - msgs = [ - {'content': 'Hello World', 'created_at': '2021-07-07T12:00:00.123456Z', 'id': 1}, - {'content': 'Test Message', 'created_at': '2021-07-07T13:00:00.123456Z', 'id': 2} - ] - # Expected output should trim the datetime and include all keys - expected = [ - {'content': 'Hello World', 'created_at': '2021-07-07T12:00:00', 'id': 1}, - {'content': 'Test Message', 'created_at': '2021-07-07T13:00:00', 'id': 2} - ] - assert parse_json(msgs) == expected - -def test_parse_json_empty_input(): - # Testing the function with an empty list - assert parse_json([]) == [] - -def test_parse_json_missing_keys(): - # Input where some messages are missing one or more keys - msgs = [ - {'content': 'Incomplete', 'id': 3}, # missing 'created_at' - {'created_at': '2021-07-07T14:00:00.123456Z', 'id': 4} # missing 'content' - ] - # Expected output handles missing keys gracefully - expected = [ - {'content': 'Incomplete', 'created_at': None, 'id': 3}, - {'content': None, 'created_at': '2021-07-07T14:00:00', 'id': 4} - ] - result = parse_json(msgs) - # Custom assertion to handle possible None values in dictionary - for res, exp in zip(result, expected): - for key in ['content', 'created_at', 'id']: - assert res.get(key) == exp.get(key) - -# You can also include additional test cases for malformed input, etc. +def test_parse_json_normal_input(app, mocker): + with app.app_context(): + # Input with a typical list of message dictionaries + msgs = {'content': 'Hello World', 'created_at': '2021-07-07T12:00:00.123456Z', 'id': 1, 'sentiment':None} + # Expected output should trim the datetime and include all keys + expected = {'id': 1, 'created_at': '2021-07-07T12:00:00', 'date': '07/07/21', 'content': 'Hello World', 'sentiment': 1.0} + mocker.patch('flask.current_app.logger.info', return_value=None) + + mock_analyzer = mocker.Mock(spec=SentimentIntensityAnalyzer) + + # Define the return value of the polarity_scores method + mock_analyzer.polarity_scores.return_value = { + 'neg': 0.0, + 'neu': 0.0, + 'pos': 1.0, + 'compound': 1.0 + } + assert parse_json(msgs, mock_analyzer) == expected + +def test_parse_json_empty_input(app, mocker): + with app.app_context(): + mocker.patch('flask.current_app.logger.info', return_value=None) + # Define the return value of the polarity_scores method + mock_analyzer = mocker.Mock(spec=SentimentIntensityAnalyzer) + + mock_analyzer.polarity_scores.return_value = { + None + } + # Testing the function with an empty list + + assert parse_json({}, mock_analyzer) == {'id': None, 'created_at': None, 'content': None, 'sentiment': 0} + +def test_parse_json_missing_keys(app, mocker): + with app.app_context(): + mocker.patch('flask.current_app.logger.info', return_value=None) + # Input where some messages are missing one or more keys + msgs = {'content': 'Incomplete', 'id': 3} # missing 'created_at' + + # Expected output handles missing keys gracefully + expected = {'content': 'Incomplete', 'created_at': None, 'id': 3} + + + mock_analyzer = mocker.Mock(spec=SentimentIntensityAnalyzer) + + # Define the return value of the polarity_scores method + mock_analyzer.polarity_scores.return_value = { + 'neg': 0.0, + 'neu': 0.0, + 'pos': 1.0, + 'compound': 1.0 + } + # Custom assertion to handle possible None values in dictionary + assert parse_json(msgs, mock_analyzer) == {'id': 3, 'created_at': None, 'content': 'Incomplete', 'sentiment': 1.0} + + # You can also include additional test cases for malformed input, etc.