Skip to content

Commit

Permalink
Merge pull request #720 from robbrad/dev_cont
Browse files Browse the repository at this point in the history
feat: #708 Adding HA to the dev container for debugging
  • Loading branch information
robbrad authored May 17, 2024
2 parents dd5e828 + 3531611 commit e74d612
Show file tree
Hide file tree
Showing 9 changed files with 242 additions and 139 deletions.
15 changes: 15 additions & 0 deletions .devcontainer/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,21 @@ services:
- selenium-hub
command: sleep infinity

homeassistant:
container_name: homeassistant
image: "ghcr.io/home-assistant/home-assistant:stable"
volumes:
- .ha_config:/config:rw
- ../custom_components:/config/custom_components
- /etc/localtime:/etc/localtime:ro
- /run/dbus:/run/dbus:ro
restart: unless-stopped
privileged: true
networks:
- devnet
ports:
- "8124:8123/tcp"

chrome:
image: selenium/node-chrome:4.20.0-20240505
shm_size: 2gb
Expand Down
183 changes: 91 additions & 92 deletions .github/workflows/behave.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,14 @@ on:
paths-ignore:
- "wiki/**"
- "**/**.md"
- "**.md"
- "uk_bin_collection_api_server/**"
branches: [ "master" ]
pull_request:
# Trigger unless only the wiki directory changed
paths-ignore:
- "wiki/**"
- "**/**.md"
- "**.md"
- "uk_bin_collection_api_server/**"
# The branches below must be a subset of the branches above
branches: [ "master" ]
schedule:
- cron: '0 0 * * *'
Expand All @@ -34,113 +31,115 @@ jobs:
selenium:
image: selenium/standalone-chrome:latest
options: --shm-size=2gb --name selenium --hostname selenium
ports:
ports:
- 4444:4444
steps:
- uses: actions/checkout@v4

- name: Install poetry
run: pipx install poetry==${{ matrix.poetry-version }}

- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'poetry'

- name: Install
run: make install

- name: Get all councils files that have changed
id: changed-council-files
uses: tj-actions/changed-files@v44
with:
files: |
uk_bin_collection/uk_bin_collection/councils/**.py
- name: Get all councils
env:
ALL_CHANGED_FILES: ${{ steps.changed-council-files.outputs.all_changed_files }}
run: |
IFS=' ' read -ra FILES <<< "$ALL_CHANGED_FILES"
COUNCIL_TESTS=""
for file in "${FILES[@]}"; do
FILENAME=$(basename "$file" .py)
if [ -z "$COUNCIL_TESTS" ]; then
COUNCIL_TESTS="$FILENAME"
else
COUNCIL_TESTS="$COUNCIL_TESTS or $FILENAME"
fi
done
echo "COUNCIL_TESTS=${COUNCIL_TESTS}" >> $GITHUB_ENV
- name: run integration-tests
env:
HEADLESS: True
run: make matrix=${{ matrix.python-version }} councils="${{ env.COUNCIL_TESTS }}" integration-tests
continue-on-error: true

- name: run unit-tests
run: make unit-tests
continue-on-error: true

- name: Upload Test Coverage to Codecov
uses: codecov/codecov-action@v4
with:
gcov_ignore: uk_bin_collection/tests/**

- name: Get Allure history
uses: actions/checkout@v4
if: github.ref == 'refs/heads/master'
continue-on-error: true
with:
ref: gh-pages
path: gh-pages

- name: Allure Report action from marketplace
uses: simple-elf/allure-report-action@master
if: github.ref == 'refs/heads/master'
#id: allure-report
with:
allure_results: build/${{ matrix.python-version }}/allure-results
#gh_pages: gh-pages
#allure_report: allure-report
subfolder: ${{ matrix.python-version }}
allure_history: allure-history
keep_reports: 20

- name: Tar Report
if: github.ref == 'refs/heads/master'
run: tar -cvf allure_history_${{ matrix.python-version }}.tar allure-history/${{ matrix.python-version }}
- name: upload artifact
uses: actions/upload-artifact@v4
if: github.ref == 'refs/heads/master'
with:
name: allure_history_${{ matrix.python-version }}
path: allure_history_${{ matrix.python-version }}.tar
- uses: actions/checkout@v4

- name: Install poetry
run: pipx install poetry==${{ matrix.poetry-version }}

- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'poetry'

- name: Install
run: make install

- name: Lint JSON
run: jq empty uk_bin_collection/tests/input.json

- name: Check Parity of Councils / input.json / Feature file
run: make parity-check branch="${{ github.event.pull_request.head.ref }}"

- name: Get all councils files that have changed
id: changed-council-files
uses: tj-actions/changed-files@v44
with:
files: |
uk_bin_collection/uk_bin_collection/councils/**.py
- name: Get all councils
env:
ALL_CHANGED_FILES: ${{ steps.changed-council-files.outputs.all_changed_files }}
run: |
IFS=' ' read -ra FILES <<< "$ALL_CHANGED_FILES"
COUNCIL_TESTS=""
for file in "${FILES[@]}"; do
FILENAME=$(basename "$file" .py)
if [ -z "$COUNCIL_TESTS" ]; then
COUNCIL_TESTS="$FILENAME"
else
COUNCIL_TESTS="$COUNCIL_TESTS or $FILENAME"
fi
done
echo "COUNCIL_TESTS=${COUNCIL_TESTS}" >> $GITHUB_ENV
- name: Run integration tests
env:
HEADLESS: True
run: make matrix=${{ matrix.python-version }} councils="${{ env.COUNCIL_TESTS }}" integration-tests
continue-on-error: true

- name: Run unit tests
run: make unit-tests
continue-on-error: true

- name: Upload test coverage to Codecov
uses: codecov/codecov-action@v4
with:
gcov_ignore: uk_bin_collection/tests/**

- name: Get Allure history
uses: actions/checkout@v4
if: github.ref == 'refs/heads/master'
continue-on-error: true
with:
ref: gh-pages
path: gh-pages

- name: Allure report action from marketplace
uses: simple-elf/allure-report-action@master
if: github.ref == 'refs/heads/master'
with:
allure_results: build/${{ matrix.python-version }}/allure-results
subfolder: ${{ matrix.python-version }}
allure_history: allure-history
keep_reports: 20

- name: Tar report
if: github.ref == 'refs/heads/master'
run: tar -cvf allure_history_${{ matrix.python-version }}.tar allure-history/${{ matrix.python-version }}

- name: Upload artifact
uses: actions/upload-artifact@v4
if: github.ref == 'refs/heads/master'
with:
name: allure_history_${{ matrix.python-version }}
path: allure_history_${{ matrix.python-version }}.tar

deploy:
runs-on: ubuntu-latest
needs: build
if: github.ref == 'refs/heads/master'
steps:

- uses: actions/download-artifact@v4
name: Download 3.10 Artifacts
name: Download 3.10 artifacts
with:
name: allure_history_3.10
path: allure-history/tars

- uses: actions/download-artifact@v4
name: Download 3.11 Artifacts
name: Download 3.11 artifacts
with:
name: allure_history_3.11
path: allure-history/tars
- name: Untar Reports

- name: Untar reports
run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history ;done

- name: Remove Tar Reports
- name: Remove tar reports
run: rm -rf allure-history/tars

- name: Display structure of downloaded files
Expand Down
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ integration-tests: ## runs tests for the project
poetry run pytest uk_bin_collection/tests/step_defs/ -k "$(councils)" -n logical --alluredir=build/$(matrix)/allure-results; \
fi

parity-check:
poetry run python uk_bin_collection/tests/council_feature_input_parity.py $(branch)

unit-tests:
poetry run coverage erase
poetry run coverage run --append --omit "*/tests/*" -m pytest uk_bin_collection/tests --ignore=uk_bin_collection/tests/step_defs/
Expand Down
20 changes: 17 additions & 3 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ pytest-bdd = "*"
allure-pytest-bdd = "*"
pytest-xdist = {extras = ["psutil"], version = "*"}
pyhamcrest = "*"

tabulate = "^0.9.0"

[tool.poetry.scripts]
uk_bin_collection = "uk_bin_collection.uk_bin_collection.collect_data:run"
Expand All @@ -58,6 +58,7 @@ selenium = "*"
lxml = "*"
urllib3 = "*"
webdriver-manager = "^4.0.1"
tabulate = "^0.9.0"

[tool.commitizen]
major_version_zero = true
Expand Down
82 changes: 82 additions & 0 deletions uk_bin_collection/tests/council_feature_input_parity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import json
import re
import requests
import sys
from tabulate import tabulate

def get_councils_from_files(branch):
url = f"https://api.github.com/repos/robbrad/UKBinCollectionData/contents/uk_bin_collection/uk_bin_collection/councils?ref={branch}"
response = requests.get(url)
data = response.json()
return [item['name'].replace('.py', '') for item in data if item['name'].endswith('.py')]

def get_councils_from_json(branch):
url = f"https://raw.githubusercontent.com/robbrad/UKBinCollectionData/{branch}/uk_bin_collection/tests/input.json"
response = requests.get(url)
data = json.loads(response.text)
return list(data.keys())

def get_councils_from_features(branch):
url = f"https://raw.githubusercontent.com/robbrad/UKBinCollectionData/{branch}/uk_bin_collection/tests/features/validate_council_outputs.feature"
response = requests.get(url)
content = response.text
return re.findall(r'Examples:\s+(\w+)', content)

def compare_councils(councils1, councils2, councils3):
set1 = set(councils1)
set2 = set(councils2)
set3 = set(councils3)
all_councils = set1 | set2 | set3
all_council_data = {}
discrepancies_found = False
for council in all_councils:
in_files = council in set1
in_json = council in set2
in_features = council in set3
discrepancies_count = [in_files, in_json, in_features].count(False)
all_council_data[council] = {
'in_files': in_files,
'in_json': in_json,
'in_features': in_features,
'discrepancies_count': discrepancies_count
}
if discrepancies_count > 0:
discrepancies_found = True
return all_council_data, discrepancies_found

def main(branch):
# Execute and print the comparison
file_councils = get_councils_from_files(branch)
json_councils = get_councils_from_json(branch)
feature_councils = get_councils_from_features(branch)

all_councils_data, discrepancies_found = compare_councils(file_councils, json_councils, feature_councils)

# Create a list of lists for tabulate, sort by discrepancies count and then by name
table_data = []
headers = ["Council Name", "In Files", "In JSON", "In Features", "Discrepancies"]
for council, presence in sorted(all_councils_data.items(), key=lambda x: (x[1]['discrepancies_count'], x[0])):
row = [
council,
"✔" if presence['in_files'] else "✘",
"✔" if presence['in_json'] else "✘",
"✔" if presence['in_features'] else "✘",
presence['discrepancies_count']
]
table_data.append(row)

# Print the table using tabulate
print(tabulate(table_data, headers=headers, tablefmt='grid'))

if discrepancies_found:
print("Discrepancies found! Failing the workflow.")
sys.exit(1)
else:
print("No discrepancies found. Workflow successful.")

if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: python script.py <branch>")
sys.exit(1)
branch = sys.argv[1]
main(branch)
Loading

0 comments on commit e74d612

Please sign in to comment.