Merge branch 'master' into PA-testing

.github/workflows/selfdrive_tests.yaml

@@ -27,7 +27,6 @@ env:
   RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
   PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0
-  XDIST: -n auto --dist=loadscope
 
 jobs:
   build_release:
@@ -58,7 +57,7 @@ jobs:
       run: |
         cd $STRIPPED_DIR
         ${{ env.RUN }} "release/check-dirty.sh && \
-                        MAX_EXAMPLES=5 $PYTEST $XDIST selfdrive/car"
+                        MAX_EXAMPLES=5 $PYTEST selfdrive/car"
     - name: pre-commit
       timeout-minutes: 3
       run: |
@@ -176,7 +175,7 @@ jobs:
     - name: Run unit tests
       timeout-minutes: 15
       run: |
-        ${{ env.RUN }} "$PYTEST $XDIST --timeout 30 -o cpp_files=test_* -m 'not slow' && \
+        ${{ env.RUN }} "$PYTEST --timeout 30 -m 'not slow' && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                         ./selfdrive/ui/tests/test_translations.py && \
@@ -209,30 +208,20 @@ jobs:
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run replay
+      id: run-replay
       timeout-minutes: 30
       run: |
-        ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
-                        chmod -R 777 /tmp/comma_download_cache && \
-                        coverage combine && \
-                        coverage xml"
-    - name: Print diff
-      id: print-diff
-      if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
-    - uses: actions/upload-artifact@v3
-      if: always()
-      continue-on-error: true
-      with:
-        name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
-    - name: Upload reference logs
-      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
-      run: |
-        ${{ env.RUN }} "unset PYTHONWARNINGS && CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+        ${{ env.RUN }} "CI=1 $PYTEST -n auto --dist=loadscope selfdrive/test/process_replay/test_processes.py --long-diff && \
+                        chmod -R 777 /tmp/comma_download_cache"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
       with:
         name: ${{ github.job }}
+    - name: Upload reference logs
+      if: ${{ failure() && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
+      run: |
+        ${{ env.RUN }} "unset PYTHONWARNINGS && CI=1 AZURE_TOKEN='$AZURE_TOKEN' \
+                        pytest -n auto --dist=loadscope selfdrive/test/process_replay/test_processes.py --upload-only"
 
   regen:
     name: regen
@@ -258,7 +247,7 @@ jobs:
     - name: Run regen
       timeout-minutes: 30
       run: |
-        ${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST $XDIST selfdrive/test/process_replay/test_regen.py && \
+        ${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
                            chmod -R 777 /tmp/comma_download_cache"
 
   test_modeld:
@@ -318,7 +307,7 @@ jobs:
     - name: Test car models
       timeout-minutes: 25
       run: |
-        ${{ env.RUN }} "$PYTEST $XDIST selfdrive/car/tests/test_models.py && \
+        ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \
                         chmod -R 777 /tmp/comma_download_cache"
       env:
         NUM_JOBS: 5

.github/workflows/stale.yaml

@@ -0,0 +1,28 @@
+name: stale
+on:
+  schedule:
+    - cron: '30 1 * * *'
+  workflow_dispatch:
+
+env:
+  DAYS_BEFORE_PR_CLOSE: 7
+  DAYS_BEFORE_PR_STALE: 30
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v8
+        with:
+          exempt-milestones: true
+
+          # pull request config
+          stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
+          close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
+          delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
+          exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
+          days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
+          days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
+
+          # issue config
+          days-before-issue-stale: -1 # ignore issues for now

.gitmodules

@@ -4,9 +4,6 @@
 [submodule "opendbc"]
   path = opendbc
   url = ../../Edison-CBS/opendbc.git
-[submodule "laika_repo"]
-  path = laika_repo
-  url = ../../Edison-CBS/laika.git
 [submodule "cereal"]
   path = cereal
   url = ../../Edison-CBS/cereal.git

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
     rev: v2.2.6
     hooks:
     -   id: codespell
-        exclude: '^(third_party/)|(body/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)'
+        exclude: '^(third_party/)|(body/)|(cereal/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)'
         args:
         # if you've got a short variable name that's getting flagged, add it here
         - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints
@@ -38,13 +38,13 @@ repos:
         entry: mypy
         language: system
         types: [python]
-        args: ['--explicit-package-bases']
-        exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)'
+        args: ['--explicit-package-bases', '--local-partial-types']
+        exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)'
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.4
+    rev: v0.1.6
     hooks:
     -   id: ruff
-        exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'
+        exclude: '^(third_party/)|(cereal/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'
 -   repo: local
     hooks:
     -   id: cppcheck

Dockerfile.openpilot

@@ -13,9 +13,8 @@ COPY SConstruct ${OPENPILOT_PATH}
 COPY ./openpilot ${OPENPILOT_PATH}/openpilot
 COPY ./third_party ${OPENPILOT_PATH}/third_party
 COPY ./site_scons ${OPENPILOT_PATH}/site_scons
-COPY ./laika ${OPENPILOT_PATH}/laika
-COPY ./laika_repo ${OPENPILOT_PATH}/laika_repo
 COPY ./rednose ${OPENPILOT_PATH}/rednose
+COPY ./rednose_repo/site_scons ${OPENPILOT_PATH}/rednose_repo/site_scons
 COPY ./tools ${OPENPILOT_PATH}/tools
 COPY ./release ${OPENPILOT_PATH}/release
 COPY ./common ${OPENPILOT_PATH}/common

Jenkinsfile

@@ -14,6 +14,8 @@ export GIT_BRANCH=${env.GIT_BRANCH}
 export GIT_COMMIT=${env.GIT_COMMIT}
 export AZURE_TOKEN='${env.AZURE_TOKEN}'
 export MAPBOX_TOKEN='${env.MAPBOX_TOKEN}'
+export PYTEST_ADDOPTS="-c selfdrive/test/pytest-tici.ini --rootdir ."
+
 
 export GIT_SSH_COMMAND="ssh -i /data/gitkey"
 
@@ -159,7 +161,7 @@ node {
           ["build openpilot", "cd selfdrive/manager && ./build.py"],
           ["check dirty", "release/check-dirty.sh"],
           ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"],
-          ["time to onroad", "cd selfdrive/test/ && pytest test_time_to_onroad.py"],
+          ["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"],
         ])
       },
       'HW + Unit Tests': {
@@ -194,17 +196,17 @@ node {
       'sensord': {
         deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
           ["build", "cd selfdrive/manager && ./build.py"],
-          ["test sensord", "cd system/sensord/tests && pytest test_sensord.py"],
+          ["test sensord", "pytest system/sensord/tests/test_sensord.py"],
         ])
         deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
           ["build", "cd selfdrive/manager && ./build.py"],
-          ["test sensord", "cd system/sensord/tests && pytest test_sensord.py"],
+          ["test sensord", "pytest system/sensord/tests/test_sensord.py"],
         ])
       },
       'replay': {
         deviceStage("tici", "tici-replay", ["UNSAFE=1"], [
           ["build", "cd selfdrive/manager && ./build.py"],
-          ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"],
+          ["model replay", "selfdrive/test/process_replay/model_replay.py"],
         ])
       },
       'tizi': {
@@ -214,7 +216,7 @@ node {
           ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
           ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"],
           ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"],
-          ["test rawgpsd", "pytest system/sensord/rawgps/test_rawgps.py"],
+          ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"],
         ])
       },
 
@@ -231,7 +233,7 @@ node {
           sh label: "build", script: "selfdrive/manager/build.py"
           sh label: "test_models.py", script: "INTERNAL_SEG_CNT=250 INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 \
               pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py"
-          sh label: "test_car_interfaces.py", script: "MAX_EXAMPLES=100 pytest -n42 selfdrive/car/tests/test_car_interfaces.py"
+          sh label: "test_car_interfaces.py", script: "MAX_EXAMPLES=100 pytest -n42 --dist=load selfdrive/car/tests/test_car_interfaces.py"
         }
       },
 

RELEASES.md

@@ -1,4 +1,7 @@
-Version 0.9.5 (2023-11-16)
+Version 0.9.6 (2023-XX-XX)
+========================
+
+Version 0.9.5 (2023-11-17)
 ========================
 * New driving model
   * Improved navigate on openpilot performance using navigation instructions as an additional model input
@@ -8,11 +11,12 @@ Version 0.9.5 (2023-11-16)
 * Hyundai Azera 2022 support thanks to sunnyhaibin!
 * Hyundai Azera Hybrid 2020 support thanks to chanhojung and haram-KONA!
 * Hyundai Custin 2023 support thanks to sunnyhaibin and Saber422!
-* Hyundai Ioniq 6 2023 support thanks to sunnyhaibin, alamo3, and sshane!
+* Hyundai Ioniq 6 2023 support thanks to sunnyhaibin and alamo3!
 * Hyundai Kona Electric 2023 (Korean version) support thanks to sunnyhaibin and haram-KONA!
 * Kia K8 Hybrid (with HDA II) 2023 support thanks to sunnyhaibin!
-* Kia Sorento Hybrid 2023 support thanks to sunnyhaibin!
 * Kia Optima Hybrid 2019 support
+* Kia Sorento Hybrid 2023 support thanks to sunnyhaibin!
+* Lexus GS F 2016 support thanks to snyperifle!
 * Lexus IS 2023 support thanks to L3R5!
 
 Version 0.9.4 (2023-07-27)

SConstruct

@@ -37,6 +37,12 @@ AddOption('--clazy',
 AddOption('--compile_db',
           action='store_true',
           help='build clang compilation database')
+
+AddOption('--ccflags',
+          action='store',
+          type='string',
+          default='',
+          help='pass arbitrary flags over the command line')
 
 AddOption('--snpe',
           action='store_true',
@@ -56,7 +62,7 @@ AddOption('--pc-thneed',
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=os.path.islink(Dir('#laika/').abspath),
+          default=os.path.islink(Dir('#rednose/').abspath), # minimal by default on release branch (where rednose is not a link)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
 ## Architecture name breakdown (arch)
@@ -110,10 +116,7 @@ else:
   cflags = []
   cxxflags = []
   cpppath = []
-  rpath += [
-    Dir("#cereal").abspath,
-    Dir("#common").abspath
-  ]
+  rpath += []
 
   # MacOS
   if arch == "Darwin":
@@ -138,8 +141,6 @@ else:
       f"#third_party/acados/{arch}/lib",
       f"#third_party/libyuv/{arch}/lib",
       f"#third_party/mapbox-gl-native-qt/{arch}",
-      "#cereal",
-      "#common",
       "/usr/lib",
       "/usr/local/lib",
     ]
@@ -170,6 +171,10 @@ if arch != "Darwin":
 cflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
 cxxflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
 
+ccflags_option = GetOption('ccflags')
+if ccflags_option:
+  ccflags += ccflags_option.split(' ')
+
 env = Environment(
   ENV=lenv,
   CCFLAGS=[
@@ -219,10 +224,13 @@ env = Environment(
     "#opendbc/can",
     "#selfdrive/boardd",
     "#common",
+    "#rednose/helpers",
   ],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
-  tools=["default", "cython", "compilation_db"],
+  REDNOSE_ROOT="#",
+  tools=["default", "cython", "compilation_db", "rednose_filter"],
+  toolpath=["#rednose_repo/site_scons/site_tools"],
 )
 
 if arch == "Darwin":
@@ -357,31 +365,7 @@ SConscript([
   'panda/SConscript',
 ])
 
-# Build rednose library and ekf models
-rednose_deps = [
-  "#selfdrive/locationd/models/constants.py",
-  "#selfdrive/locationd/models/gnss_helpers.py",
-]
-
-rednose_config = {
-  'generated_folder': '#selfdrive/locationd/models/generated',
-  'to_build': {
-    'gnss': ('#selfdrive/locationd/models/gnss_kf.py', True, [], rednose_deps),
-    'live': ('#selfdrive/locationd/models/live_kf.py', True, ['live_kf_constants.h'], rednose_deps),
-    'car': ('#selfdrive/locationd/models/car_kf.py', True, [], rednose_deps),
-  },
-}
-
-if arch != "larch64":
-  rednose_config['to_build'].update({
-    'loc_4': ('#selfdrive/locationd/models/loc_kf.py', True, [], rednose_deps),
-    'lane': ('#selfdrive/locationd/models/lane_kf.py', True, [], rednose_deps),
-    'pos_computer_4': ('#rednose/helpers/lst_sq_computer.py', False, [], []),
-    'pos_computer_5': ('#rednose/helpers/lst_sq_computer.py', False, [], []),
-    'feature_handler_5': ('#rednose/helpers/feature_handler.py', False, [], []),
-  })
-
-Export('rednose_config')
+# Build rednose library
 SConscript(['rednose/SConscript'])
 
 # Build system services

common/prefix.py

@@ -21,6 +21,7 @@ def __enter__(self):
     except FileExistsError:
       pass
     os.makedirs(Paths.log_root(), exist_ok=True)
+    os.makedirs(Paths.download_cache_root(), exist_ok=True)
 
     return self
 

common/version.h

@@ -1 +1 @@
-#define COMMA_VERSION "0.9.5"
+#define COMMA_VERSION "0.9.6"