diff --git a/ci/run_cudf_kafka_pytests.sh b/ci/run_cudf_kafka_pytests.sh
new file mode 100755
index 00000000000..de227c84872
--- /dev/null
+++ b/ci/run_cudf_kafka_pytests.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+# Support invoking run_cudf_kafka_pytests.sh outside the script directory
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf_kafka/cudf_kafka
+
+pytest --cache-clear "$@" tests
diff --git a/ci/run_custreamz_pytests.sh b/ci/run_custreamz_pytests.sh
index 53e27ec64b3..67b152fc187 100755
--- a/ci/run_custreamz_pytests.sh
+++ b/ci/run_custreamz_pytests.sh
@@ -3,7 +3,7 @@
 
 set -euo pipefail
 
-# It is essential to cd into python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level.
+# It is essential to cd into python/custreamz/custreamz/ as `pytest-xdist` + `coverage` seem to work only at this directory level.
 
 # Support invoking run_custreamz_pytests.sh outside the script directory
 cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/custreamz/custreamz/
diff --git a/ci/run_dask_cudf_pytests.sh b/ci/run_dask_cudf_pytests.sh
index 07658c6d234..37aadb5fee9 100755
--- a/ci/run_dask_cudf_pytests.sh
+++ b/ci/run_dask_cudf_pytests.sh
@@ -3,7 +3,7 @@
 
 set -euo pipefail
 
-# It is essential to cd into python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level.
+# It is essential to cd into python/dask_cudf/dask_cudf/ as `pytest-xdist` + `coverage` seem to work only at this directory level.
 
 # Support invoking run_dask_cudf_pytests.sh outside the script directory
 cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/dask_cudf/dask_cudf/
diff --git a/ci/test_python_other.sh b/ci/test_python_other.sh
index cbc1dc1cb87..06a24773cae 100755
--- a/ci/test_python_other.sh
+++ b/ci/test_python_other.sh
@@ -36,6 +36,10 @@ DASK_DATAFRAME__QUERY_PLANNING=False ./ci/run_dask_cudf_pytests.sh \
   --dist=loadscope \
   .
 
+rapids-logger "pytest cudf_kafka"
+./ci/run_cudf_kafka_pytests.sh \
+  --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-kafka.xml"
+
 rapids-logger "pytest custreamz"
 ./ci/run_custreamz_pytests.sh \
   --junitxml="${RAPIDS_TESTS_DIR}/junit-custreamz.xml" \
diff --git a/python/cudf_kafka/cudf_kafka/tests/__init__.py b/python/cudf_kafka/cudf_kafka/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/python/cudf_kafka/cudf_kafka/tests/test_version.py b/python/cudf_kafka/cudf_kafka/tests/test_version.py
new file mode 100644
index 00000000000..2dc2846c4cf
--- /dev/null
+++ b/python/cudf_kafka/cudf_kafka/tests/test_version.py
@@ -0,0 +1,12 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+import cudf_kafka
+
+
+def test_version_constants_are_populated():
+    # __git_commit__ will only be non-empty in a built distribution
+    assert isinstance(cudf_kafka.__git_commit__, str)
+
+    # __version__ should always be non-empty
+    assert isinstance(cudf_kafka.__version__, str)
+    assert len(cudf_kafka.__version__) > 0
diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml
index d34a1260422..9233d0e92dd 100644
--- a/python/cudf_kafka/pyproject.toml
+++ b/python/cudf_kafka/pyproject.toml
@@ -82,6 +82,11 @@ skip = [
     "__init__.py",
 ]
 
+[tool.pytest.ini_options]
+filterwarnings = [
+  "error"
+]
+
 [tool.scikit-build]
 build-dir = "build/{wheel_tag}"
 cmake.build-type = "Release"