xfail test and add functionality for precise test marking

tests/e2e/base.py

@@ -13,16 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-__all__ = ["BaseE2ETest", "AVAILABLE_CONTAINERS", "fake"]
+__all__ = ["BaseE2ETest", "AVAILABLE_CONTAINERS", "fake", "Parameters", "ParametersFactory"]
 
 import inspect
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Union,
+)
 
 import pytest
 from faker import Faker
 
+ParameterSet = Any
+
 fake = Faker()
 
-AVAILABLE_CONTAINERS = ["project", "run"]
 AVAILABLE_CONTAINERS = [
     pytest.param("run"),
     pytest.param("project", marks=pytest.mark.skip(reason="Project not supported")),
@@ -37,6 +46,51 @@
 ]
 
 
+class Parameters:
+    param_d: Dict[str, Any]
+
+    def __init__(self, params: List[ParameterSet]) -> "Parameters":
+        self.param_d = {p.values[0]: p for p in params}
+
+    def _modify(self, *args: Union[str, str], func: Callable[[str], ParameterSet]) -> "Parameters":
+        for arg in args:
+            self.param_d[arg] = func(arg)
+        return self
+
+    def skip(self, *args: str, reason: Optional[str] = None) -> "Parameters":
+        return self._modify(*args, func=lambda x: pytest.param(x, marks=pytest.mark.skip(reason=reason)))
+
+    def xfail(
+        self, *args: str, reason: Optional[str] = None, strict=True, raises: Optional[Exception] = None
+    ) -> "Parameters":
+        return self._modify(
+            *args, func=lambda x: pytest.param(x, marks=pytest.mark.xfail(reason=reason, strict=strict, raises=raises))
+        )
+
+    def run(self, *args: str) -> "Parameters":
+        return self._modify(*args, func=lambda x: pytest.param(x))
+
+    def eval(self) -> List[ParameterSet]:
+        return list(self.param_d.values())
+
+
+class ParametersFactory:
+
+    @staticmethod
+    def custom(params: List[Union[str, ParameterSet]]) -> Parameters:
+        normalized = []
+        for p in params:
+            if isinstance(p, str):
+                normalized.append(pytest.param(p))
+            else:
+                normalized.append(p)
+        return Parameters(normalized)
+
+    @staticmethod
+    def available_containers() -> Parameters:
+        return Parameters(AVAILABLE_CONTAINERS.copy())
+
+
 class BaseE2ETest:
     def gen_key(self):
         # Get test name

tests/e2e/standard/test_artifacts.py

@@ -22,6 +22,7 @@
 
 import pytest
 
+from neptune.exceptions import NeptuneUnsupportedFunctionalityException
 from neptune.objects import NeptuneObject
 from tests.e2e.base import (
     AVAILABLE_CONTAINERS,
@@ -34,6 +35,7 @@
 )
 
 
+@pytest.mark.xfail(reason="File functionality disabled", strict=True, raises=NeptuneUnsupportedFunctionalityException)
 class TestArtifacts(BaseE2ETest):
     @pytest.mark.parametrize("container", AVAILABLE_CONTAINERS, indirect=True)
     def test_local_creation(self, container: NeptuneObject):

tests/e2e/standard/test_files.py

@@ -24,6 +24,7 @@
 
 import pytest
 
+from neptune.exceptions import NeptuneUnsupportedFunctionalityException
 from neptune.internal.backends import hosted_file_operations
 from neptune.internal.backends.api_model import (
     MultipartConfig,
@@ -39,6 +40,7 @@
 from tests.e2e.base import (
     AVAILABLE_CONTAINERS,
     BaseE2ETest,
+    ParametersFactory,
     fake,
 )
 from tests.e2e.plot_utils import (
@@ -58,8 +60,15 @@
 )
 
 
+@pytest.mark.xfail(reason="File functionality disabled", strict=True, raises=NeptuneUnsupportedFunctionalityException)
 class TestUpload(BaseE2ETest):
-    @pytest.mark.parametrize("container", AVAILABLE_CONTAINERS, indirect=True)
+    @pytest.mark.parametrize(
+        "container",
+        ParametersFactory.available_containers()
+        .skip("run", reason="Skipped as whole class is expected to fail")
+        .eval(),
+        indirect=True,
+    )
     def test_using_new_api(self, container: NeptuneObject):
         assert isinstance(container._backend, HostedNeptuneBackend)
         assert container._backend._client_config.has_feature(OptionalFeatures.MULTIPART_UPLOAD)
@@ -216,6 +225,7 @@ def test_upload_with_changed_working_directory(self, environment):
         run.stop()
 
 
+@pytest.mark.xfail(reason="File functionality disabled", strict=True, raises=NeptuneUnsupportedFunctionalityException)
 class TestFileSet(BaseE2ETest):
     def _test_fileset(self, container: NeptuneObject, large_file_size: int, small_files_no: int):
         key = self.gen_key()
@@ -491,6 +501,7 @@ def test_list_fileset_files(self, container: NeptuneObject):
             assert container[key].list_fileset_files() == []
 
 
+@pytest.mark.xfail(reason="File functionality disabled", strict=True, raises=NeptuneUnsupportedFunctionalityException)
 class TestPlotObjectsAssignment(BaseE2ETest):
     @pytest.mark.parametrize("container", ["run"], indirect=True)
     def test_pil_image(self, container: NeptuneObject):

tests/e2e/standard/test_series.py

@@ -20,6 +20,7 @@
 import pytest
 from PIL import Image
 
+from neptune.exceptions import NeptuneUnsupportedFunctionalityException
 from neptune.objects import NeptuneObject
 from neptune.types import (
     FileSeries,
@@ -29,6 +30,7 @@
 from tests.e2e.base import (
     AVAILABLE_CONTAINERS,
     BaseE2ETest,
+    ParametersFactory,
     fake,
 )
 from tests.e2e.utils import (
@@ -37,7 +39,11 @@
     tmp_context,
 )
 
-BASIC_SERIES_TYPES = ["strings", "floats", "files"]
+BASIC_SERIES_TYPES = (
+    ParametersFactory.custom(["strings", "floats", "files"])
+    .xfail("files", reason="File funcitonality disabled", raises=NeptuneUnsupportedFunctionalityException)
+    .eval()
+)
 
 
 class TestSeries(BaseE2ETest):
@@ -77,6 +83,9 @@ def test_string_series_type_assign(self, container: NeptuneObject):
         with self.run_then_assert(container, "strings") as (namespace, values, steps, timestamps):
             namespace.assign(StringSeries(values=values, steps=steps, timestamps=timestamps))
 
+    @pytest.mark.xfail(
+        reason="File funcitonality disabled", strict=True, raises=NeptuneUnsupportedFunctionalityException
+    )
     @pytest.mark.parametrize("container", AVAILABLE_CONTAINERS, indirect=True)
     def test_file_series_type_assign(self, container: NeptuneObject):
         with self.run_then_assert(container, "files") as (namespace, values, steps, timestamps):

tests/unit/neptune/new/attributes/atoms/test_file.py

@@ -47,7 +47,6 @@
 from tests.unit.neptune.new.attributes.test_attribute_base import TestAttributeBase
 
 
-@pytest.mark.xfail(reason="File functionality disabled")
 class TestFile(TestAttributeBase):
     @pytest.mark.xfail(
         reason="File functionality disabled", strict=True, raises=NeptuneUnsupportedFunctionalityException