mark xfail

Author: JATAYU000

tests/test_datasets/test_dataset_functions.py

@@ -244,6 +244,7 @@ def test_get_datasets(self):
         assert len(datasets) == 2
         _assert_datasets_retrieved_successfully([1, 2])
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_dataset_by_name(self):
         dataset = openml.datasets.get_dataset("anneal")
         assert type(dataset) == OpenMLDataset
@@ -262,6 +263,7 @@ def test_get_dataset_download_all_files(self):
         # test_get_dataset_lazy
         raise NotImplementedError
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_dataset_uint8_dtype(self):
         dataset = openml.datasets.get_dataset(1)
         assert type(dataset) == OpenMLDataset
@@ -280,6 +282,7 @@ def test_dataset_by_name_cannot_access_private_data(self):
         self.use_production_server()
         self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, "NAME_GOES_HERE")
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_dataset_lazy_all_functions(self):
         """Test that all expected functionality is available without downloading the dataset."""
         dataset = openml.datasets.get_dataset(1)
@@ -309,6 +312,7 @@ def ensure_absence_of_real_data():
         assert classes == ["1", "2", "3", "4", "5", "U"]
         ensure_absence_of_real_data()
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_dataset_sparse(self):
         dataset = openml.datasets.get_dataset(102)
         X, *_ = dataset.get_data()
@@ -327,6 +331,7 @@ def test__get_dataset_description(self):
         description_xml_path = os.path.join(self.workdir, "description.xml")
         assert os.path.exists(description_xml_path)
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test__getarff_path_dataset_arff(self):
         openml.config.set_root_cache_directory(self.static_cache_dir)
         description = _get_dataset_description(self.workdir, 2)
@@ -430,12 +435,14 @@ def test__getarff_md5_issue(self):
 
         openml.config.connection_n_retries = n
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test__get_dataset_features(self):
         features_file = _get_dataset_features_file(self.workdir, 2)
         assert isinstance(features_file, Path)
         features_xml_path = self.workdir / "features.xml"
         assert features_xml_path.exists()
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test__get_dataset_qualities(self):
         qualities = _get_dataset_qualities_file(self.workdir, 2)
         assert isinstance(qualities, Path)
@@ -853,6 +860,7 @@ def test_create_invalid_dataset(self):
         param["data"] = data[0]
         self.assertRaises(ValueError, create_dataset, **param)
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_online_dataset_arff(self):
         dataset_id = 100  # Australian
         # lazy loading not used as arff file is checked.
@@ -1332,6 +1340,7 @@ def test_list_qualities(self):
         assert isinstance(qualities, list) is True
         assert all(isinstance(q, str) for q in qualities) is True
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_dataset_cache_format_pickle(self):
         dataset = openml.datasets.get_dataset(1)
         dataset.get_data()
@@ -1347,6 +1356,7 @@ def test_get_dataset_cache_format_pickle(self):
         assert len(categorical) == X.shape[1]
         assert len(attribute_names) == X.shape[1]
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_dataset_cache_format_feather(self):
         # This test crashed due to using the parquet file by default, which is downloaded
         # from minio. However, there is a mismatch between OpenML test server and minio IDs.
@@ -1523,6 +1533,7 @@ def test_list_datasets_with_high_size_parameter(self):
         (None, None, ["wrong", "sunny"]),
     ],
 )
+@pytest.mark.xfail(reason="failures_issue_1544")
 def test_invalid_attribute_validations(
     default_target_attribute,
     row_id_attribute,
@@ -1584,6 +1595,7 @@ def test_invalid_attribute_validations(
         (None, None, ["outlook", "windy"]),
     ],
 )
+@pytest.mark.xfail(reason="failures_issue_1544")
 def test_valid_attribute_validations(default_target_attribute, row_id_attribute, ignore_attribute):
     data = [
         ["a", "sunny", 85.0, 85.0, "FALSE", "no"],
@@ -1802,6 +1814,7 @@ def test_list_datasets_by_number_instances(all_datasets: pd.DataFrame):
     _assert_datasets_have_id_and_valid_status(small_datasets)
 
 
+@pytest.mark.xfail(reason="failures_issue_1544")
 def test_list_datasets_by_number_features(all_datasets: pd.DataFrame):
     wide_datasets = openml.datasets.list_datasets(number_features="50..100")
     assert 8 <= len(wide_datasets) < len(all_datasets)
@@ -1814,12 +1827,14 @@ def test_list_datasets_by_number_classes(all_datasets: pd.DataFrame):
     _assert_datasets_have_id_and_valid_status(five_class_datasets)
 
 
+@pytest.mark.xfail(reason="failures_issue_1544")
 def test_list_datasets_by_number_missing_values(all_datasets: pd.DataFrame):
     na_datasets = openml.datasets.list_datasets(number_missing_values="5..100")
     assert 5 <= len(na_datasets) < len(all_datasets)
     _assert_datasets_have_id_and_valid_status(na_datasets)
 
 
+@pytest.mark.xfail(reason="failures_issue_1544")
 def test_list_datasets_combined_filters(all_datasets: pd.DataFrame):
     combined_filter_datasets = openml.datasets.list_datasets(
         tag="study_14",
@@ -1892,6 +1907,7 @@ def isolate_for_test():
     ("with_data", "with_qualities", "with_features"),
     itertools.product([True, False], repeat=3),
 )
+@pytest.mark.xfail(reason="failures_issue_1544")
 def test_get_dataset_lazy_behavior(
     isolate_for_test, with_data: bool, with_qualities: bool, with_features: bool
 ):

tests/test_runs/test_run_functions.py

@@ -1695,6 +1695,7 @@ def test_format_prediction_non_supervised(self):
         ):
             format_prediction(clustering, *ignored_input)
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_format_prediction_classification_no_probabilities(self):
         classification = openml.tasks.get_task(
             self.TEST_SERVER_TASK_SIMPLE["task_id"],
@@ -1704,6 +1705,7 @@ def test_format_prediction_classification_no_probabilities(self):
         with pytest.raises(ValueError, match="`proba` is required for classification task"):
             format_prediction(classification, *ignored_input, proba=None)
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_format_prediction_classification_incomplete_probabilities(self):
         classification = openml.tasks.get_task(
             self.TEST_SERVER_TASK_SIMPLE["task_id"],
@@ -1714,6 +1716,7 @@ def test_format_prediction_classification_incomplete_probabilities(self):
         with pytest.raises(ValueError, match="Each class should have a predicted probability"):
             format_prediction(classification, *ignored_input, proba=incomplete_probabilities)
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_format_prediction_task_without_classlabels_set(self):
         classification = openml.tasks.get_task(
             self.TEST_SERVER_TASK_SIMPLE["task_id"],
@@ -1724,6 +1727,7 @@ def test_format_prediction_task_without_classlabels_set(self):
         with pytest.raises(ValueError, match="The classification task must have class labels set"):
             format_prediction(classification, *ignored_input, proba={})
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_format_prediction_task_learning_curve_sample_not_set(self):
         learning_curve = openml.tasks.get_task(801, download_data=False)  # diabetes;crossvalidation
         probabilities = {c: 0.2 for c in learning_curve.class_labels}

tests/test_setups/test_setup_functions.py

@@ -166,6 +166,7 @@ def test_list_setups_output_format(self):
         assert isinstance(setups, pd.DataFrame)
         assert len(setups) == 10
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_setuplist_offset(self):
         size = 10
         setups = openml.setups.list_setups(offset=0, size=size)

tests/test_study/test_study_functions.py

@@ -148,6 +148,7 @@ def test_publish_empty_study_implicit(self):
         self._test_publish_empty_study_is_allowed(explicit=False)
 
     @pytest.mark.flaky()
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_publish_study(self):
         # get some random runs to attach
         run_list = openml.evaluations.list_evaluations("predictive_accuracy", size=10)
@@ -217,6 +218,7 @@ def test_publish_study(self):
         res = openml.study.delete_study(study.id)
         assert res
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_study_attach_illegal(self):
         run_list = openml.runs.list_runs(size=10)
         assert len(run_list) == 10

tests/test_tasks/test_classification_task.py

@@ -18,19 +18,22 @@ def setUp(self, n_levels: int = 1):
         self.task_type = TaskType.SUPERVISED_CLASSIFICATION
         self.estimation_procedure = 5
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_download_task(self):
         task = super().test_download_task()
         assert task.task_id == self.task_id
         assert task.task_type_id == TaskType.SUPERVISED_CLASSIFICATION
         assert task.dataset_id == 20
         assert task.estimation_procedure_id == self.estimation_procedure
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_class_labels(self):
         task = get_task(self.task_id)
         assert task.class_labels == ["tested_negative", "tested_positive"]
 
 
 @pytest.mark.server()
+@pytest.mark.xfail(reason="failures_issue_1544")
 def test_get_X_and_Y():
     task = get_task(119)
     X, Y = task.get_X_and_y()

tests/test_tasks/test_learning_curve_task.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import pandas as pd
+import pytest
 
 from openml.tasks import TaskType, get_task
 
@@ -17,6 +18,7 @@ def setUp(self, n_levels: int = 1):
         self.task_type = TaskType.LEARNING_CURVE
         self.estimation_procedure = 13
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_X_and_Y(self):
         X, Y = super().test_get_X_and_Y()
         assert X.shape == (768, 8)
@@ -25,12 +27,14 @@ def test_get_X_and_Y(self):
         assert isinstance(Y, pd.Series)
         assert pd.api.types.is_categorical_dtype(Y)
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_download_task(self):
         task = super().test_download_task()
         assert task.task_id == self.task_id
         assert task.task_type_id == TaskType.LEARNING_CURVE
         assert task.dataset_id == 20
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_class_labels(self):
         task = get_task(self.task_id)
         assert task.class_labels == ["tested_negative", "tested_positive"]

tests/test_tasks/test_regression_task.py

@@ -4,6 +4,7 @@
 import ast
 
 import pandas as pd
+import pytest
 
 import openml
 from openml.exceptions import OpenMLServerException
@@ -48,6 +49,7 @@ def setUp(self, n_levels: int = 1):
         self.task_type = TaskType.SUPERVISED_REGRESSION
 
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_X_and_Y(self):
         X, Y = super().test_get_X_and_Y()
         assert X.shape == (194, 32)

tests/test_tasks/test_task_functions.py

@@ -100,6 +100,7 @@ def test_list_tasks(self):
         for task in tasks.to_dict(orient="index").values():
             self._check_task(task)
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_list_tasks_paginate(self):
         size = 10
         max = 100
@@ -139,6 +140,7 @@ def test__get_task_live(self):
         # https://github.com/openml/openml-python/issues/378
         openml.tasks.get_task(34536)
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_task(self):
         task = openml.tasks.get_task(1, download_data=True)  # anneal; crossvalidation
         assert isinstance(task, OpenMLTask)
@@ -152,6 +154,7 @@ def test_get_task(self):
             os.path.join(self.workdir, "org", "openml", "test", "datasets", "1", "dataset.arff")
         )
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_task_lazy(self):
         task = openml.tasks.get_task(2, download_data=False)  # anneal; crossvalidation
         assert isinstance(task, OpenMLTask)
@@ -191,6 +194,7 @@ def assert_and_raise(*args, **kwargs):
         # Now the file should no longer exist
         assert not os.path.exists(os.path.join(os.getcwd(), "tasks", "1", "tasks.xml"))
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_get_task_with_cache(self):
         openml.config.set_root_cache_directory(self.static_cache_dir)
         task = openml.tasks.get_task(1)
@@ -206,6 +210,7 @@ def test_get_task_different_types(self):
         # Issue 538, get_task failing with clustering task.
         openml.tasks.functions.get_task(126033)
 
+    @pytest.mark.xfail(reason="failures_issue_1544")
     def test_download_split(self):
         task = openml.tasks.get_task(1)  # anneal; crossvalidation
         split = task.download_split()