Rename 'production' to 'production_server' for markers and CLI shorthands

Author: AniruthKarthik

.github/workflows/test.yml

@@ -112,9 +112,9 @@ jobs:
         fi
 
         if [ "${{ matrix.sklearn-only }}" = "true" ]; then
-          marks="sklearn and not production and not uses_test_server"
+          marks="sklearn and not production_server and not uses_test_server"
         else
-          marks="not production and not uses_test_server"
+          marks="not production_server and not uses_test_server"
         fi
 
         pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
@@ -127,9 +127,9 @@ jobs:
         fi
 
         if [ "${{ matrix.sklearn-only }}" = "true" ]; then
-          marks="sklearn and production and not uses_test_server"
+          marks="sklearn and production_server and not uses_test_server"
         else
-          marks="production and not uses_test_server"
+          marks="production_server and not uses_test_server"
         fi
 
         pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

openml/cli.py

@@ -102,15 +102,15 @@ def check_apikey(apikey: str) -> str:
 
 def configure_server(value: str) -> None:
     def check_server(server: str) -> str:
-        is_shorthand = server in ["test", "production"]
+        is_shorthand = server in ["test", "production_server"]
         if is_shorthand or looks_like_url(server):
             return ""
-        return "Must be 'test', 'production' or a url."
+        return "Must be 'test', 'production_server' or a url."
 
     def replace_shorthand(server: str) -> str:
         if server == "test":
             return "https://test.openml.org/api/v1/xml"
-        if server == "production":
+        if server == "production_server":
             return "https://www.openml.org/api/v1/xml"
         return server
 
@@ -119,7 +119,7 @@ def replace_shorthand(server: str) -> str:
         value=value,
         check_with_message=check_server,
         intro_message="Specify which server you wish to connect to.",
-        input_message="Specify a url or use 'test' or 'production' as a shorthand: ",
+        input_message="Specify a url or use 'test' or 'production_server' as a shorthand: ",
         sanitize=replace_shorthand,
     )
 

pyproject.toml

@@ -134,7 +134,8 @@ filterwarnings=[
 ]
 markers = [
   "upload: anything that uploads to a server",
-  "production: any interaction with the production server",
+      "production_server: any interaction with the production server",
+  
   "cache: anything that interacts with the (test) cache",
   "test_server: tests that require the OpenML test server",
 ]

tests/conftest.py

@@ -189,6 +189,7 @@ def pytest_sessionfinish() -> None:
 
 def pytest_configure(config):
     config.addinivalue_line("markers", "sklearn: marks tests that use scikit-learn")
+    config.addinivalue_line("markers", "production_server: marks tests that use the production server")
 
 
 def pytest_addoption(parser):
@@ -272,7 +273,7 @@ def as_robot() -> Iterator[None]:
 
 @pytest.fixture(autouse=True)
 def with_server(request):
-    if "production" in request.keywords:
+    if "production_server" in request.keywords:
         openml.config.server = "https://www.openml.org/api/v1/xml"
         openml.config.apikey = None
         yield

tests/test_datasets/test_dataset.py

@@ -18,7 +18,7 @@
 import pytest
 
 
-@pytest.mark.production()
+@pytest.mark.production_server()
 class OpenMLDatasetTest(TestBase):
     _multiprocess_can_split_ = True
 
@@ -347,7 +347,7 @@ def test_add_illegal_url_ontology():
         assert e.code == 1106
 
 
-@pytest.mark.production()
+@pytest.mark.production_server()
 class OpenMLDatasetTestSparse(TestBase):
     _multiprocess_can_split_ = True
 

tests/test_datasets/test_dataset_functions.py

@@ -140,7 +140,7 @@ def test_list_datasets_empty(self):
         datasets = openml.datasets.list_datasets(tag="NoOneWouldUseThisTagAnyway")
         assert datasets.empty
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_check_datasets_active(self):
         # Have to test on live because there is no deactivated dataset on the test server.
         self.use_production_server()
@@ -179,27 +179,27 @@ def test_illegal_length_tag(self):
         except openml.exceptions.OpenMLServerException as e:
             assert e.code == 477
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test__name_to_id_with_deactivated(self):
         """Check that an activated dataset is returned if an earlier deactivated one exists."""
         self.use_production_server()
         # /d/1 was deactivated
         assert openml.datasets.functions._name_to_id("anneal") == 2
         openml.config.server = self.test_server
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test__name_to_id_with_multiple_active(self):
         """With multiple active datasets, retrieve the least recent active."""
         self.use_production_server()
         assert openml.datasets.functions._name_to_id("iris") == 61
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test__name_to_id_with_version(self):
         """With multiple active datasets, retrieve the least recent active."""
         self.use_production_server()
         assert openml.datasets.functions._name_to_id("iris", version=3) == 969
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test__name_to_id_with_multiple_active_error(self):
         """With multiple active datasets, retrieve the least recent active."""
         self.use_production_server()
@@ -282,7 +282,7 @@ def test_get_dataset_uint8_dtype(self):
         df, _, _, _ = dataset.get_data()
         assert df["carbon"].dtype == "uint8"
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_get_dataset_cannot_access_private_data(self):
         # Issue324 Properly handle private datasets when trying to access them
         self.use_production_server()
@@ -1545,7 +1545,7 @@ def test_data_fork(self):
         )
 
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_list_datasets_with_high_size_parameter(self):
         # Testing on prod since concurrent deletion of uploded datasets make the test fail
         self.use_production_server()

tests/test_evaluations/test_evaluation_functions.py

@@ -50,7 +50,7 @@ def _check_list_evaluation_setups(self, **kwargs):
             self.assertSequenceEqual(sorted(list1), sorted(list2))
         return evals_setups
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_evaluation_list_filter_task(self):
         self.use_production_server()
 
@@ -70,7 +70,7 @@ def test_evaluation_list_filter_task(self):
             assert evaluations[run_id].value is not None
             assert evaluations[run_id].values is None
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_evaluation_list_filter_uploader_ID_16(self):
         self.use_production_server()
 
@@ -85,7 +85,7 @@ def test_evaluation_list_filter_uploader_ID_16(self):
 
         assert len(evaluations) > 50
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_evaluation_list_filter_uploader_ID_10(self):
         self.use_production_server()
 
@@ -104,7 +104,7 @@ def test_evaluation_list_filter_uploader_ID_10(self):
             assert evaluations[run_id].value is not None
             assert evaluations[run_id].values is None
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_evaluation_list_filter_flow(self):
         self.use_production_server()
 
@@ -124,7 +124,7 @@ def test_evaluation_list_filter_flow(self):
             assert evaluations[run_id].value is not None
             assert evaluations[run_id].values is None
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_evaluation_list_filter_run(self):
         self.use_production_server()
 
@@ -144,7 +144,7 @@ def test_evaluation_list_filter_run(self):
             assert evaluations[run_id].value is not None
             assert evaluations[run_id].values is None
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_evaluation_list_limit(self):
         self.use_production_server()
 
@@ -163,7 +163,7 @@ def test_list_evaluations_empty(self):
 
         assert isinstance(evaluations, dict)
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_evaluation_list_per_fold(self):
         self.use_production_server()
         size = 1000
@@ -201,7 +201,7 @@ def test_evaluation_list_per_fold(self):
             assert evaluations[run_id].value is not None
             assert evaluations[run_id].values is None
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_evaluation_list_sort(self):
         self.use_production_server()
         size = 10
@@ -239,7 +239,7 @@ def test_list_evaluation_measures(self):
         assert isinstance(measures, list) is True
         assert all(isinstance(s, str) for s in measures) is True
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_list_evaluations_setups_filter_flow(self):
         self.use_production_server()
         flow_id = [405]
@@ -257,7 +257,7 @@ def test_list_evaluations_setups_filter_flow(self):
         keys = list(evals["parameters"].values[0].keys())
         assert all(elem in columns for elem in keys)
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     @pytest.mark.xfail(reason="failures_issue_1544", strict=False)
     def test_list_evaluations_setups_filter_task(self):
         self.use_production_server()

tests/test_flows/test_flow.py

@@ -44,7 +44,7 @@ def setUp(self):
     def tearDown(self):
         super().tearDown()
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_get_flow(self):
         # We need to use the production server here because 4024 is not the
         # test server
@@ -77,7 +77,7 @@ def test_get_flow(self):
         assert subflow_3.parameters["L"] == "-1"
         assert len(subflow_3.components) == 0
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     @pytest.mark.xfail(reason="failures_issue_1544", strict=False)
     def test_get_structure(self):
         # also responsible for testing: flow.get_subflow
@@ -565,7 +565,7 @@ def test_extract_tags(self):
         tags = openml.utils.extract_xml_tags("oml:tag", flow_dict["oml:flow"])
         assert tags == ["OpenmlWeka", "weka"]
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_download_non_scikit_learn_flows(self):
         self.use_production_server()
 

tests/test_flows/test_flow_functions.py

@@ -47,7 +47,7 @@ def _check_flow(self, flow):
         )
         assert ext_version_str_or_none
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_list_flows(self):
         self.use_production_server()
         # We can only perform a smoke test here because we test on dynamic
@@ -58,7 +58,7 @@ def test_list_flows(self):
         for flow in flows.to_dict(orient="index").values():
             self._check_flow(flow)
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_list_flows_output_format(self):
         self.use_production_server()
         # We can only perform a smoke test here because we test on dynamic
@@ -67,21 +67,21 @@ def test_list_flows_output_format(self):
         assert isinstance(flows, pd.DataFrame)
         assert len(flows) >= 1500
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_list_flows_empty(self):
         self.use_production_server()
         flows = openml.flows.list_flows(tag="NoOneEverUsesThisTag123")
         assert flows.empty
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_list_flows_by_tag(self):
         self.use_production_server()
         flows = openml.flows.list_flows(tag="weka")
         assert len(flows) >= 5
         for flow in flows.to_dict(orient="index").values():
             self._check_flow(flow)
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_list_flows_paginate(self):
         self.use_production_server()
         size = 10
@@ -301,7 +301,7 @@ def test_sklearn_to_flow_list_of_lists(self):
         assert server_flow.parameters["categories"] == "[[0, 1], [0, 1]]"
         assert server_flow.model.categories == flow.model.categories
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_get_flow1(self):
         # Regression test for issue #305
         # Basically, this checks that a flow without an external version can be loaded
@@ -338,7 +338,7 @@ def test_get_flow_reinstantiate_model_no_extension(self):
         Version(sklearn.__version__) == Version("0.19.1"),
         reason="Requires scikit-learn!=0.19.1, because target flow is from that version.",
     )
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_get_flow_with_reinstantiate_strict_with_wrong_version_raises_exception(self):
         self.use_production_server()
         flow = 8175
@@ -359,7 +359,7 @@ def test_get_flow_with_reinstantiate_strict_with_wrong_version_raises_exception(
         # Because scikit-learn dropped min_impurity_split hyperparameter in 1.0,
         # and the requested flow is from 1.0.0 exactly.
     )
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_get_flow_reinstantiate_flow_not_strict_post_1(self):
         self.use_production_server()
         flow = openml.flows.get_flow(flow_id=19190, reinstantiate=True, strict_version=False)
@@ -373,7 +373,7 @@ def test_get_flow_reinstantiate_flow_not_strict_post_1(self):
         reason="Requires scikit-learn 0.23.2 or ~0.24.",
         # Because these still have min_impurity_split, but with new scikit-learn module structure."
     )
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_get_flow_reinstantiate_flow_not_strict_023_and_024(self):
         self.use_production_server()
         flow = openml.flows.get_flow(flow_id=18587, reinstantiate=True, strict_version=False)
@@ -385,7 +385,7 @@ def test_get_flow_reinstantiate_flow_not_strict_023_and_024(self):
         Version(sklearn.__version__) > Version("0.23"),
         reason="Requires scikit-learn<=0.23, because the scikit-learn module structure changed.",
     )
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_get_flow_reinstantiate_flow_not_strict_pre_023(self):
         self.use_production_server()
         flow = openml.flows.get_flow(flow_id=8175, reinstantiate=True, strict_version=False)

tests/test_openml/test_config.py

@@ -106,7 +106,7 @@ def test_setup_with_config(self):
 
 
 class TestConfigurationForExamples(openml.testing.TestBase):
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_switch_to_example_configuration(self):
         """Verifies the test configuration is loaded properly."""
         # Below is the default test key which would be used anyway, but just for clarity:
@@ -118,7 +118,7 @@ def test_switch_to_example_configuration(self):
         assert openml.config.apikey == TestBase.user_key
         assert openml.config.server == self.test_server
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_switch_from_example_configuration(self):
         """Verifies the previous configuration is loaded after stopping."""
         # Below is the default test key which would be used anyway, but just for clarity:
@@ -143,7 +143,7 @@ def test_example_configuration_stop_before_start(self):
             openml.config.stop_using_configuration_for_example,
         )
 
-    @pytest.mark.production()
+    @pytest.mark.production_server()
     def test_example_configuration_start_twice(self):
         """Checks that the original config can be returned to if `start..` is called twice."""
         openml.config.apikey = TestBase.user_key