From 07f2fa6cddeb26a1fdeb88f478f2fa494177bf6d Mon Sep 17 00:00:00 2001 From: Konboi Date: Wed, 23 Jul 2025 17:38:11 +0900 Subject: [PATCH 01/30] to be able to check that pts v2 is enabled or not --- launchable/utils/launchable_client.py | 30 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/launchable/utils/launchable_client.py b/launchable/utils/launchable_client.py index 0f5347814..b523ff2b4 100644 --- a/launchable/utils/launchable_client.py +++ b/launchable/utils/launchable_client.py @@ -30,7 +30,7 @@ def __init__(self, tracking_client: Optional[TrackingClient] = None, base_url: s "Confirm that you set LAUNCHABLE_TOKEN " "(or LAUNCHABLE_ORGANIZATION and LAUNCHABLE_WORKSPACE) environment variable(s)\n" "See https://docs.launchableinc.com/getting-started#setting-your-api-key") - self._workspace_state_cache: dict = {} + self._workspace_state_cache: Optional[Dict] = None def request( self, @@ -102,15 +102,29 @@ def print_exception_and_recover(self, e: Exception, warning: Optional[str] = Non click.echo(click.style(warning, fg=warning_color), err=True) def is_fail_fast_mode(self) -> bool: - if 'fail_fast_mode' in self._workspace_state_cache: - return self._workspace_state_cache['fail_fast_mode'] - # TODO: call api and set the result to cache + state = self._get_workspace_state() + return state.get('fail_fast_mode', False) + + def is_enabled_pts_v2(self) -> bool: + state = self._get_workspace_state() + return state.get('state', "") == "HANDS_ON_LAB_V2" + + def _get_workspace_state(self) -> dict: + """ + Get the current state of the workspace. + """ + if self._workspace_state_cache is not None: + return self._workspace_state_cache try: res = self.request("get", "state") if res.status_code == 200: - self._workspace_state_cache['fail_fast_mode'] = res.json().get('isFailFastMode', False) - return self._workspace_state_cache['fail_fast_mode'] + state = res.json() + self._workspace_state_cache = { + 'state': state.get('state', ""), + 'fail_fast_mode': state.get('isFailFastMode', False) + } + return self._workspace_state_cache except Exception as e: - self.print_exception_and_recover(e, "Failed to check fail-fast mode status") + self.print_exception_and_recover(e, "Failed to get workspace state") - return False + return {} From d66f829b9711f019942abf0f6b2c516ff8ec097b Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 24 Jul 2025 09:39:14 +0900 Subject: [PATCH 02/30] if the workspace is enabled pts v2 and input is zero, the cli enables the zero input subset automatically --- launchable/commands/subset.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 2426ee165..db08e9ace 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -529,12 +529,18 @@ def get_payload( def run(self): """called after tests are scanned to compute the optimized order""" if not is_get_tests_from_previous_sessions and len(self.test_paths) == 0: + msg = None if self.input_given: msg = "ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent." # noqa E501 + elif client.is_enabled_pts_v2(): + click.echo("INFO: Subset input is empty, enabling `--get-tests-from-previous-sessions` option", err=True) + self.is_get_tests_from_previous_sessions = True else: msg = "ERROR: Expecting tests to be given, but none provided. See https://www.launchableinc.com/docs/features/predictive-test-selection/requesting-and-running-a-subset-of-tests/subsetting-with-the-launchable-cli/ and provide ones, or use the `--get-tests-from-previous-sessions` option" # noqa E501 - click.echo(click.style(msg, fg="red"), err=True) - exit(1) + + if msg: + click.echo(click.style(msg, fg="red"), err=True) + exit(1) # When Error occurs, return the test name as it is passed. original_subset = self.test_paths From fafe161e2d6607364eb708e5c34ceef876775e88 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 24 Jul 2025 09:40:44 +0900 Subject: [PATCH 03/30] add test case --- tests/commands/test_subset.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/commands/test_subset.py b/tests/commands/test_subset.py index 247ec2c67..d720d238d 100644 --- a/tests/commands/test_subset.py +++ b/tests/commands/test_subset.py @@ -547,3 +547,26 @@ def test_subset_prioritize_tests_failed_within_hours(self): payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) self.assertEqual(payload.get('hoursToPrioritizeFailedTest'), 24) + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_subset_enable_get_tests_from_previous_sessions_when_pts_v2_enabled(self): + responses.replace( + responses.GET, + "{}/intake/organizations/{}/workspaces/{}/state".format( + get_base_url(), + self.organization, + self.workspace), + json={"state": 'HANDS_ON_LAB_V2', "isFailFastMode": False}, + status=200) + + result = self.cli( + "subset", + "--session", + self.session, + "file", + ) + + self.assert_success(result) + payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) + self.assertEqual(payload.get("getTestsFromPreviousSessions"), True) From c0bfd63fc173efdb3ec0508adce17d56e5537fc1 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 24 Jul 2025 09:41:25 +0900 Subject: [PATCH 04/30] use instance field --- launchable/commands/subset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index db08e9ace..5521d97d8 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -489,7 +489,7 @@ def get_payload( "id": os.path.basename(session_id) }, "ignoreNewTests": ignore_new_tests, - "getTestsFromPreviousSessions": is_get_tests_from_previous_sessions, + "getTestsFromPreviousSessions": self.is_get_tests_from_previous_sessions, } if target is not None: @@ -613,7 +613,7 @@ def run(self): else: output_subset, output_rests = original_subset, original_rests - if is_observation: + if subset_result.is_observation: output_subset = output_subset + output_rests output_rests = [] From c41d9a6270d448168e722155b25ac04b7bdad5fa Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 24 Jul 2025 17:26:02 +0900 Subject: [PATCH 05/30] introduce SubsetResult class to handle subset result --- launchable/commands/subset.py | 179 +++++++++++++++++++++------------- 1 file changed, 112 insertions(+), 67 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 5521d97d8..9c8b4ddb1 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -526,9 +526,65 @@ def get_payload( return payload + def _get_test_files(self): + LOOSE_TEST_FILE_PATTERN = r'(\.(test|spec)\.|_test\.|Test\.|Spec\.|test/|tests/|__tests__/|src/test/)' + EXCLUDE_PATTERN = r'\.(xml|json|txt|yml|yaml|md)$' + + git_managed_files = subprocess.run(['git', 'ls-files'], capture_output=True, text=True).stdout.strip().split('\n') + + git_managed_test_files = [] + for f in git_managed_files: + if re.search(LOOSE_TEST_FILE_PATTERN, f) and not re.search(EXCLUDE_PATTERN, f): + git_managed_test_files.append(self.to_test_path(f)) + + self.test_paths = git_managed_test_files + self.is_get_tests_from_previous_sessions = False + + + def request_subset(self) -> SubsetResult: + test_runner = context.invoked_subcommand + # temporarily extend the timeout because subset API response has become slow + # TODO: remove this line when API response return respose + # within 300 sec + timeout = (5, 300) + payload = self.get_payload(session_id, target, duration, test_runner) + + if is_non_blocking: + # Create a new process for requesting a subset. + process = Process(target=subset_request, args=(client, timeout, payload)) + process.start() + click.echo("The subset was requested in non-blocking mode.", err=True) + self.output_handler(self.test_paths, []) + sys.exit(0) + + try: + res = subset_request(client=client, timeout=timeout, payload=payload) + # The status code 422 is returned when validation error of the test mapping file occurs. + if res.status_code == 422: + msg = "Error: {}".format(res.reason) + tracking_client.send_error_event( + event_name=Tracking.ErrorEvent.USER_ERROR, + stack_trace=msg, + ) + click.echo( + click.style(msg, fg="red"), + err=True) + sys.exit(1) + + return SubsetResult.from_response(res.json()) + except Exception as e: + tracking_client.send_error_event( + event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, + stack_trace=str(e), + ) + client.print_exception_and_recover( + e, "Warning: the service failed to subset. Falling back to running all tests") + return SubsetResult.from_test_paths(self.test_paths) + + def run(self): """called after tests are scanned to compute the optimized order""" - if not is_get_tests_from_previous_sessions and len(self.test_paths) == 0: + if not self.is_get_tests_from_previous_sessions and len(self.test_paths) == 0: msg = None if self.input_given: msg = "ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent." # noqa E501 @@ -543,75 +599,22 @@ def run(self): exit(1) # When Error occurs, return the test name as it is passed. - original_subset = self.test_paths - original_rests = [] - summary = {} - subset_id = "" - is_brainless = False - is_observation = False - + subset_result = None if not session_id: # Session ID in --session is missing. It might be caused by # Launchable API errors. - pass + subset_result = SubsetResult.from_test_paths(self.test_paths) else: - try: - test_runner = context.invoked_subcommand - - # temporarily extend the timeout because subset API response has become slow - # TODO: remove this line when API response return respose - # within 300 sec - timeout = (5, 300) - payload = self.get_payload(session_id, target, duration, test_runner) - - if is_non_blocking: - # Create a new process for requesting a subset. - process = Process(target=subset_request, args=(client, timeout, payload)) - process.start() - click.echo("The subset was requested in non-blocking mode.", err=True) - self.output_handler(self.test_paths, []) - return - - res = subset_request(client=client, timeout=timeout, payload=payload) - - # The status code 422 is returned when validation error of the test mapping file occurs. - if res.status_code == 422: - msg = "Error: {}".format(res.reason) - tracking_client.send_error_event( - event_name=Tracking.ErrorEvent.USER_ERROR, - stack_trace=msg, - ) - click.echo( - click.style(msg, fg="red"), - err=True) - sys.exit(1) - - res.raise_for_status() - - original_subset = res.json().get("testPaths", []) - original_rests = res.json().get("rest", []) - subset_id = res.json().get("subsettingId", 0) - summary = res.json().get("summary", {}) - is_brainless = res.json().get("isBrainless", False) - is_observation = res.json().get("isObservation", False) - - except Exception as e: - tracking_client.send_error_event( - event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, - stack_trace=str(e), - ) - - client.print_exception_and_recover( - e, "Warning: the service failed to subset. Falling back to running all tests") + subset_result = self.request_subset() - if len(original_subset) == 0: + if len(subset_result.subset) == 0: warn_and_exit_if_fail_fast_mode("Error: no tests found matching the path.") return if split: - click.echo("subset/{}".format(subset_id)) + click.echo("subset/{}".format(subset_result.subset_id)) else: - output_subset, output_rests = original_subset, original_rests + output_subset, output_rests = subset_result.subset, subset_result.rest if subset_result.is_observation: output_subset = output_subset + output_rests @@ -624,7 +627,10 @@ def run(self): # When Launchable returns an error, the cli skips showing summary # report - if "subset" not in summary.keys() or "rest" not in summary.keys(): + original_subset = subset_result.subset + original_rest = subset_result.rest + summary = subset_result.summary + if "subset" not in summary.keys() or "rest" not in sum.keys(): return build_name, test_session_id = parse_session(session_id) @@ -641,32 +647,32 @@ def run(self): ], [ "Remainder", - len(original_rests), + len(original_rest), summary["rest"].get("rate", 0.0), summary["rest"].get("duration", 0.0), ], [], [ "Total", - len(original_subset) + len(original_rests), + len(original_subset) + len(original_rest), summary["subset"].get("rate", 0.0) + summary["rest"].get("rate", 0.0), summary["subset"].get("duration", 0.0) + summary["rest"].get("duration", 0.0), ], ] - if is_brainless: + if subset_result.is_brainless: click.echo( "Your model is currently in training", err=True) click.echo( "Launchable created subset {} for build {} (test session {}) in workspace {}/{}".format( - subset_id, + subset_result.subset_id, build_name, test_session_id, org, workspace, ), err=True, ) - if is_observation: + if subset_result.is_observation: click.echo( "(This test session is under observation mode)", err=True) @@ -675,7 +681,7 @@ def run(self): click.echo(tabulate(rows, header, tablefmt="github", floatfmt=".2f"), err=True) click.echo( - "\nRun `launchable inspect subset --subset-id {}` to view full subset details".format(subset_id), + "\nRun `launchable inspect subset --subset-id {}` to view full subset details".format(subset_result.subset_id), err=True) context.obj = Optimize(app=context.obj) @@ -683,3 +689,42 @@ def run(self): def subset_request(client: LaunchableClient, timeout: Tuple[int, int], payload: Dict[str, Any]): return client.request("post", "subset", timeout=timeout, payload=payload, compress=True) + + +class SubsetResult: + def __init__( + self, + subset: List[TestPath] = None, + rest: List[TestPath] = None, + subset_id: str = "", + summary: Dict[str, Any] = None, + is_brainless: bool = False, + is_observation: bool = False): + self.subset = subset or [] + self.rest = rest or [] + self.subset_id = subset_id + self.summary = summary or {} + self.is_brainless = is_brainless + self.is_observation = is_observation + + @classmethod + def from_response(cls, response: Dict[str, Any]) -> 'SubsetResult': + return cls( + subset=response.get("testPaths", []), + rest=response.get("rest", []), + subset_id=response.get("subsettingId", ""), + summary=response.get("summary", {}), + is_brainless=response.get("isBrainless", False), + is_observation=response.get("isObservation", False) + ) + + @classmethod + def from_test_paths(cls, test_paths: List[List[Dict[str, str]]]) -> 'SubsetResult': + return cls( + subset=test_paths, + rest=[], + subset_id='', + summary={}, + is_brainless=False, + is_observation=False + ) From 61cce47f60782dde684e5c0249ee005d0c2e4be1 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 24 Jul 2025 17:38:45 +0900 Subject: [PATCH 06/30] introduce auto test file like collecting logic for hands_on_v2 workspaces --- launchable/commands/subset.py | 38 ++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 9c8b4ddb1..60042de23 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -2,6 +2,8 @@ import json import os import pathlib +import re +import subprocess import sys from multiprocessing import Process from os.path import join @@ -416,7 +418,7 @@ def rel_base_path(path): def stdin(self) -> Union[TextIO, List]: # To avoid the cli continue to wait from stdin - if is_get_tests_from_previous_sessions: + if self.is_get_tests_from_previous_sessions: return [] """ @@ -426,6 +428,12 @@ def stdin(self) -> Union[TextIO, List]: they didn't feed anything from stdin """ if sys.stdin.isatty(): + """ + Note: If pts v2 (state if HANDS_ON_LAB_V2), we allow empty stdin + """ + if client.is_enabled_pts_v2(): + return [] + warn_and_exit_if_fail_fast_mode( "Warning: this command reads from stdin but it doesn't appear to be connected to anything. " "Did you forget to pipe from another command?" @@ -526,7 +534,7 @@ def get_payload( return payload - def _get_test_files(self): + def _collect_test_like_files(self): LOOSE_TEST_FILE_PATTERN = r'(\.(test|spec)\.|_test\.|Test\.|Spec\.|test/|tests/|__tests__/|src/test/)' EXCLUDE_PATTERN = r'\.(xml|json|txt|yml|yaml|md)$' @@ -540,14 +548,13 @@ def _get_test_files(self): self.test_paths = git_managed_test_files self.is_get_tests_from_previous_sessions = False - def request_subset(self) -> SubsetResult: test_runner = context.invoked_subcommand # temporarily extend the timeout because subset API response has become slow # TODO: remove this line when API response return respose # within 300 sec timeout = (5, 300) - payload = self.get_payload(session_id, target, duration, test_runner) + payload = self.get_payload(str(session_id), target, duration, str(test_runner)) if is_non_blocking: # Create a new process for requesting a subset. @@ -581,7 +588,6 @@ def request_subset(self) -> SubsetResult: e, "Warning: the service failed to subset. Falling back to running all tests") return SubsetResult.from_test_paths(self.test_paths) - def run(self): """called after tests are scanned to compute the optimized order""" if not self.is_get_tests_from_previous_sessions and len(self.test_paths) == 0: @@ -607,6 +613,14 @@ def run(self): else: subset_result = self.request_subset() + """ + If the subset response is empty and the workspace is enabled to PTS v2, + CLI will try to collect the test files automatically, and request the subset again. + """ + if len(subset_result.subset) == 0 and client.is_enabled_pts_v2(): + self._collect_test_like_files() + subset_result = self.request_subset() + if len(subset_result.subset) == 0: warn_and_exit_if_fail_fast_mode("Error: no tests found matching the path.") return @@ -630,7 +644,7 @@ def run(self): original_subset = subset_result.subset original_rest = subset_result.rest summary = subset_result.summary - if "subset" not in summary.keys() or "rest" not in sum.keys(): + if "subset" not in summary.keys() or "rest" not in summary.keys(): return build_name, test_session_id = parse_session(session_id) @@ -694,16 +708,16 @@ def subset_request(client: LaunchableClient, timeout: Tuple[int, int], payload: class SubsetResult: def __init__( self, - subset: List[TestPath] = None, - rest: List[TestPath] = None, + subset: List[TestPath] = [], + rest: List[TestPath] = [], subset_id: str = "", - summary: Dict[str, Any] = None, + summary: Dict[str, Any] = {}, is_brainless: bool = False, is_observation: bool = False): - self.subset = subset or [] - self.rest = rest or [] + self.subset = subset + self.rest = rest self.subset_id = subset_id - self.summary = summary or {} + self.summary = summary self.is_brainless = is_brainless self.is_observation = is_observation From 4a347de3b493ee1516d2d1ca413128dd1f95d2bd Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 24 Jul 2025 18:05:26 +0900 Subject: [PATCH 07/30] fix type --- launchable/commands/subset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 60042de23..38c419afe 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -733,7 +733,7 @@ def from_response(cls, response: Dict[str, Any]) -> 'SubsetResult': ) @classmethod - def from_test_paths(cls, test_paths: List[List[Dict[str, str]]]) -> 'SubsetResult': + def from_test_paths(cls, test_paths: List[TestPath]) -> 'SubsetResult': return cls( subset=test_paths, rest=[], From 86093c96f5ad0ce82845ff3da7fa9e206a145370 Mon Sep 17 00:00:00 2001 From: Konboi Date: Thu, 24 Jul 2025 19:43:19 +0900 Subject: [PATCH 08/30] to support 3.6 --- launchable/commands/subset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 38c419afe..4684d3f31 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -538,7 +538,8 @@ def _collect_test_like_files(self): LOOSE_TEST_FILE_PATTERN = r'(\.(test|spec)\.|_test\.|Test\.|Spec\.|test/|tests/|__tests__/|src/test/)' EXCLUDE_PATTERN = r'\.(xml|json|txt|yml|yaml|md)$' - git_managed_files = subprocess.run(['git', 'ls-files'], capture_output=True, text=True).stdout.strip().split('\n') + git_managed_files = subprocess.run(['git', 'ls-files'], stdout=subprocess.PIPE, + universal_newlines=True).stdout.strip().split('\n') git_managed_test_files = [] for f in git_managed_files: From f566f6fe616836ba56352914d314e4b7be215048 Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 25 Jul 2025 09:21:25 +0900 Subject: [PATCH 09/30] add error handling --- launchable/commands/subset.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 4684d3f31..3388c08a1 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -538,8 +538,16 @@ def _collect_test_like_files(self): LOOSE_TEST_FILE_PATTERN = r'(\.(test|spec)\.|_test\.|Test\.|Spec\.|test/|tests/|__tests__/|src/test/)' EXCLUDE_PATTERN = r'\.(xml|json|txt|yml|yaml|md)$' - git_managed_files = subprocess.run(['git', 'ls-files'], stdout=subprocess.PIPE, - universal_newlines=True).stdout.strip().split('\n') + git_managed_files = [] + try: + git_managed_files = subprocess.run(['git', 'ls-files'], stdout=subprocess.PIPE, + universal_newlines=True).stdout.strip().split('\n') + except subprocess.CalledProcessError: + click.echo( + click.style( + "Warning: git ls-files failed. Falling back to globbing.", + fg="yellow"), + err=True) git_managed_test_files = [] for f in git_managed_files: From 728c70737a1147e9407c96101019d8d60a41e81b Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 25 Jul 2025 09:21:41 +0900 Subject: [PATCH 10/30] fix comments --- launchable/commands/subset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 3388c08a1..4564b5624 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -560,7 +560,7 @@ def _collect_test_like_files(self): def request_subset(self) -> SubsetResult: test_runner = context.invoked_subcommand # temporarily extend the timeout because subset API response has become slow - # TODO: remove this line when API response return respose + # TODO: remove this line when API response return response # within 300 sec timeout = (5, 300) payload = self.get_payload(str(session_id), target, duration, str(test_runner)) @@ -571,6 +571,7 @@ def request_subset(self) -> SubsetResult: process.start() click.echo("The subset was requested in non-blocking mode.", err=True) self.output_handler(self.test_paths, []) + # With non-blocking mode, we don't need to wait for the response sys.exit(0) try: From 4f7323dfa47b37fbf8c0e1843ebcd2d7cd549462 Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 25 Jul 2025 09:24:25 +0900 Subject: [PATCH 11/30] to check more strictly --- launchable/utils/launchable_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchable/utils/launchable_client.py b/launchable/utils/launchable_client.py index b523ff2b4..637edb860 100644 --- a/launchable/utils/launchable_client.py +++ b/launchable/utils/launchable_client.py @@ -30,7 +30,7 @@ def __init__(self, tracking_client: Optional[TrackingClient] = None, base_url: s "Confirm that you set LAUNCHABLE_TOKEN " "(or LAUNCHABLE_ORGANIZATION and LAUNCHABLE_WORKSPACE) environment variable(s)\n" "See https://docs.launchableinc.com/getting-started#setting-your-api-key") - self._workspace_state_cache: Optional[Dict] = None + self._workspace_state_cache: Optional[Dict[str, Union[str, bool]]] = None def request( self, From 707f0c1cff8e54416cf7d88e4d98402a0e4aafcb Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 25 Jul 2025 10:14:53 +0900 Subject: [PATCH 12/30] add test case for pts v2 usecase --- launchable/commands/subset.py | 4 +--- tests/commands/test_subset.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 4564b5624..98be31c3f 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -549,12 +549,10 @@ def _collect_test_like_files(self): fg="yellow"), err=True) - git_managed_test_files = [] for f in git_managed_files: if re.search(LOOSE_TEST_FILE_PATTERN, f) and not re.search(EXCLUDE_PATTERN, f): - git_managed_test_files.append(self.to_test_path(f)) + self.test_paths.append(self.to_test_path(f)) - self.test_paths = git_managed_test_files self.is_get_tests_from_previous_sessions = False def request_subset(self) -> SubsetResult: diff --git a/tests/commands/test_subset.py b/tests/commands/test_subset.py index d720d238d..98e9631cc 100644 --- a/tests/commands/test_subset.py +++ b/tests/commands/test_subset.py @@ -557,8 +557,28 @@ def test_subset_enable_get_tests_from_previous_sessions_when_pts_v2_enabled(self get_base_url(), self.organization, self.workspace), - json={"state": 'HANDS_ON_LAB_V2', "isFailFastMode": False}, + json={"state": 'HANDS_ON_LAB_V2', "isFailFastMode": True}, status=200) + """ + 1. Returns 400 error since the zero input test paths isn't calculated yet + 2. Retruns 200 OK with the test paths from auto collection + """ + responses.replace( + responses.POST, + "{}/intake/organizations/{}/workspaces/{}/subset".format( + get_base_url(), + self.organization, + self.workspace), + json=[{}, { + "testPaths": [ + [{"type": "file", "name": "tests/commands/test_subset.py"}], + ], + "testRunner": "file", + "rest": [], + "subsettingId": 123, + }], + status=[400, 200] + ) result = self.cli( "subset", @@ -568,5 +588,16 @@ def test_subset_enable_get_tests_from_previous_sessions_when_pts_v2_enabled(self ) self.assert_success(result) + + """ + 1. request to /state + 2. request to /subset with getTestsFromPreviousSessions = True + 3, 4. request to cli_tracking + 4. request to /subset with getTestsFromPreviousSessions = False and testPaths from auto collection + """ payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) self.assertEqual(payload.get("getTestsFromPreviousSessions"), True) + + payload = json.loads(gzip.decompress(responses.calls[4].request.body).decode()) + self.assertEqual(payload.get("getTestsFromPreviousSessions"), False) + self.assertIn([{"type": "file", "name": "tests/commands/test_subset.py"}], payload.get("testPaths", [])) From c36d308b6a325d1c0ea67b60f728989a7568f08b Mon Sep 17 00:00:00 2001 From: Konboi Date: Fri, 25 Jul 2025 11:50:33 +0900 Subject: [PATCH 13/30] fix comments --- launchable/commands/subset.py | 2 +- tests/commands/test_subset.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 98be31c3f..6f4e5d085 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -545,7 +545,7 @@ def _collect_test_like_files(self): except subprocess.CalledProcessError: click.echo( click.style( - "Warning: git ls-files failed. Falling back to globbing.", + "Warning: git ls-files failed.", fg="yellow"), err=True) diff --git a/tests/commands/test_subset.py b/tests/commands/test_subset.py index 98e9631cc..d96495eab 100644 --- a/tests/commands/test_subset.py +++ b/tests/commands/test_subset.py @@ -561,7 +561,7 @@ def test_subset_enable_get_tests_from_previous_sessions_when_pts_v2_enabled(self status=200) """ 1. Returns 400 error since the zero input test paths isn't calculated yet - 2. Retruns 200 OK with the test paths from auto collection + 2. Returns 200 OK with the test paths from auto collection """ responses.replace( responses.POST, @@ -593,7 +593,7 @@ def test_subset_enable_get_tests_from_previous_sessions_when_pts_v2_enabled(self 1. request to /state 2. request to /subset with getTestsFromPreviousSessions = True 3, 4. request to cli_tracking - 4. request to /subset with getTestsFromPreviousSessions = False and testPaths from auto collection + 5. request to /subset with getTestsFromPreviousSessions = False and testPaths from auto collection """ payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) self.assertEqual(payload.get("getTestsFromPreviousSessions"), True) From aecfbbff77cc7275c05643c4646fc4729576a461 Mon Sep 17 00:00:00 2001 From: Konboi Date: Mon, 28 Jul 2025 17:12:32 +0900 Subject: [PATCH 14/30] rename the method to make it easiter to understand --- launchable/commands/subset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 6f4e5d085..d19d2a67e 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -534,7 +534,7 @@ def get_payload( return payload - def _collect_test_like_files(self): + def _collect_potential_test_files(self): LOOSE_TEST_FILE_PATTERN = r'(\.(test|spec)\.|_test\.|Test\.|Spec\.|test/|tests/|__tests__/|src/test/)' EXCLUDE_PATTERN = r'\.(xml|json|txt|yml|yaml|md)$' @@ -626,7 +626,7 @@ def run(self): CLI will try to collect the test files automatically, and request the subset again. """ if len(subset_result.subset) == 0 and client.is_enabled_pts_v2(): - self._collect_test_like_files() + self._collect_potential_test_files() subset_result = self.request_subset() if len(subset_result.subset) == 0: From 6b745eac29fa3a762e78cc3666e57a8e3ec31e44 Mon Sep 17 00:00:00 2001 From: Konboi Date: Mon, 28 Jul 2025 17:22:06 +0900 Subject: [PATCH 15/30] if test paths is 0 after collecting test files from git history, do not request to the subset again --- launchable/commands/subset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index d19d2a67e..16e45bc93 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -627,7 +627,8 @@ def run(self): """ if len(subset_result.subset) == 0 and client.is_enabled_pts_v2(): self._collect_potential_test_files() - subset_result = self.request_subset() + if len(self.test_paths) > 0: + subset_result = self.request_subset() if len(subset_result.subset) == 0: warn_and_exit_if_fail_fast_mode("Error: no tests found matching the path.") From 1a87bf90354333940a266b0a90f11ba8132bed90 Mon Sep 17 00:00:00 2001 From: Konboi Date: Mon, 28 Jul 2025 17:22:50 +0900 Subject: [PATCH 16/30] collect test files from git history only if the zero input subset is enabled and pts v2 is enabled --- launchable/commands/subset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 16e45bc93..416da7067 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -622,10 +622,10 @@ def run(self): subset_result = self.request_subset() """ - If the subset response is empty and the workspace is enabled to PTS v2, + If the zero input subset response is empty and the workspace is enabled to PTS v2, CLI will try to collect the test files automatically, and request the subset again. """ - if len(subset_result.subset) == 0 and client.is_enabled_pts_v2(): + if client.is_enabled_pts_v2() and self.is_get_tests_from_previous_sessions and len(subset_result.subset) == 0: self._collect_potential_test_files() if len(self.test_paths) > 0: subset_result = self.request_subset() From f91fc49518e3c99a9f8c966826eed251f206045e Mon Sep 17 00:00:00 2001 From: Konboi Date: Mon, 28 Jul 2025 17:32:02 +0900 Subject: [PATCH 17/30] use raise_for_status instead of checking status code is 200 --- launchable/utils/launchable_client.py | 15 ++++++++------- tests/commands/test_api_error.py | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/launchable/utils/launchable_client.py b/launchable/utils/launchable_client.py index 637edb860..518e3895c 100644 --- a/launchable/utils/launchable_client.py +++ b/launchable/utils/launchable_client.py @@ -117,13 +117,14 @@ def _get_workspace_state(self) -> dict: return self._workspace_state_cache try: res = self.request("get", "state") - if res.status_code == 200: - state = res.json() - self._workspace_state_cache = { - 'state': state.get('state', ""), - 'fail_fast_mode': state.get('isFailFastMode', False) - } - return self._workspace_state_cache + res.raise_for_status() + + state = res.json() + self._workspace_state_cache = { + 'state': state.get('state', ""), + 'fail_fast_mode': state.get('isFailFastMode', False) + } + return self._workspace_state_cache except Exception as e: self.print_exception_and_recover(e, "Failed to get workspace state") diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index 97fdff4ee..9570591a9 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -454,12 +454,12 @@ def test_all_workflow_when_server_down(self): self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=9) + self.assert_tracking_count(tracking=tracking, count=10) result = self.cli("record", "tests", "--session", self.session, "minitest", str(self.test_files_dir) + "/") self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=13) + self.assert_tracking_count(tracking=tracking, count=14) def assert_tracking_count(self, tracking, count: int): # Prior to 3.6, `Response` object can't be obtained. From fd568e8e8ecadce3c5176be42576e13406e4527a Mon Sep 17 00:00:00 2001 From: Konboi Date: Tue, 29 Jul 2025 09:22:43 +0900 Subject: [PATCH 18/30] rename method name --- launchable/commands/subset.py | 6 +++--- launchable/utils/launchable_client.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 416da7067..329103767 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -431,7 +431,7 @@ def stdin(self) -> Union[TextIO, List]: """ Note: If pts v2 (state if HANDS_ON_LAB_V2), we allow empty stdin """ - if client.is_enabled_pts_v2(): + if client.is_pts_v2_enabled(): return [] warn_and_exit_if_fail_fast_mode( @@ -602,7 +602,7 @@ def run(self): msg = None if self.input_given: msg = "ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent." # noqa E501 - elif client.is_enabled_pts_v2(): + elif client.is_pts_v2_enabled(): click.echo("INFO: Subset input is empty, enabling `--get-tests-from-previous-sessions` option", err=True) self.is_get_tests_from_previous_sessions = True else: @@ -625,7 +625,7 @@ def run(self): If the zero input subset response is empty and the workspace is enabled to PTS v2, CLI will try to collect the test files automatically, and request the subset again. """ - if client.is_enabled_pts_v2() and self.is_get_tests_from_previous_sessions and len(subset_result.subset) == 0: + if client.is_pts_v2_enabled() and self.is_get_tests_from_previous_sessions and len(subset_result.subset) == 0: self._collect_potential_test_files() if len(self.test_paths) > 0: subset_result = self.request_subset() diff --git a/launchable/utils/launchable_client.py b/launchable/utils/launchable_client.py index 518e3895c..d00fd243a 100644 --- a/launchable/utils/launchable_client.py +++ b/launchable/utils/launchable_client.py @@ -105,7 +105,7 @@ def is_fail_fast_mode(self) -> bool: state = self._get_workspace_state() return state.get('fail_fast_mode', False) - def is_enabled_pts_v2(self) -> bool: + def is_pts_v2_enabled(self) -> bool: state = self._get_workspace_state() return state.get('state', "") == "HANDS_ON_LAB_V2" From eec70c00e10f7fec43679f00f478eef9821a5fd3 Mon Sep 17 00:00:00 2001 From: Konboi Date: Tue, 29 Jul 2025 09:23:22 +0900 Subject: [PATCH 19/30] rm unnecessary code --- launchable/commands/subset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 329103767..0b72e58a4 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -553,8 +553,6 @@ def _collect_potential_test_files(self): if re.search(LOOSE_TEST_FILE_PATTERN, f) and not re.search(EXCLUDE_PATTERN, f): self.test_paths.append(self.to_test_path(f)) - self.is_get_tests_from_previous_sessions = False - def request_subset(self) -> SubsetResult: test_runner = context.invoked_subcommand # temporarily extend the timeout because subset API response has become slow From 09360e274d6abbdf0bd89ebcf161621a94fd2bac Mon Sep 17 00:00:00 2001 From: Konboi Date: Tue, 29 Jul 2025 09:47:14 +0900 Subject: [PATCH 20/30] don't use state directly --- launchable/utils/launchable_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/launchable/utils/launchable_client.py b/launchable/utils/launchable_client.py index d00fd243a..135ba4869 100644 --- a/launchable/utils/launchable_client.py +++ b/launchable/utils/launchable_client.py @@ -107,7 +107,7 @@ def is_fail_fast_mode(self) -> bool: def is_pts_v2_enabled(self) -> bool: state = self._get_workspace_state() - return state.get('state', "") == "HANDS_ON_LAB_V2" + return state.get('pts_v2', False) def _get_workspace_state(self) -> dict: """ @@ -121,8 +121,8 @@ def _get_workspace_state(self) -> dict: state = res.json() self._workspace_state_cache = { - 'state': state.get('state', ""), - 'fail_fast_mode': state.get('isFailFastMode', False) + 'fail_fast_mode': state.get('isFailFastMode', False), + 'pts_v2': state.get('isPtsV2Enabled', False), } return self._workspace_state_cache except Exception as e: From e3abd46bade32e076cc365385a686ed3fbb6faca Mon Sep 17 00:00:00 2001 From: Konboi Date: Tue, 29 Jul 2025 10:06:00 +0900 Subject: [PATCH 21/30] need `check` to catch exception --- launchable/commands/subset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 0b72e58a4..4fea939a9 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -541,7 +541,7 @@ def _collect_potential_test_files(self): git_managed_files = [] try: git_managed_files = subprocess.run(['git', 'ls-files'], stdout=subprocess.PIPE, - universal_newlines=True).stdout.strip().split('\n') + universal_newlines=True, check=True).stdout.strip().split('\n') except subprocess.CalledProcessError: click.echo( click.style( From 5f1c05803e38472b0721fbe8898604b920da0766 Mon Sep 17 00:00:00 2001 From: Konboi Date: Tue, 29 Jul 2025 12:14:58 +0900 Subject: [PATCH 22/30] introduce --get-tests-from-guess option instead of fallback and auto discovery --- launchable/commands/subset.py | 33 ++++++++++++--------------------- tests/cli_test_case.py | 2 +- tests/commands/test_subset.py | 23 +++++++---------------- 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 4fea939a9..47dfed029 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -201,6 +201,12 @@ type=str, metavar='TEST_SUITE', ) +@click.option( + "--get-tests-from-guess", + "is_get_tests_from_guess", + help="get subset list from git managed files", + is_flag=True, +) @click.pass_context def subset( context: click.core.Context, @@ -228,6 +234,7 @@ def subset( prioritize_tests_failed_within_hours: Optional[int] = None, prioritized_tests_mapping_file: Optional[TextIO] = None, test_suite: Optional[str] = None, + is_get_tests_from_guess: bool = False, ): app = context.obj tracking_client = TrackingClient(Command.SUBSET, app=app) @@ -387,6 +394,7 @@ def __init__(self, app: Application): self.exclusion_output_handler = self._default_exclusion_output_handler self.is_get_tests_from_previous_sessions = is_get_tests_from_previous_sessions self.is_output_exclusion_rules = is_output_exclusion_rules + self.is_get_tests_from_guess = is_get_tests_from_guess super(Optimize, self).__init__(app=app) def _default_output_handler(self, output: List[TestPath], rests: List[TestPath]): @@ -418,7 +426,7 @@ def rel_base_path(path): def stdin(self) -> Union[TextIO, List]: # To avoid the cli continue to wait from stdin - if self.is_get_tests_from_previous_sessions: + if self.is_get_tests_from_previous_sessions or self.is_get_tests_from_guess: return [] """ @@ -428,12 +436,6 @@ def stdin(self) -> Union[TextIO, List]: they didn't feed anything from stdin """ if sys.stdin.isatty(): - """ - Note: If pts v2 (state if HANDS_ON_LAB_V2), we allow empty stdin - """ - if client.is_pts_v2_enabled(): - return [] - warn_and_exit_if_fail_fast_mode( "Warning: this command reads from stdin but it doesn't appear to be connected to anything. " "Did you forget to pipe from another command?" @@ -595,14 +597,13 @@ def request_subset(self) -> SubsetResult: return SubsetResult.from_test_paths(self.test_paths) def run(self): + if self.is_get_tests_from_guess: + self._collect_potential_test_files() + """called after tests are scanned to compute the optimized order""" if not self.is_get_tests_from_previous_sessions and len(self.test_paths) == 0: - msg = None if self.input_given: msg = "ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent." # noqa E501 - elif client.is_pts_v2_enabled(): - click.echo("INFO: Subset input is empty, enabling `--get-tests-from-previous-sessions` option", err=True) - self.is_get_tests_from_previous_sessions = True else: msg = "ERROR: Expecting tests to be given, but none provided. See https://www.launchableinc.com/docs/features/predictive-test-selection/requesting-and-running-a-subset-of-tests/subsetting-with-the-launchable-cli/ and provide ones, or use the `--get-tests-from-previous-sessions` option" # noqa E501 @@ -611,7 +612,6 @@ def run(self): exit(1) # When Error occurs, return the test name as it is passed. - subset_result = None if not session_id: # Session ID in --session is missing. It might be caused by # Launchable API errors. @@ -619,15 +619,6 @@ def run(self): else: subset_result = self.request_subset() - """ - If the zero input subset response is empty and the workspace is enabled to PTS v2, - CLI will try to collect the test files automatically, and request the subset again. - """ - if client.is_pts_v2_enabled() and self.is_get_tests_from_previous_sessions and len(subset_result.subset) == 0: - self._collect_potential_test_files() - if len(self.test_paths) > 0: - subset_result = self.request_subset() - if len(subset_result.subset) == 0: warn_and_exit_if_fail_fast_mode("Error: no tests found matching the path.") return diff --git a/tests/cli_test_case.py b/tests/cli_test_case.py index e64e0f57d..e79611cc5 100644 --- a/tests/cli_test_case.py +++ b/tests/cli_test_case.py @@ -192,7 +192,7 @@ def setUp(self): get_base_url(), self.organization, self.workspace), - json={'isFailFastMode': False}, + json={'isFailFastMode': False, 'isPtsV2Enabled': False}, status=200) def get_test_files_dir(self): diff --git a/tests/commands/test_subset.py b/tests/commands/test_subset.py index d96495eab..d3670acfe 100644 --- a/tests/commands/test_subset.py +++ b/tests/commands/test_subset.py @@ -550,40 +550,37 @@ def test_subset_prioritize_tests_failed_within_hours(self): @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) - def test_subset_enable_get_tests_from_previous_sessions_when_pts_v2_enabled(self): + def test_subset_with_get_tests_from_guess(self): responses.replace( responses.GET, "{}/intake/organizations/{}/workspaces/{}/state".format( get_base_url(), self.organization, self.workspace), - json={"state": 'HANDS_ON_LAB_V2', "isFailFastMode": True}, + json={"state": 'HANDS_ON_LAB_V2', "isFailFastMode": True, "isPtsV2Enabled": True}, status=200) - """ - 1. Returns 400 error since the zero input test paths isn't calculated yet - 2. Returns 200 OK with the test paths from auto collection - """ responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), - json=[{}, { + json={ "testPaths": [ [{"type": "file", "name": "tests/commands/test_subset.py"}], ], "testRunner": "file", "rest": [], "subsettingId": 123, - }], - status=[400, 200] + }, + status=[200] ) result = self.cli( "subset", "--session", self.session, + "--get-tests-from-guess", "file", ) @@ -591,13 +588,7 @@ def test_subset_enable_get_tests_from_previous_sessions_when_pts_v2_enabled(self """ 1. request to /state - 2. request to /subset with getTestsFromPreviousSessions = True - 3, 4. request to cli_tracking - 5. request to /subset with getTestsFromPreviousSessions = False and testPaths from auto collection + 2. request to /subset with test paths that are collected from auto collection """ payload = json.loads(gzip.decompress(responses.calls[1].request.body).decode()) - self.assertEqual(payload.get("getTestsFromPreviousSessions"), True) - - payload = json.loads(gzip.decompress(responses.calls[4].request.body).decode()) - self.assertEqual(payload.get("getTestsFromPreviousSessions"), False) self.assertIn([{"type": "file", "name": "tests/commands/test_subset.py"}], payload.get("testPaths", [])) From e3fdfdf0cead87e07f672cf1967758707e2d6adf Mon Sep 17 00:00:00 2001 From: Konboi Date: Tue, 29 Jul 2025 13:22:54 +0900 Subject: [PATCH 23/30] fix api call count --- tests/commands/test_api_error.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index 9570591a9..97fdff4ee 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -454,12 +454,12 @@ def test_all_workflow_when_server_down(self): self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=10) + self.assert_tracking_count(tracking=tracking, count=9) result = self.cli("record", "tests", "--session", self.session, "minitest", str(self.test_files_dir) + "/") self.assert_success(result) # Since Timeout error is caught inside of LaunchableClient, the tracking event is sent twice. - self.assert_tracking_count(tracking=tracking, count=14) + self.assert_tracking_count(tracking=tracking, count=13) def assert_tracking_count(self, tracking, count: int): # Prior to 3.6, `Response` object can't be obtained. From 05ce7e46837d036cf6e34f514f971d09f85b211e Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 29 Jul 2025 10:17:42 -0700 Subject: [PATCH 24/30] [refactor] docstring must be at the top of a function --- launchable/commands/subset.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 47dfed029..d2f4da316 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -425,16 +425,17 @@ def rel_base_path(path): self.test_paths.append(self.to_test_path(rel_base_path(path))) def stdin(self) -> Union[TextIO, List]: - # To avoid the cli continue to wait from stdin - if self.is_get_tests_from_previous_sessions or self.is_get_tests_from_guess: - return [] - """ Returns sys.stdin, but after ensuring that it's connected to something reasonable. This prevents a typical problem where users think CLI is hanging because they didn't feed anything from stdin """ + + # To avoid the cli continue to wait from stdin + if self.is_get_tests_from_previous_sessions or self.is_get_tests_from_guess: + return [] + if sys.stdin.isatty(): warn_and_exit_if_fail_fast_mode( "Warning: this command reads from stdin but it doesn't appear to be connected to anything. " @@ -597,10 +598,11 @@ def request_subset(self) -> SubsetResult: return SubsetResult.from_test_paths(self.test_paths) def run(self): + """called after tests are scanned to compute the optimized order""" + if self.is_get_tests_from_guess: self._collect_potential_test_files() - """called after tests are scanned to compute the optimized order""" if not self.is_get_tests_from_previous_sessions and len(self.test_paths) == 0: if self.input_given: msg = "ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent." # noqa E501 From 4b0d59e601efcbcd623b4cb08ce5b9a67cd292a2 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 29 Jul 2025 10:22:51 -0700 Subject: [PATCH 25/30] [refactor] report and error and die is a common pattern This logic should be promoted further up ideally. --- launchable/commands/subset.py | 69 ++++++++++------------------------- 1 file changed, 20 insertions(+), 49 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index d2f4da316..b145f9c6c 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -255,6 +255,11 @@ def subset( test_suite=test_suite, )) + def print_error_and_die(msg: str, event: Tracking.ErrorEvent): + click.echo(click.style(msg, fg="red"), err=True) + tracking_client.send_error_event(event_name=event, stack_trace=msg) + sys.exit(1) + if is_observation and is_output_exclusion_rules: msg = ( "WARNING: --observation and --output-exclusion-rules are set. " @@ -273,21 +278,10 @@ def subset( if prioritize_tests_failed_within_hours is not None and prioritize_tests_failed_within_hours > 0: if ignore_new_tests or (ignore_flaky_tests_above is not None and ignore_flaky_tests_above > 0): - msg = ( - "Cannot use --ignore-new-tests or --ignore-flaky-tests-above options " - "with --prioritize-tests-failed-within-hours" - ) - click.echo( - click.style( - msg, - fg="red"), - err=True, - ) - tracking_client.send_error_event( - event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, - stack_trace=msg, + print_error_and_die( + "Cannot use --ignore-new-tests or --ignore-flaky-tests-above options with --prioritize-tests-failed-within-hours", + Tracking.ErrorEvent.INTERNAL_CLI_ERROR ) - sys.exit(1) if is_no_build and session: warn_and_exit_if_fail_fast_mode( @@ -320,14 +314,7 @@ def subset( test_suite=test_suite, ) except click.UsageError as e: - click.echo( - click.style( - str(e), - fg="red"), - err=True, - ) - tracking_client.send_error_event(event_name=Tracking.ErrorEvent.USER_ERROR, stack_trace=str(e)) - sys.exit(1) + print_error_and_die(str(e), Tracking.ErrorEvent.USER_ERROR) except Exception as e: tracking_client.send_error_event( event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, @@ -348,18 +335,9 @@ def subset( res = client.request("get", session_id) is_observation_in_recorded_session = res.json().get("isObservation", False) if not is_observation_in_recorded_session: - msg = "You have to specify --observation option to use non-blocking mode" - click.echo( - click.style( - msg, - fg="red"), - err=True, - ) - tracking_client.send_error_event( - event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, - stack_trace=msg, - ) - sys.exit(1) + print_error_and_die( + "You have to specify --observation option to use non-blocking mode", + Tracking.ErrorEvent.INTERNAL_CLI_ERROR) except Exception as e: tracking_client.send_error_event( event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR, @@ -577,15 +555,7 @@ def request_subset(self) -> SubsetResult: res = subset_request(client=client, timeout=timeout, payload=payload) # The status code 422 is returned when validation error of the test mapping file occurs. if res.status_code == 422: - msg = "Error: {}".format(res.reason) - tracking_client.send_error_event( - event_name=Tracking.ErrorEvent.USER_ERROR, - stack_trace=msg, - ) - click.echo( - click.style(msg, fg="red"), - err=True) - sys.exit(1) + print_error_and_die("Error: {}".format(res.reason), Tracking.ErrorEvent.USER_ERROR) return SubsetResult.from_response(res.json()) except Exception as e: @@ -605,13 +575,14 @@ def run(self): if not self.is_get_tests_from_previous_sessions and len(self.test_paths) == 0: if self.input_given: - msg = "ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent." # noqa E501 + print_error_and_die("ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent.", Tracking.ErrorEvent.USER_ERROR) # noqa E501 + if client.is_enabled_pts_v2(): + click.echo("INFO: Subset input is empty, enabling `--get-tests-from-previous-sessions` option", err=True) + self.is_get_tests_from_previous_sessions = True else: - msg = "ERROR: Expecting tests to be given, but none provided. See https://www.launchableinc.com/docs/features/predictive-test-selection/requesting-and-running-a-subset-of-tests/subsetting-with-the-launchable-cli/ and provide ones, or use the `--get-tests-from-previous-sessions` option" # noqa E501 - - if msg: - click.echo(click.style(msg, fg="red"), err=True) - exit(1) + print_error_and_die( + "ERROR: Expecting tests to be given, but none provided. See https://www.launchableinc.com/docs/features/predictive-test-selection/requesting-and-running-a-subset-of-tests/subsetting-with-the-launchable-cli/ and provide ones, or use the `--get-tests-from-previous-sessions` option", # noqa E501 + Tracking.ErrorEvent.USER_ERROR) # When Error occurs, return the test name as it is passed. if not session_id: From 244b82fe900d9e1c3fec32a585f560b27223d863 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 29 Jul 2025 10:28:26 -0700 Subject: [PATCH 26/30] These two options are mutually exclusive This is a configuration error so I think it makes sense to make this fatal. --- launchable/commands/subset.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index b145f9c6c..48c63afc0 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -260,6 +260,12 @@ def print_error_and_die(msg: str, event: Tracking.ErrorEvent): tracking_client.send_error_event(event_name=event, stack_trace=msg) sys.exit(1) + if is_get_tests_from_guess and is_get_tests_from_previous_sessions: + print_error_and_die( + "--get-tests-from-guess (list up tests from git ls-files and subset from there) and --get-tests-from-previous-sessions (list up tests from the recent runs and subset from there) are mutually exclusive. Which one do you want to use?", # noqa E501 + Tracking.ErrorEvent.USER_ERROR + ) + if is_observation and is_output_exclusion_rules: msg = ( "WARNING: --observation and --output-exclusion-rules are set. " From a9dd320f6e7f62eead40d93a9f1884c405797da9 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 29 Jul 2025 10:44:20 -0700 Subject: [PATCH 27/30] Improved error handling - Emitting a warning is a common pattern. This needs to be promoted up. - `git ls-files` can result in other types of exceptions thrown - Not finding anything in `git ls-files` should trigger a warning --- launchable/commands/subset.py | 40 +++++++++++++++++------------------ tests/commands/test_subset.py | 2 +- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 48c63afc0..3780eaa6c 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -260,6 +260,13 @@ def print_error_and_die(msg: str, event: Tracking.ErrorEvent): tracking_client.send_error_event(event_name=event, stack_trace=msg) sys.exit(1) + def warn(msg: str): + click.echo(click.style("Warning: " + msg, fg="yellow"), err=True) + tracking_client.send_error_event( + event_name=Tracking.ErrorEvent.WARNING_ERROR, + stack_trace=msg + ) + if is_get_tests_from_guess and is_get_tests_from_previous_sessions: print_error_and_die( "--get-tests-from-guess (list up tests from git ls-files and subset from there) and --get-tests-from-previous-sessions (list up tests from the recent runs and subset from there) are mutually exclusive. Which one do you want to use?", # noqa E501 @@ -267,20 +274,7 @@ def print_error_and_die(msg: str, event: Tracking.ErrorEvent): ) if is_observation and is_output_exclusion_rules: - msg = ( - "WARNING: --observation and --output-exclusion-rules are set. " - "No output will be generated." - ) - click.echo( - click.style( - msg, - fg="yellow"), - err=True, - ) - tracking_client.send_error_event( - event_name=Tracking.ErrorEvent.WARNING_ERROR, - stack_trace=msg - ) + warn("--observation and --output-exclusion-rules are set. No output will be generated.") if prioritize_tests_failed_within_hours is not None and prioritize_tests_failed_within_hours > 0: if ignore_new_tests or (ignore_flaky_tests_above is not None and ignore_flaky_tests_above > 0): @@ -525,20 +519,24 @@ def _collect_potential_test_files(self): LOOSE_TEST_FILE_PATTERN = r'(\.(test|spec)\.|_test\.|Test\.|Spec\.|test/|tests/|__tests__/|src/test/)' EXCLUDE_PATTERN = r'\.(xml|json|txt|yml|yaml|md)$' - git_managed_files = [] try: git_managed_files = subprocess.run(['git', 'ls-files'], stdout=subprocess.PIPE, universal_newlines=True, check=True).stdout.strip().split('\n') - except subprocess.CalledProcessError: - click.echo( - click.style( - "Warning: git ls-files failed.", - fg="yellow"), - err=True) + except subprocess.CalledProcessError as e: + warn(f"git ls-files failed (exit code={e.returncode})") + return + except OSError as e: + warn(f"git ls-files failed: {e}") + return + found = False for f in git_managed_files: if re.search(LOOSE_TEST_FILE_PATTERN, f) and not re.search(EXCLUDE_PATTERN, f): self.test_paths.append(self.to_test_path(f)) + found = True + + if not found: + warn("Nothing that looks like a test file in the current git repository.") def request_subset(self) -> SubsetResult: test_runner = context.invoked_subcommand diff --git a/tests/commands/test_subset.py b/tests/commands/test_subset.py index d3670acfe..7781ba71c 100644 --- a/tests/commands/test_subset.py +++ b/tests/commands/test_subset.py @@ -499,7 +499,7 @@ def test_subset_with_observation_and_output_exclusion_rules(self): self.assert_success(result) self.assertEqual(result.stdout, "") - self.assertIn("WARNING: --observation and --output-exclusion-rules are set.", result.stderr) + self.assertIn("Warning: --observation and --output-exclusion-rules are set.", result.stderr) self.assertEqual(rest.read().decode(), os.linesep.join( ["test_aaa.py", "test_bbb.py", "test_ccc.py", "test_111.py", "test_222.py", "test_333.py"])) From 5228a1e6de33b88f5f36aa5dde129cb03419facf Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 29 Jul 2025 10:52:35 -0700 Subject: [PATCH 28/30] They should probably be an error in the fail-fast mode --- launchable/commands/subset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 3780eaa6c..e943e59bd 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -523,10 +523,10 @@ def _collect_potential_test_files(self): git_managed_files = subprocess.run(['git', 'ls-files'], stdout=subprocess.PIPE, universal_newlines=True, check=True).stdout.strip().split('\n') except subprocess.CalledProcessError as e: - warn(f"git ls-files failed (exit code={e.returncode})") + warn_and_exit_if_fail_fast_mode(f"git ls-files failed (exit code={e.returncode})") return except OSError as e: - warn(f"git ls-files failed: {e}") + warn_and_exit_if_fail_fast_mode(f"git ls-files failed: {e}") return found = False @@ -536,7 +536,7 @@ def _collect_potential_test_files(self): found = True if not found: - warn("Nothing that looks like a test file in the current git repository.") + warn_and_exit_if_fail_fast_mode("Nothing that looks like a test file in the current git repository.") def request_subset(self) -> SubsetResult: test_runner = context.invoked_subcommand From 9ca63ee0c7b8741df3cbe7de274a6075fb758fa4 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 29 Jul 2025 11:48:43 -0700 Subject: [PATCH 29/30] fixup --- launchable/commands/subset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index e943e59bd..1a89b83fe 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -580,7 +580,7 @@ def run(self): if not self.is_get_tests_from_previous_sessions and len(self.test_paths) == 0: if self.input_given: print_error_and_die("ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent.", Tracking.ErrorEvent.USER_ERROR) # noqa E501 - if client.is_enabled_pts_v2(): + if client.is_pts_v2_enabled(): click.echo("INFO: Subset input is empty, enabling `--get-tests-from-previous-sessions` option", err=True) self.is_get_tests_from_previous_sessions = True else: From 445601d8623b852a56539563366015e162811279 Mon Sep 17 00:00:00 2001 From: Konboi Date: Wed, 30 Jul 2025 11:17:52 +0900 Subject: [PATCH 30/30] removed, since it was unintentionally reverted --- launchable/commands/subset.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index 1a89b83fe..3cedd5e0a 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -580,9 +580,6 @@ def run(self): if not self.is_get_tests_from_previous_sessions and len(self.test_paths) == 0: if self.input_given: print_error_and_die("ERROR: Given arguments did not match any tests. They appear to be incorrect/non-existent.", Tracking.ErrorEvent.USER_ERROR) # noqa E501 - if client.is_pts_v2_enabled(): - click.echo("INFO: Subset input is empty, enabling `--get-tests-from-previous-sessions` option", err=True) - self.is_get_tests_from_previous_sessions = True else: print_error_and_die( "ERROR: Expecting tests to be given, but none provided. See https://www.launchableinc.com/docs/features/predictive-test-selection/requesting-and-running-a-subset-of-tests/subsetting-with-the-launchable-cli/ and provide ones, or use the `--get-tests-from-previous-sessions` option", # noqa E501