From 8152f117f1199a024cc60c0743c1aadf889425ce Mon Sep 17 00:00:00 2001 From: Nero10578 Date: Mon, 14 Apr 2025 09:25:19 -0700 Subject: [PATCH 1/2] scheduler changes to allow LoRA + Chunked Prefill --- aphrodite/processing/scheduler.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/aphrodite/processing/scheduler.py b/aphrodite/processing/scheduler.py index d1d1842e81..e527ca0fa6 100644 --- a/aphrodite/processing/scheduler.py +++ b/aphrodite/processing/scheduler.py @@ -149,9 +149,18 @@ def is_empty(self) -> bool: and not self.blocks_to_swap_out and not self.blocks_to_copy) def _sort_by_lora_ids(self): - self.scheduled_seq_groups = sorted( - self.scheduled_seq_groups, - key=lambda g: (g.seq_group.lora_int_id, g.seq_group.request_id)) + assert 0 <= self.num_prefill_groups <= len(self.scheduled_seq_groups) + + def key_fn(group: ScheduledSequenceGroup): + key = (group.seq_group.lora_int_id, group.seq_group.request_id) + if 0 < self.num_prefill_groups < len(self.scheduled_seq_groups): + # Sort sequence groups so that all prefills come before all + # decodes as required by chunked prefill. + return (not group.seq_group.is_prefill(), *key) + return key + + self.scheduled_seq_groups = sorted(self.scheduled_seq_groups, + key=key_fn) @property def lora_requests(self) -> Set[LoRARequest]: From c9bd90449e0751ba941edc182d9358cf7a001b5d Mon Sep 17 00:00:00 2001 From: Nero10578 Date: Mon, 14 Apr 2025 21:59:54 -0700 Subject: [PATCH 2/2] allow block-manager-v2-disable and fix metrics negative value error --- aphrodite/engine/args_tools.py | 5 ++--- aphrodite/engine/metrics.py | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/aphrodite/engine/args_tools.py b/aphrodite/engine/args_tools.py index 84bdb38f24..80a488c3aa 100644 --- a/aphrodite/engine/args_tools.py +++ b/aphrodite/engine/args_tools.py @@ -619,9 +619,8 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: parser.add_argument( '--use-v2-block-manager', default=EngineArgs.use_v2_block_manager, - action='store_true', - help='Use BlockSpaceMangerV2. By default this is set to True. ' - 'Set to False to use BlockSpaceManagerV1') + type=lambda x: (str(x).lower() in ['true', '1', 'yes']), + help='Use BlockSpaceManagerV2 (default: True). Set explicitly to False to use BlockSpaceManagerV1') parser.add_argument( "--scheduler-delay-factor", "-sdf", diff --git a/aphrodite/engine/metrics.py b/aphrodite/engine/metrics.py index c4b66dadc4..fafc06127b 100644 --- a/aphrodite/engine/metrics.py +++ b/aphrodite/engine/metrics.py @@ -468,6 +468,9 @@ def _log_gauge(self, gauge, data: Union[int, float]) -> None: def _log_counter(self, counter, data: Union[int, float]) -> None: # Convenience function for logging to counter. + if data < 0: + logger.warning(f"Attempted to increment counter with negative value: {data}") + return counter.labels(**self.labels).inc(data) def _log_counter_labels(self, counter, data: CollectionsCounter,