From 6ab6f3c0ea5e8a7e905eef2bbf18e9e6acca8a45 Mon Sep 17 00:00:00 2001 From: NNN Date: Wed, 15 Jul 2026 23:05:10 +0900 Subject: [PATCH 1/2] fix: recognize EXL3 base layers in LoRA device detection _get_lora_device() determines where to place LoRA adapter tensors by checking a fixed allowlist of quant-method-specific weight attribute names (weight, weight_packed, qweight, w2_weight, ..., w2_qweight). EXL3's Exl3LinearMethod stores its quantized weights under different attribute names (suh, svh, trellis, mcg, mul1), which were never added to this list, so any attempt to enable LoRA on an EXL3-quantized model fails during LoRA layer construction with: ValueError: Unsupported base layer: MergedColumnParallelLinear(...) This affects EXL3 models generally, not just GDN/hybrid architectures (see companion PR for a GDN-specific EXL3+LoRA weight-loading fix). Fix: add a branch recognizing the 'trellis' attribute. EXL3 is CUDA-only by construction (Exl3LinearMethod.create_weights raises NotImplementedError on non-CUDA platforms), and its tensors are always placed on torch.cuda.current_device() in process_weights_after_loading(), so using the current CUDA device directly is correct and avoids relying on a placeholder attribute whose .device would not reflect where the real per-shard tensors live. --- aphrodite/lora/layers/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aphrodite/lora/layers/utils.py b/aphrodite/lora/layers/utils.py index 9873f2e029..9862398bab 100644 --- a/aphrodite/lora/layers/utils.py +++ b/aphrodite/lora/layers/utils.py @@ -69,6 +69,10 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device: # MoE GPTQ/AWQ/GGUF elif hasattr(base_layer, "w2_qweight"): return base_layer.w2_qweight.device + # EXL3 (exllamav3) — CUDA-only; tensors are placed on the current device + # in Exl3LinearMethod.process_weights_after_loading. + elif hasattr(base_layer, "trellis"): + return torch.device("cuda", torch.cuda.current_device()) else: raise ValueError(f"Unsupported base layer: {base_layer}") From 528bf7ad5e7cd3941b928be66a3e01778f5f169d Mon Sep 17 00:00:00 2001 From: nakaken3013-code Date: Fri, 17 Jul 2026 05:34:43 +0900 Subject: [PATCH 2/2] fix: use torch.accelerator instead of torch.cuda in EXL3 device detection Address review feedback on #1712 - torch.cuda usage makes hardware-agnostic inference harder to maintain. --- aphrodite/lora/layers/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aphrodite/lora/layers/utils.py b/aphrodite/lora/layers/utils.py index 9862398bab..3a64b910b3 100644 --- a/aphrodite/lora/layers/utils.py +++ b/aphrodite/lora/layers/utils.py @@ -72,7 +72,7 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device: # EXL3 (exllamav3) — CUDA-only; tensors are placed on the current device # in Exl3LinearMethod.process_weights_after_loading. elif hasattr(base_layer, "trellis"): - return torch.device("cuda", torch.cuda.current_device()) + return torch.device("cuda", torch.accelerator.current_device_index()) else: raise ValueError(f"Unsupported base layer: {base_layer}")