From 5c93f8d7d66d3041c319407165b162439967ff79 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 31 Mar 2026 16:56:22 -0300 Subject: [PATCH] iommu: Always fill in gather when unmapping The fixed commit assumed that the gather would always be populated if an iotlb_sync was required. arm-smmu-v3, amd, VT-d, riscv, s390, mtk all use information from the gather during their iotlb_sync() and this approach works for them. However, arm-smmu, qcom_iommu, ipmmu-vmsa, sun50i, sprd, virtio, apple-dart all ignore the gather during their iotlb_sync(). They mostly issue a full flush. Unfortunately the latter set of drivers often don't bother to add anything to the gather since they don't intend on using it. Since the core code now blocks gathers that were never filled, this caused those drivers to stop getting their iotlb_sync() calls and breaks them. Since it is impossible to tell the difference between gathers that are empty because there is nothing to do and gathers that are empty because they are not used, fill in the gathers for the missing cases. io-pgtable might have intended to allow the driver to choose between gather or immediate flush because it passed gather to ops->tlb_add_page(), however no driver does anything with it. mtk uses io-pgtable-arm-v7s but added the range to the gather in the unmap callback. Move this into the io-pgtable-arm unmap itself. That will fix all the armv7 using drivers (arm-smmu, qcom_iommu, ipmmu-vmsa). arm-smmu uses both ARM_V7S and ARM LPAE formats. The LPAE formats already have the gather population because SMMUv3 requires it, so it becomes consistent. Add a trivial gather population to io-pgtable-dart. Add trivial populations to sprd, sun50i and virtio-iommu in their unmap functions. Fixes: 90c5def10bea ("iommu: Do not call drivers for empty gathers") Reported-by: Jon Hunter Closes: https://lore.kernel.org/r/8800a38b-8515-4bbe-af15-0dae81274bf7@nvidia.com Signed-off-by: Jason Gunthorpe Signed-off-by: Linux RISC-V bot --- drivers/iommu/io-pgtable-arm.c | 4 +++- drivers/iommu/io-pgtable-dart.c | 3 +++ drivers/iommu/mtk_iommu.c | 1 - drivers/iommu/sprd-iommu.c | 1 + drivers/iommu/sun50i-iommu.c | 1 + drivers/iommu/virtio-iommu.c | 2 ++ 6 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 05d63fe92e4361..fce75ccbd6bd8b 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -666,9 +666,11 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, /* Clear the remaining entries */ __arm_lpae_clear_pte(ptep, &iop->cfg, i); - if (gather && !iommu_iotlb_gather_queued(gather)) + if (gather && !iommu_iotlb_gather_queued(gather)) { + iommu_iotlb_gather_add_range(gather, iova, i * size); for (int j = 0; j < i; j++) io_pgtable_tlb_add_page(iop, gather, iova + j * size, size); + } return i * size; } else if (iopte_leaf(pte, lvl, iop->fmt)) { diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index 54d287cc0dd1b8..cbef7de45d7e27 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -330,6 +330,9 @@ static size_t dart_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, i++; } + if (i && !iommu_iotlb_gather_queued(gather)) + iommu_iotlb_gather_add_range(gather, iova, i * pgsize); + return i * pgsize; } diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 60fcd3d3b5ebdf..1456535d0924ea 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -828,7 +828,6 @@ static size_t mtk_iommu_unmap(struct iommu_domain *domain, { struct mtk_iommu_domain *dom = to_mtk_domain(domain); - iommu_iotlb_gather_add_range(gather, iova, pgsize * pgcount); return dom->iop->unmap_pages(dom->iop, iova, pgsize, pgcount, gather); } diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c index 555d4505c747a9..f37030f5ed172a 100644 --- a/drivers/iommu/sprd-iommu.c +++ b/drivers/iommu/sprd-iommu.c @@ -340,6 +340,7 @@ static size_t sprd_iommu_unmap(struct iommu_domain *domain, unsigned long iova, spin_lock_irqsave(&dom->pgtlock, flags); memset(pgt_base_iova, 0, pgcount * sizeof(u32)); spin_unlock_irqrestore(&dom->pgtlock, flags); + iommu_iotlb_gather_add_range(iotlb_gather, iova, size); return size; } diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index 90b26fe2181765..18c2658bbfed70 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -655,6 +655,7 @@ static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova memset(pte_addr, 0, sizeof(*pte_addr)); sun50i_table_flush(sun50i_domain, pte_addr, 1); + iommu_iotlb_gather_add_range(gather, iova, SZ_4K); return SZ_4K; } diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index d314fa5cd84768..bb2b2abbb87af8 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -897,6 +897,8 @@ static size_t viommu_unmap_pages(struct iommu_domain *domain, unsigned long iova if (unmapped < size) return 0; + iommu_iotlb_gather_add_range(gather, iova, unmapped); + /* Device already removed all mappings after detach. */ if (!vdomain->nr_endpoints) return unmapped;