Skip to content

Commit 80187fd

Browse files
joergroedeljoergroedel
committed
iommu/amd: Optimize map_sg and unmap_sg
Optimize these functions so that they need only one call into the address alloctor. This also saves a couple of io-tlb flushes in the unmap_sg path. Signed-off-by: Joerg Roedel <[email protected]>
1 parent f37f7f3 commit 80187fd

1 file changed

Lines changed: 85 additions & 26 deletions

File tree

‎drivers/iommu/amd_iommu.c‎

Lines changed: 85 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2396,50 +2396,110 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
23962396
__unmap_single(domain->priv, dma_addr, size, dir);
23972397
}
23982398

2399+
static int sg_num_pages(struct device *dev,
2400+
struct scatterlist *sglist,
2401+
int nelems)
2402+
{
2403+
unsigned long mask, boundary_size;
2404+
struct scatterlist *s;
2405+
int i, npages = 0;
2406+
2407+
mask = dma_get_seg_boundary(dev);
2408+
boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
2409+
1UL << (BITS_PER_LONG - PAGE_SHIFT);
2410+
2411+
for_each_sg(sglist, s, nelems, i) {
2412+
int p, n;
2413+
2414+
s->dma_address = npages << PAGE_SHIFT;
2415+
p = npages % boundary_size;
2416+
n = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);
2417+
if (p + n > boundary_size)
2418+
npages += boundary_size - p;
2419+
npages += n;
2420+
}
2421+
2422+
return npages;
2423+
}
2424+
23992425
/*
24002426
* The exported map_sg function for dma_ops (handles scatter-gather
24012427
* lists).
24022428
*/
24032429
static int map_sg(struct device *dev, struct scatterlist *sglist,
2404-
int nelems, enum dma_data_direction dir,
2430+
int nelems, enum dma_data_direction direction,
24052431
struct dma_attrs *attrs)
24062432
{
2433+
int mapped_pages = 0, npages = 0, prot = 0, i;
24072434
struct protection_domain *domain;
2408-
int i;
2435+
struct dma_ops_domain *dma_dom;
24092436
struct scatterlist *s;
2410-
phys_addr_t paddr;
2411-
int mapped_elems = 0;
2437+
unsigned long address;
24122438
u64 dma_mask;
24132439

24142440
domain = get_domain(dev);
24152441
if (IS_ERR(domain))
24162442
return 0;
24172443

2444+
dma_dom = domain->priv;
24182445
dma_mask = *dev->dma_mask;
24192446

2447+
npages = sg_num_pages(dev, sglist, nelems);
2448+
2449+
address = dma_ops_alloc_iova(dev, dma_dom, npages, dma_mask);
2450+
if (address == DMA_ERROR_CODE)
2451+
goto out_err;
2452+
2453+
prot = dir2prot(direction);
2454+
2455+
/* Map all sg entries */
24202456
for_each_sg(sglist, s, nelems, i) {
2421-
paddr = sg_phys(s);
2457+
int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);
2458+
2459+
for (j = 0; j < pages; ++j) {
2460+
unsigned long bus_addr, phys_addr;
2461+
int ret;
24222462

2423-
s->dma_address = __map_single(dev, domain->priv,
2424-
paddr, s->length, dir, dma_mask);
2463+
bus_addr = address + s->dma_address + (j << PAGE_SHIFT);
2464+
phys_addr = (sg_phys(s) & PAGE_MASK) + (j << PAGE_SHIFT);
2465+
ret = iommu_map_page(domain, bus_addr, phys_addr, PAGE_SIZE, prot, GFP_ATOMIC);
2466+
if (ret)
2467+
goto out_unmap;
24252468

2426-
if (s->dma_address) {
2427-
s->dma_length = s->length;
2428-
mapped_elems++;
2429-
} else
2430-
goto unmap;
2469+
mapped_pages += 1;
2470+
}
24312471
}
24322472

2433-
return mapped_elems;
2473+
/* Everything is mapped - write the right values into s->dma_address */
2474+
for_each_sg(sglist, s, nelems, i) {
2475+
s->dma_address += address + s->offset;
2476+
s->dma_length = s->length;
2477+
}
2478+
2479+
return nelems;
2480+
2481+
out_unmap:
2482+
pr_err("%s: IOMMU mapping error in map_sg (io-pages: %d)\n",
2483+
dev_name(dev), npages);
2484+
2485+
for_each_sg(sglist, s, nelems, i) {
2486+
int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);
2487+
2488+
for (j = 0; j < pages; ++j) {
2489+
unsigned long bus_addr;
24342490

2435-
unmap:
2436-
for_each_sg(sglist, s, mapped_elems, i) {
2437-
if (s->dma_address)
2438-
__unmap_single(domain->priv, s->dma_address,
2439-
s->dma_length, dir);
2440-
s->dma_address = s->dma_length = 0;
2491+
bus_addr = address + s->dma_address + (j << PAGE_SHIFT);
2492+
iommu_unmap_page(domain, bus_addr, PAGE_SIZE);
2493+
2494+
if (--mapped_pages)
2495+
goto out_free_iova;
2496+
}
24412497
}
24422498

2499+
out_free_iova:
2500+
free_iova_fast(&dma_dom->iovad, address, npages);
2501+
2502+
out_err:
24432503
return 0;
24442504
}
24452505

@@ -2452,18 +2512,17 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
24522512
struct dma_attrs *attrs)
24532513
{
24542514
struct protection_domain *domain;
2455-
struct scatterlist *s;
2456-
int i;
2515+
unsigned long startaddr;
2516+
int npages = 2;
24572517

24582518
domain = get_domain(dev);
24592519
if (IS_ERR(domain))
24602520
return;
24612521

2462-
for_each_sg(sglist, s, nelems, i) {
2463-
__unmap_single(domain->priv, s->dma_address,
2464-
s->dma_length, dir);
2465-
s->dma_address = s->dma_length = 0;
2466-
}
2522+
startaddr = sg_dma_address(sglist) & PAGE_MASK;
2523+
npages = sg_num_pages(dev, sglist, nelems);
2524+
2525+
__unmap_single(domain->priv, startaddr, npages << PAGE_SHIFT, dir);
24672526
}
24682527

24692528
/*

0 commit comments

Comments
 (0)