patch-2.3.16 linux/arch/sparc64/kernel/pci_iommu.c

• Lines: 511
• Date: Tue Aug 31 11:23:30 1999
• Orig file: v2.3.15/linux/arch/sparc64/kernel/pci_iommu.c
• Orig date: Wed Dec 31 16:00:00 1969

diff -u --recursive --new-file v2.3.15/linux/arch/sparc64/kernel/pci_iommu.c linux/arch/sparc64/kernel/pci_iommu.c
@@ -0,0 +1,510 @@
+/* $Id: pci_iommu.c,v 1.1 1999/08/30 10:00:47 davem Exp $
+ * pci_iommu.c: UltraSparc PCI controller IOM/STC support.
+ *
+ * Copyright (C) 1999 David S. Miller (davem@redhat.com)
+ */
+
+#include <asm/pbm.h>
+#include <asm/iommu.h>
+#include <asm/scatterlist.h>
+
+#define PCI_STC_CTXMATCH_ADDR(STC, CTX)	\
+	((STC)->strbuf_ctxmatch_base + ((CTX) << 3))
+
+/* Accessing IOMMU and Streaming Buffer registers.
+ * REG parameter is a physical address.  All registers
+ * are 64-bits in size.
+ */
+#define pci_iommu_read(__reg) \
+({	u64 __ret; \
+	__asm__ __volatile__("ldxa [%1] %2, %0" \
+			     : "=r" (__ret) \
+			     : "r" (__reg), "i" (ASI_PHYS_BYPASS_EC_E) \
+			     : "memory"); \
+	__ret; \
+})
+#define pci_iommu_write(__reg, __val) \
+	__asm__ __volatile__("stxa %0, [%1] %2" \
+			     : /* no outputs */ \
+			     : "r" (__val), "r" (__reg), \
+			       "i" (ASI_PHYS_BYPASS_EC_E))
+
+/* Find a range of iommu mappings of size NPAGES in page
+ * table PGT.  Return pointer to first iopte.
+ */
+static iopte_t *iommu_find_range(unsigned long npages, iopte_t *pgt, int pgt_size)
+{
+	int i;
+
+	pgt_size -= npages;
+	for (i = 0; i < pgt_size; i++) {
+		if (!iopte_val(pgt[i]) & IOPTE_VALID) {
+			int scan;
+
+			for (scan = 1; scan < npages; scan++) {
+				if (iopte_val(pgt[i + scan]) & IOPTE_VALID) {
+					i += scan;
+					goto do_next;
+				}
+			}
+			return &pgt[i];
+		}
+	do_next:
+	}
+	return NULL;
+}
+
+#define IOPTE_CONSISTANT(CTX, PADDR) \
+	(IOPTE_VALID | IOPTE_CACHE | IOPTE_WRITE | \
+	 (((CTX) << 47) & IOPTE_CONTEXT) | \
+	 ((PADDR) & IOPTE_PAGE))
+
+#define IOPTE_STREAMING(CTX, PADDR) \
+	(IOPTE_CONSISTANT(CTX, PADDR) | IOPTE_STBUF)
+
+#define IOPTE_INVALID	0UL
+
+/* Map kernel buffer at ADDR of size SZ using consistant mode
+ * DMA for PCI device PDEV.  Return 32-bit PCI DMA address.
+ */
+u32 pci_map_consistant(struct pci_dev *pdev, void *addr, int sz)
+{
+	struct pcidev_cookie *pcp = pdev->sysdata;
+	struct pci_iommu *iommu = &pcp->pbm->parent->iommu;
+	iopte_t *base;
+	unsigned long flags, npages, oaddr;
+	u32 ret;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+	oaddr = (unsigned long)addr;
+	npages = PAGE_ALIGN(oaddr + sz) - (oaddr & PAGE_MASK);
+	npages >>= PAGE_SHIFT;
+	base = iommu_find_range(npages,
+				iommu->page_table, iommu->page_table_sz);
+	ret = 0;
+	if (base != NULL) {
+		unsigned long i, base_paddr, ctx;
+
+		ret = (iommu->page_table_map_base +
+		       ((base - iommu->page_table) << PAGE_SHIFT));
+		ret |= (oaddr & ~PAGE_MASK);
+		base_paddr = __pa(oaddr & PAGE_MASK);
+		ctx = 0;
+		if (iommu->iommu_has_ctx_flush)
+			ctx = iommu->iommu_cur_ctx++;
+		for (i = 0; i < npages; i++, base++, base_paddr += PAGE_SIZE)
+			iopte_val(*base) = IOPTE_CONSISTANT(ctx, base_paddr);
+	}
+	spin_unlock_irqrestore(&iommu->lock, flags);
+
+	return ret;
+}
+
+/* Unmap a consistant DMA translation. */
+void pci_unmap_consistant(struct pci_dev *pdev, u32 bus_addr, int sz)
+{
+	struct pcidev_cookie *pcp = pdev->sysdata;
+	struct pci_iommu *iommu = &pcp->pbm->parent->iommu;
+	iopte_t *base;
+	unsigned long flags, npages, i, ctx;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+	npages = PAGE_ALIGN(bus_addr + sz) - (bus_addr & PAGE_MASK);
+	npages >>= PAGE_SHIFT;
+	base = iommu->page_table +
+		((bus_addr - iommu->page_table_map_base) >> PAGE_SHIFT);
+
+	/* Data for consistant mappings cannot enter the streaming
+	 * buffers, so we only need to update the TSB and flush
+	 * those entries from the IOMMU's TLB.
+	 */
+
+	/* Step 1: Clear out the TSB entries.  Save away
+	 *         the context if necessary.
+	 */
+	ctx = 0;
+	if (iommu->iommu_has_ctx_flush)
+		ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
+	for (i = 0; i < npages; i++, base++)
+		iopte_val(*base) = IOPTE_INVALID;
+
+	/* Step 2: Flush from IOMMU TLB. */
+	if (iommu->iommu_has_ctx_flush) {
+		pci_iommu_write(iommu->iommu_ctxflush, ctx);
+	} else {
+		bus_addr &= PAGE_MASK;
+		for (i = 0; i < npages; i++, bus_addr += PAGE_SIZE)
+			pci_iommu_write(iommu->iommu_flush, bus_addr);
+	}
+
+	/* Step 3: Ensure completion of previous PIO writes. */
+	(void) pci_iommu_read(iommu->write_complete_reg);
+
+	spin_unlock_irqrestore(&iommu->lock, flags);
+}
+
+/* Map a single buffer at PTR of SZ bytes for PCI DMA
+ * in streaming mode.
+ */
+u32 pci_map_single(struct pci_dev *pdev, void *ptr, int sz)
+{
+	struct pcidev_cookie *pcp = pdev->sysdata;
+	struct pci_iommu *iommu = &pcp->pbm->parent->iommu;
+	iopte_t *base;
+	unsigned long flags, npages, oaddr;
+	u32 ret;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+	oaddr = (unsigned long)ptr;
+	npages = PAGE_ALIGN(oaddr + sz) - (oaddr & PAGE_MASK);
+	npages >>= PAGE_SHIFT;
+	base = iommu_find_range(npages,
+				iommu->page_table, iommu->page_table_sz);
+	ret = 0;
+	if (base != NULL) {
+		unsigned long i, base_paddr, ctx;
+
+		ret = (iommu->page_table_map_base +
+		       ((base - iommu->page_table) << PAGE_SHIFT));
+		ret |= (oaddr & ~PAGE_MASK);
+		base_paddr = __pa(oaddr & PAGE_MASK);
+		ctx = 0;
+		if (iommu->iommu_has_ctx_flush)
+			ctx = iommu->iommu_cur_ctx++;
+		for (i = 0; i < npages; i++, base++, base_paddr += PAGE_SIZE)
+			iopte_val(*base) = IOPTE_STREAMING(ctx, base_paddr);
+	}
+	spin_unlock_irqrestore(&iommu->lock, flags);
+
+	return ret;
+}
+
+/* Unmap a single streaming mode DMA translation. */
+void pci_unmap_single(struct pci_dev *pdev, u32 bus_addr, int sz)
+{
+	struct pcidev_cookie *pcp = pdev->sysdata;
+	struct pci_iommu *iommu = &pcp->pbm->parent->iommu;
+	struct pci_strbuf *strbuf = &pcp->pbm->stc;
+	iopte_t *base;
+	unsigned long flags, npages, i, ctx;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+	npages = PAGE_ALIGN(bus_addr + sz) - (bus_addr & PAGE_MASK);
+	npages >>= PAGE_SHIFT;
+	base = iommu->page_table +
+		((bus_addr - iommu->page_table_map_base) >> PAGE_SHIFT);
+	bus_addr &= PAGE_MASK;
+
+	/* Step 1: Record the context, if any. */
+	ctx = 0;
+	if (iommu->iommu_has_ctx_flush)
+		ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
+
+	/* Step 2: Kick data out of streaming buffers if necessary. */
+	if (strbuf->strbuf_enabled) {
+		u32 vaddr = bus_addr;
+
+		PCI_STC_FLUSHFLAG_INIT(strbuf);
+		if (strbuf->strbuf_has_ctx_flush &&
+		    iommu->iommu_has_ctx_flush) {
+			unsigned long matchreg, flushreg;
+
+			flushreg = strbuf->strbuf_ctxflush;
+			matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
+			do {
+				pci_iommu_write(flushreg, ctx);
+			} while(((long)pci_iommu_read(matchreg)) < 0L);
+		} else {
+			for (i = 0; i < npages; i++, vaddr += PAGE_SIZE)
+				pci_iommu_write(strbuf->strbuf_pflush, vaddr);
+		}
+
+		pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
+		(void) pci_iommu_read(iommu->write_complete_reg);
+		while (!PCI_STC_FLUSHFLAG_SET(strbuf))
+			membar("#LoadLoad");
+	}
+
+	/* Step 3: Clear out TSB entries. */
+	for (i = 0; i < npages; i++, base++)
+		iopte_val(*base) = IOPTE_INVALID;
+
+	/* Step 4: Flush the IOMMU TLB. */
+	if (iommu->iommu_has_ctx_flush) {
+		pci_iommu_write(iommu->iommu_ctxflush, ctx);
+	} else {
+		for (i = 0; i < npages; i++, bus_addr += PAGE_SIZE)
+			pci_iommu_write(iommu->iommu_flush, bus_addr);
+	}
+
+	/* Step 5: Ensure completion of previous PIO writes. */
+	(void) pci_iommu_read(iommu->write_complete_reg);
+
+	spin_unlock_irqrestore(&iommu->lock, flags);
+}
+
+/* Map a set of buffers described by SGLIST with NELEMS array
+ * elements in streaming mode for PCI DMA.
+ */
+void pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems)
+{
+	struct pcidev_cookie *pcp = pdev->sysdata;
+	struct pci_iommu *iommu = &pcp->pbm->parent->iommu;
+	unsigned long flags, ctx, i;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+
+	/* Step 1: Choose a context if necessary. */
+	ctx = 0;
+	if (iommu->iommu_has_ctx_flush)
+		ctx = iommu->iommu_cur_ctx++;
+
+	/* Step 2: Create the mappings. */
+	for (i = 0; i < nelems; i++) {
+		unsigned long oaddr, npages;
+		iopte_t *base;
+
+		oaddr = (unsigned long)sglist[i].address;
+		npages = PAGE_ALIGN(oaddr + sglist[i].length) - (oaddr & PAGE_MASK);
+		npages >>= PAGE_SHIFT;
+		base = iommu_find_range(npages,
+					iommu->page_table, iommu->page_table_sz);
+		if (base != NULL) {
+			unsigned long j, base_paddr;
+			u32 dvma_addr;
+
+			dvma_addr = (iommu->page_table_map_base +
+				     ((base - iommu->page_table) << PAGE_SHIFT));
+			dvma_addr |= (oaddr & ~PAGE_MASK);
+			sglist[i].dvma_address = dvma_addr;
+			sglist[i].dvma_length = sglist[i].length;
+			base_paddr = __pa(oaddr & PAGE_MASK);
+			for (j = 0; j < npages; j++, base++, base_paddr += PAGE_SIZE)
+				iopte_val(*base) = IOPTE_STREAMING(ctx, base_paddr);
+		} else {
+			sglist[i].dvma_address = 0;
+			sglist[i].dvma_length = 0;
+		}
+	}
+
+	spin_unlock_irqrestore(&iommu->lock, flags);
+}
+
+/* Unmap a set of streaming mode DMA translations. */
+void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems)
+{
+	struct pcidev_cookie *pcp = pdev->sysdata;
+	struct pci_iommu *iommu = &pcp->pbm->parent->iommu;
+	struct pci_strbuf *strbuf = &pcp->pbm->stc;
+	unsigned long flags, ctx, i;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+
+	/* Step 1: Record the context, if any. */
+	ctx = 0;
+	if (iommu->iommu_has_ctx_flush) {
+		iopte_t *iopte;
+
+		iopte = iommu->page_table +
+			((sglist[0].dvma_address - iommu->page_table_map_base) >> PAGE_SHIFT);
+		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
+	}
+
+	/* Step 2: Kick data out of streaming buffers if necessary. */
+	if (strbuf->strbuf_enabled) {
+		PCI_STC_FLUSHFLAG_INIT(strbuf);
+		if (strbuf->strbuf_has_ctx_flush &&
+		    iommu->iommu_has_ctx_flush) {
+			unsigned long matchreg, flushreg;
+
+			flushreg = strbuf->strbuf_ctxflush;
+			matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
+			do {
+				pci_iommu_write(flushreg, ctx);
+			} while(((long)pci_iommu_read(matchreg)) < 0L);
+		} else {
+			for (i = 0; i < nelems; i++) {
+				unsigned long j, npages;
+				u32 vaddr;
+
+				j = sglist[i].dvma_length;
+				if (!j)
+					break;
+				vaddr = sglist[i].dvma_address;
+				npages = PAGE_ALIGN(vaddr + j) - (vaddr & PAGE_MASK);
+				npages >>= PAGE_SHIFT;
+				vaddr &= PAGE_MASK;
+				for (j = 0; j < npages; j++, vaddr += PAGE_SIZE)
+					pci_iommu_write(strbuf->strbuf_pflush, vaddr);
+			}
+
+			pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
+			(void) pci_iommu_read(iommu->write_complete_reg);
+			while (!PCI_STC_FLUSHFLAG_SET(strbuf))
+				membar("#LoadLoad");
+		}
+	}
+
+	/* Step 3: Clear out TSB entries. */
+	for (i = 0; i < nelems; i++) {
+		unsigned long j, npages;
+		iopte_t *base;
+		u32 vaddr;
+
+		j = sglist[i].dvma_length;
+		if (!j)
+			break;
+		vaddr = sglist[i].dvma_address;
+		npages = PAGE_ALIGN(vaddr + j) - (vaddr & PAGE_MASK);
+		npages >>= PAGE_SHIFT;
+		base = iommu->page_table +
+			((vaddr - iommu->page_table_map_base) >> PAGE_SHIFT);
+		for (j = 0; j < npages; j++, base++)
+			iopte_val(*base) = IOPTE_INVALID;
+	}
+
+	/* Step 4: Flush the IOMMU TLB. */
+	if (iommu->iommu_has_ctx_flush) {
+		pci_iommu_write(iommu->iommu_ctxflush, ctx);
+	} else {
+		for (i = 0; i < nelems; i++) {
+			unsigned long j, npages;
+			u32 vaddr;
+
+			j = sglist[i].dvma_length;
+			if (!j)
+				break;
+			vaddr = sglist[i].dvma_address;
+			npages = PAGE_ALIGN(vaddr + j) - (vaddr & PAGE_MASK);
+			npages >>= PAGE_SHIFT;
+			for (j = 0; j < npages; j++, vaddr += PAGE_SIZE)
+				pci_iommu_write(iommu->iommu_flush, vaddr);
+		}
+	}
+
+	/* Step 5: Ensure completion of previous PIO writes. */
+	(void) pci_iommu_read(iommu->write_complete_reg);
+
+	spin_unlock_irqrestore(&iommu->lock, flags);
+}
+
+/* Make physical memory consistant for a single
+ * streaming mode DMA translation after a transfer.
+ */
+void pci_dma_sync_single(struct pci_dev *pdev, u32 bus_addr, int sz)
+{
+	struct pcidev_cookie *pcp = pdev->sysdata;
+	struct pci_iommu *iommu = &pcp->pbm->parent->iommu;
+	struct pci_strbuf *strbuf = &pcp->pbm->stc;
+	unsigned long flags, ctx, npages;
+
+	if (!strbuf->strbuf_enabled)
+		return;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+
+	npages = PAGE_ALIGN(bus_addr + sz) - (bus_addr & PAGE_MASK);
+	npages >>= PAGE_SHIFT;
+	bus_addr &= PAGE_MASK;
+
+	/* Step 1: Record the context, if any. */
+	ctx = 0;
+	if (iommu->iommu_has_ctx_flush &&
+	    strbuf->strbuf_has_ctx_flush) {
+		iopte_t *iopte;
+
+		iopte = iommu->page_table +
+			((bus_addr - iommu->page_table_map_base)>>PAGE_SHIFT);
+		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
+	}
+
+	/* Step 2: Kick data out of streaming buffers. */
+	PCI_STC_FLUSHFLAG_INIT(strbuf);
+	if (iommu->iommu_has_ctx_flush &&
+	    strbuf->strbuf_has_ctx_flush) {
+		unsigned long matchreg, flushreg;
+
+		flushreg = strbuf->strbuf_ctxflush;
+		matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
+		do {
+			pci_iommu_write(flushreg, ctx);
+		} while(((long)pci_iommu_read(matchreg)) < 0L);
+	} else {
+		unsigned long i;
+
+		for (i = 0; i < npages; i++, bus_addr += PAGE_SIZE)
+			pci_iommu_write(strbuf->strbuf_pflush, bus_addr);
+	}
+
+	/* Step 3: Perform flush synchronization sequence. */
+	pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
+	(void) pci_iommu_read(iommu->write_complete_reg);
+	while (!PCI_STC_FLUSHFLAG_SET(strbuf))
+		membar("#LoadLoad");
+
+	spin_unlock_irqrestore(&iommu->lock, flags);
+}
+
+/* Make physical memory consistant for a set of streaming
+ * mode DMA translations after a transfer.
+ */
+void pci_dma_sync_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems)
+{
+	struct pcidev_cookie *pcp = pdev->sysdata;
+	struct pci_iommu *iommu = &pcp->pbm->parent->iommu;
+	struct pci_strbuf *strbuf = &pcp->pbm->stc;
+	unsigned long flags, ctx;
+
+	if (!strbuf->strbuf_enabled)
+		return;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+
+	/* Step 1: Record the context, if any. */
+	ctx = 0;
+	if (iommu->iommu_has_ctx_flush &&
+	    strbuf->strbuf_has_ctx_flush) {
+		iopte_t *iopte;
+
+		iopte = iommu->page_table +
+			((sglist[0].dvma_address - iommu->page_table_map_base) >> PAGE_SHIFT);
+		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
+	}
+
+	/* Step 2: Kick data out of streaming buffers. */
+	PCI_STC_FLUSHFLAG_INIT(strbuf);
+	if (iommu->iommu_has_ctx_flush &&
+	    strbuf->strbuf_has_ctx_flush) {
+		unsigned long matchreg, flushreg;
+
+		flushreg = strbuf->strbuf_ctxflush;
+		matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
+		do {
+			pci_iommu_write(flushreg, ctx);
+		} while (((long)pci_iommu_read(matchreg)) < 0L);
+	} else {
+		unsigned long i;
+
+		for(i = 0; i < nelems; i++) {
+			unsigned long bus_addr, npages, j;
+
+			j = sglist[i].dvma_length;
+			if (!j)
+				break;
+			bus_addr = sglist[i].dvma_address;
+			npages = PAGE_ALIGN(bus_addr + j) - (bus_addr & PAGE_MASK);
+			npages >>= PAGE_SHIFT;
+			bus_addr &= PAGE_MASK;
+			for(j = 0; i < npages; i++, bus_addr += PAGE_SIZE)
+				pci_iommu_write(strbuf->strbuf_pflush, bus_addr);
+		}
+	}
+
+	/* Step 3: Perform flush synchronization sequence. */
+	pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
+	(void) pci_iommu_read(iommu->write_complete_reg);
+	while (!PCI_STC_FLUSHFLAG_SET(strbuf))
+		membar("#LoadLoad");
+
+	spin_unlock_irqrestore(&iommu->lock, flags);
+}