patch-2.3.43 linux/arch/ppc/mm/4xx_tlb.c

• Lines: 649
• Date: Wed Feb 9 19:43:47 2000
• Orig file: v2.3.42/linux/arch/ppc/mm/4xx_tlb.c
• Orig date: Tue Jan 11 22:31:38 2000

diff -u --recursive --new-file v2.3.42/linux/arch/ppc/mm/4xx_tlb.c linux/arch/ppc/mm/4xx_tlb.c
@@ -1,6 +1,9 @@
 /*
  *
- *    Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu>
+ *    Copyright (c) 1998-1999 TiVo, Inc.
+ *      Original implementation.
+ *    Copyright (c) 1999-2000 Grant Erickson <grant@lcse.umn.edu>
+ *      Minor rework.
  *
  *    Module name: 4xx_tlb.c
  *
@@ -9,7 +12,10 @@
  *
  */
 
+#include <linux/mm.h>
+
 #include <asm/processor.h>
+#include <asm/io.h>
 #include <asm/mmu.h>
 #include <asm/pgtable.h>
 #include <asm/system.h>
@@ -26,372 +32,327 @@
 #endif
 
 
-/* Function Macros */
-
-
-/* Type Definitios */
-
-typedef struct pin_entry_s {
-	unsigned int	e_pinned: 1,	/* This TLB entry is pinned down. */
-			e_used: 23;	/* Number of users for this mapping. */
-} pin_entry_t;
-
-
 /* Global Variables */
 
-static pin_entry_t pin_table[PPC4XX_TLB_SIZE];
+static int pinned = 0;
 
 
 /* Function Prototypes */
 
+static int PPC4xx_tlb_miss(struct pt_regs *, unsigned long, int);
 
-void
-PPC4xx_tlb_pin(unsigned long va, unsigned long pa, int pagesz, int cache)
-{
-	int i, found = FALSE;
-	unsigned long tag, data;
-	unsigned long opid;
-
-	opid = mfspr(SPRN_PID);
-	mtspr(SPRN_PID, 0);
-
-	data = (pa & TLB_RPN_MASK) | TLB_WR;
-
-	if (cache)
-		data |= (TLB_EX | TLB_I);
-	else
-		data |= (TLB_G | TLB_I);
-
-	tag = (va & TLB_EPN_MASK) | TLB_VALID | pagesz;
-
-	for (i = 0; i < PPC4XX_TLB_SIZE; i++) {
-		if (pin_table[i].e_pinned == FALSE) {
-			found = TRUE;
-			break;
-		}
-	}
+extern void do_page_fault(struct pt_regs *, unsigned long, unsigned long);
 
-	if (found) {
-		/* printk("Pinning %#x -> %#x in entry %d...\n", va, pa, i); */
-		asm("tlbwe %0,%1,1" : : "r" (data), "r" (i));
-		asm("tlbwe %0,%1,0" : : "r" (tag), "r" (i));
-		asm("isync");
-		pin_table[i].e_pinned = found;
-	}
 
-	mtspr(SPRN_PID, opid);
-	return;
-}
-
-void
-PPC4xx_tlb_unpin(unsigned long va, unsigned long pa, int size)
+/*
+ * ()
+ *
+ * Description:
+ *   This routine...
+ *
+ * Input(s):
+ *
+ *
+ * Output(s):
+ *
+ *
+ * Returns:
+ *
+ *
+ */
+static inline void
+PPC4xx_tlb_write(unsigned long tag, unsigned long data, unsigned int index)
 {
-	/* XXX - To beimplemented. */
+	asm("tlbwe %0,%1,1" : : "r" (data), "r" (index));
+	asm("tlbwe %0,%1,0" : : "r" (tag), "r" (index));
 }
 
+/*
+ * ()
+ *
+ * Description:
+ *   This routine...
+ *
+ * Input(s):
+ *
+ *
+ * Output(s):
+ *
+ *
+ * Returns:
+ *
+ *
+ */
 void
-PPC4xx_tlb_flush_all(void)
+PPC4xx_flush_tlb_all(void)
 {
 	int i;
-	unsigned long flags, opid;
+	unsigned long flags, pid;
 
 	save_flags(flags);
 	cli();
 
-	opid = mfspr(SPRN_PID);
+	pid = mfspr(SPRN_PID);
 	mtspr(SPRN_PID, 0);
 
-	for (i = 0; i < PPC4XX_TLB_SIZE; i++) {
-		unsigned long ov = 0;
+	for (i = pinned; i < PPC4XX_TLB_SIZE; i++) {
+		PPC4xx_tlb_write(0, 0, i);
+	}
+	asm("sync;isync");
 
-		if (pin_table[i].e_pinned)
-			continue;
+	mtspr(SPRN_PID, pid);
+	restore_flags(flags);
+}
 
-		asm("tlbwe %0,%1,0" : : "r" (ov), "r" (i));
-		asm("tlbwe %0,%1,1" : : "r" (ov), "r" (i));
+/*
+ * ()
+ *
+ * Description:
+ *   This routine...
+ *
+ * Input(s):
+ *
+ *
+ * Output(s):
+ *
+ *
+ * Returns:
+ *
+ *
+ */
+void
+PPC4xx_dtlb_miss(struct pt_regs *regs)
+{
+	unsigned long addr = mfspr(SPRN_DEAR);
+	int write = mfspr(SPRN_ESR) & ESR_DST;
+
+	if (PPC4xx_tlb_miss(regs, addr, write) < 0) {
+		sti();
+		do_page_fault(regs, addr, write);
+		cli();
 	}
+	
+}
 
-	asm("sync;isync");
+/*
+ * ()
+ *
+ * Description:
+ *   This routine...
+ *
+ * Input(s):
+ *
+ *
+ * Output(s):
+ *
+ *
+ * Returns:
+ *
+ *
+ */
+void
+PPC4xx_itlb_miss(struct pt_regs *regs)
+{
+	unsigned long addr = regs->nip;
 
-	mtspr(SPRN_PID, opid);
-	restore_flags(flags);
+	if (PPC4xx_tlb_miss(regs, addr, 0) < 0) {
+		sti();
+		do_page_fault(regs, addr, 0);
+		cli();
+	}
 }
 
+/*
+ * ()
+ *
+ * Description:
+ *   This routine...
+ *
+ * Input(s):
+ *
+ *
+ * Output(s):
+ *
+ *
+ * Returns:
+ *
+ *
+ */
 void
-PPC4xx_tlb_flush(unsigned long va, int pid)
+PPC4xx_tlb_pin(unsigned long va, unsigned long pa, int pagesz, int cache)
 {
-	unsigned long i, tag, flags, found = 1, opid;
+	unsigned long tag, data;
+	unsigned long opid;
 
-	save_flags(flags);
-	cli();
+	if (pinned >= PPC4XX_TLB_SIZE)
+		return;
 
 	opid = mfspr(SPRN_PID);
-	mtspr(SPRN_PID, pid);
+	mtspr(SPRN_PID, 0);
 
-	asm("tlbsx. %0,0,%2;beq 1f;li %1,0;1:" : "=r" (i), "=r" (found) : "r" (va));
+	data = (pa & TLB_RPN_MASK) | TLB_WR;
 
-	if (found && pin_table[i].e_pinned == 0) {
-		asm("tlbre %0,%1,0" : "=r" (tag) : "r" (i));
-		tag &= ~ TLB_VALID;
-		asm("tlbwe %0,%1,0" : : "r" (tag), "r" (i));
-	}
+	if (cache)
+		data |= (TLB_EX);
+	else
+		data |= (TLB_G | TLB_I);
 
-	mtspr(SPRN_PID, opid);
+	tag = (va & TLB_EPN_MASK) | TLB_VALID | pagesz;
 
-	restore_flags(flags);
+	PPC4xx_tlb_write(tag, data, pinned++);
+
+	mtspr(SPRN_PID, opid);
+	return;
 }
 
-#if 0
 /*
- * TLB miss handling code.
+ * ()
+ *
+ * Description:
+ *   This routine...
+ *
+ * Input(s):
+ *
+ *
+ * Output(s):
+ *
+ *
+ * Returns:
+ *
+ *
  */
+void
+PPC4xx_tlb_unpin(unsigned long va, unsigned long pa, int size)
+{
+	/* XXX - To be implemented. */
+}
 
 /*
- * Handle TLB faults. We should push this back to assembly code eventually.
- * Caller is responsible for turning off interrupts ...
+ * ()
+ *
+ * Description:
+ *   This routine...
+ *
+ * Input(s):
+ *
+ *
+ * Output(s):
+ *
+ *
+ * Returns:
+ *
+ *
  */
 static inline void
-tlbDropin(unsigned long tlbhi, unsigned long tlblo) {
-	/*
-	 * Avoid the divide at the slight cost of a little too
-	 * much emphasis on the last few entries.
-	 */
-	unsigned long rand = mfspr(SPRN_TBLO);
-	rand &= 0x3f;
-        rand += NTLB_WIRED;
-        if (rand >= NTLB)
-		rand -= NTLB_WIRED;
-
-	asm("tlbwe %0,%1,1" : : "r" (tlblo), "r" (rand));
-	asm("tlbwe %0,%1,0" : : "r" (tlbhi), "r" (rand));
-	asm("isync;sync");
-}
+PPC4xx_tlb_update(unsigned long addr, pte_t *pte)
+{
+        unsigned long data, tag, rand;
+        int i, found = 1;
 
-static inline void
-mkTlbEntry(unsigned long addr, pte_t *pte) {
-	unsigned long tlbhi;
-	unsigned long tlblo;
-	int found = 1;
-	int idx;
+        /* Construct the hardware TLB entry from the Linux-style PTE */
 
-	/*
-	 * Construct the TLB entry.
-	 */
-	tlbhi = addr & ~(PAGE_SIZE-1);
-	tlblo = virt_to_phys(pte_page(*pte)) & TLBLO_RPN;
-	if (pte_val(*pte) & _PAGE_HWWRITE)
-		tlblo |= TLBLO_WR;
-	if (pte_val(*pte) & _PAGE_NO_CACHE)
-		tlblo |= TLBLO_I;
-	tlblo |= TLBLO_EX;
-	if (addr < KERNELBASE)
-		tlblo |= TLBLO_Z_USER;
-	tlbhi |= TLBHI_PGSZ_4K;
-	tlbhi |= TLBHI_VALID;
+        tag = tag = (addr & PAGE_MASK) | TLB_VALID | TLB_PAGESZ(PAGESZ_4K);
+        data = data = (pte_val(*pte) & PAGE_MASK) | TLB_EX | TLB_WR;
 
-	/*
-	 * See if a match already exists in the TLB.
-	 */
-	asm("tlbsx. %0,0,%2;beq 1f;li %1,0;1:" : "=r" (idx), "=r" (found) : "r" (tlbhi));
-	if (found) {
-		/*
-		 * Found an existing entry. Just reuse the index.
-		 */
-		asm("tlbwe %0,%1,0" : : "r" (tlbhi), "r" (idx));
-		asm("tlbwe %0,%1,1" : : "r" (tlblo), "r" (idx));
-	}
-	else {
-		/*
-		 * Do the more expensive operation
-		 */
-		tlbDropin(tlbhi, tlblo);
-	}
-}
-
-/*
- * Mainline of the TLB miss handler. The above inline routines should fold into
- * this one, eliminating most function call overhead.
- */
-#ifdef TLBMISS_DEBUG
-volatile unsigned long miss_start;
-volatile unsigned long miss_end;
-#endif
-
-static inline int tlbMiss(struct pt_regs *regs, unsigned long badaddr, int wasWrite)
-{
-	int spid, ospid;
-	struct mm_struct *mm;
-	pgd_t *pgd;
-	pmd_t *pmd;
-	pte_t *pte;
-
-	if (!user_mode(regs) && (badaddr >= KERNELBASE)) {
-		mm = task[0]->mm;
-		spid = 0;
-#ifdef TLBMISS_DEBUG
-                miss_start = 0;
-#endif
-	}
-	else {
-		mm = current->mm;
-		spid = mfspr(SPRN_PID);
-#ifdef TLBMISS_DEBUG
-                miss_start = 1;
-#endif
-	}
-#ifdef TLBMISS_DEBUG
-        store_cache_range((unsigned long)&miss_start, sizeof(miss_start));
+#if 0
+        if (pte_val(*pte) & _PAGE_HWWRITE)
+                data |= TLB_WR;
 #endif
 
-	pgd = pgd_offset(mm, badaddr);
-	if (pgd_none(*pgd))
-		goto NOGOOD;
-
-	pmd = pmd_offset(pgd, badaddr);
-	if (pmd_none(*pmd))
-		goto NOGOOD;
-
-	pte = pte_offset(pmd, badaddr);
-	if (pte_none(*pte))
-		goto NOGOOD;
-	if (!pte_present(*pte))
-		goto NOGOOD;
-#if 1
-        prohibit_if_guarded(badaddr, sizeof(int));
-#endif
-	if (wasWrite) {
-		if (!pte_write(*pte)) {
-			goto NOGOOD;
-		}
-		set_pte(pte, pte_mkdirty(*pte));
-	}
-	set_pte(pte, pte_mkyoung(*pte));
+        if (pte_val(*pte) & _PAGE_NO_CACHE)
+                data |= TLB_I;
 
-	ospid = mfspr(SPRN_PID);
-	mtspr(SPRN_PID, spid);
-	mkTlbEntry(badaddr, pte);
-	mtspr(SPRN_PID, ospid);
-
-#ifdef TLBMISS_DEBUG
-        miss_end = 0;
-        store_cache_range((unsigned long)&miss_end, sizeof(miss_end));
-#endif
-	return 0;
+        if (pte_val(*pte) & _PAGE_GUARDED)
+                data |= TLB_G;
 
-NOGOOD:
-#ifdef TLBMISS_DEBUG
-        miss_end = 1;
-        store_cache_range((unsigned long)&miss_end, sizeof(miss_end));
-#endif
-        return 1;
-}
+        if (addr < KERNELBASE)
+                data |= TLB_ZSEL(1);
 
-/*
- * End TLB miss handling code.
- */
-/* ---------- */
+        /* Attempt to match the new tag to an existing entry in the TLB. */
 
-/*
- * Used to flush the TLB if the page fault handler decides to change
- * something.
- */
-void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) {
-	int spid;
-	unsigned long flags;
+        asm("tlbsx. %0,0,%2;"
+	    "beq 1f;"
+	    "li %1,0;1:" : "=r" (i), "=r" (found) : "r" (tag));
 
-	save_flags(flags);
-	cli();
+	/*
+	 * If we found a match for the tag, reuse the entry index and update
+	 * the tag and data portions. Otherwise, we did not find a match. Use
+	 * the lower 5 bits of the lower time base register as a pseudo-random
+	 * index into the TLB and replace the entry at that index.
+	 */
 
-	if (addr >= KERNELBASE)
-		spid = 0;
-	else
-		spid = vma->vm_mm->context;
-	tlbFlush1(addr, spid);
+        if (found) {
+		PPC4xx_tlb_write(tag, data, i);
+        } else {
+		rand = mfspr(SPRN_TBLO) & (PPC4XX_TLB_SIZE - 1);
+		rand += pinned;
+		if (rand >= PPC4XX_TLB_SIZE)
+			rand -= pinned;
 
-	restore_flags(flags);
+		PPC4xx_tlb_write(tag, data, rand);
+		asm("isync;sync");
+        }
 }
 
 /*
- * Given a virtual address in the current address space, make
- * sure the associated physical page is present in memory,
- * and if the data is to be modified, that any copy-on-write
- * actions have taken place.
+ * ()
+ *
+ * Description:
+ *   This routine...
+ *
+ * Input(s):
+ *
+ *
+ * Output(s):
+ *
+ *
+ * Returns:
+ *
+ *
  */
-unsigned long make_page_present(unsigned long p, int rw) {
+static int
+PPC4xx_tlb_miss(struct pt_regs *regs, unsigned long addr, int write)
+{
+        unsigned long spid, ospid;
+        struct mm_struct *mm;
+        pgd_t *pgd;
+        pmd_t *pmd;
         pte_t *pte;
-        char c;
 
-        get_user(c, (char *) p);
+        if (!user_mode(regs) && (addr >= KERNELBASE)) {
+                mm = &init_mm;
+                spid = 0;
+        } else {
+                mm = current->mm;
+                spid = mfspr(SPRN_PID);
+        }
+
+        pgd = pgd_offset(mm, addr);
+        if (pgd_none(*pgd))
+                goto bad;
+
+        pmd = pmd_offset(pgd, addr);
+        if (pmd_none(*pmd))
+                goto bad;
 
-        pte = findPTE(current->mm, p);
+        pte = pte_offset(pmd, addr);
         if (pte_none(*pte) || !pte_present(*pte))
-            debug("make_page_present didn't load page", 0);
+                goto bad;
 
-        if (rw) {
-            /*
-            * You have to write-touch the page, so that
-            * zero-filled pages are forced to be copied
-            * rather than still pointing at the zero
-            * page.
-            */
-            extern void tlbFlush1(unsigned long, int);
-            tlbFlush1(p, get_context());
-            put_user(c, (char *) p);
-            if (!pte_write(*pte))
-                debug("make_page_present didn't make page writable", 0);
+        if (write) {
+                if (!pte_write(*pte))
+                        goto bad;
 
-            tlbFlush1(p, get_context());
+                set_pte(pte, pte_mkdirty(*pte));
         }
-        return pte_page(*pte);
-}
-
-void DataTLBMissException(struct pt_regs *regs)
-{
-	unsigned long badaddr = mfspr(SPRN_DEAR);
-	int wasWrite = mfspr(SPRN_ESR) & 0x800000;
-	if (tlbMiss(regs, badaddr, wasWrite)) {
-		sti();
-		do_page_fault(regs, badaddr, wasWrite);
-		cli();
-    	}
-}
+        set_pte(pte, pte_mkyoung(*pte));
 
-void InstructionTLBMissException(struct pt_regs *regs)
-{
-	if (!current) {
-		debug("ITLB Miss with no current task", regs);
-		sti();
-		bad_page_fault(regs, regs->nip);
-		cli();
-		return;
-	}
-	if (tlbMiss(regs, regs->nip, 0)) {
-		sti();
-		do_page_fault(regs, regs->nip, 0);
-		cli();
-    	}
+        ospid = mfspr(SPRN_PID);
+        mtspr(SPRN_PID, spid);
+        PPC4xx_tlb_update(addr, pte);
+        mtspr(SPRN_PID, ospid);
+
+	return (0);
+bad:
+	return (-1);
 }
-
-void DataPageFault(struct pt_regs *regs)
-{
-	unsigned long badaddr = mfspr(SPRN_DEAR);
-	int wasWrite = mfspr(SPRN_ESR) & 0x800000;
-	sti();
-	do_page_fault(regs, badaddr, wasWrite);
-	cli();
-}
-
-void InstructionPageFault(struct pt_regs *regs)
-{
-	if (!current) {
-		debug("ITLB fault with no current task", regs);
-		sti();
-		bad_page_fault(regs, regs->nip);
-		cli();
-		return;
-	}
-	sti();
-	do_page_fault(regs, regs->nip, 0);
-	cli();
-}
-#endif