patch-2.4.14 linux/mm/memory.c

• Lines: 363
• Date: Sun Nov 4 09:29:14 2001
• Orig file: v2.4.13/linux/mm/memory.c
• Orig date: Tue Oct 23 22:48:53 2001

diff -u --recursive --new-file v2.4.13/linux/mm/memory.c linux/mm/memory.c
@@ -78,15 +78,8 @@
 	struct page *page = pte_page(pte);
 	if ((!VALID_PAGE(page)) || PageReserved(page))
 		return;
-	/*
-	 * free_page() used to be able to clear swap cache
-	 * entries.  We may now have to do it manually.
-	 */
-	if (page->mapping) {
-		if (pte_dirty(pte))
-			set_page_dirty(page);
-	}
-		
+	if (pte_dirty(pte))
+		set_page_dirty(page);		
 	free_page_and_swap_cache(page);
 }
 
@@ -184,7 +177,7 @@
 	pgd_t * src_pgd, * dst_pgd;
 	unsigned long address = vma->vm_start;
 	unsigned long end = vma->vm_end;
-	unsigned long cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
+	unsigned long cow = (vma->vm_flags & (VM_SHARED | VM_WRITE)) == VM_WRITE;
 
 	src_pgd = pgd_offset(src, address)-1;
 	dst_pgd = pgd_offset(dst, address)-1;
@@ -325,7 +318,7 @@
 			/* This will eventually call __free_pte on the pte. */
 			tlb_remove_page(tlb, ptep, address + offset);
 		} else {
-			swap_free(pte_to_swp_entry(pte));
+			free_swap_and_cache(pte_to_swp_entry(pte));
 			pte_clear(ptep);
 		}
 	}
@@ -906,7 +899,8 @@
  * change only once the write actually happens. This avoids a few races,
  * and potentially makes it more efficient.
  *
- * We hold the mm semaphore and the page_table_lock on entry and exit.
+ * We hold the mm semaphore and the page_table_lock on entry and exit
+ * with the page_table_lock released.
  */
 static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
 	unsigned long address, pte_t *page_table, pte_t pte)
@@ -916,41 +910,16 @@
 	old_page = pte_page(pte);
 	if (!VALID_PAGE(old_page))
 		goto bad_wp_page;
-	
-	/*
-	 * We can avoid the copy if:
-	 * - we're the only user (count == 1)
-	 * - the only other user is the swap cache,
-	 *   and the only swap cache user is itself,
-	 *   in which case we can just continue to
-	 *   use the same swap cache (it will be
-	 *   marked dirty).
-	 */
-	switch (page_count(old_page)) {
-	int can_reuse;
-	case 3:
-		if (!old_page->buffers)
-			break;
-		/* FallThrough */
-	case 2:
-		if (!PageSwapCache(old_page))
-			break;
-		if (TryLockPage(old_page))
-			break;
-		/* Recheck swapcachedness once the page is locked */
-		can_reuse = exclusive_swap_page(old_page);
-		if (can_reuse)
-			delete_from_swap_cache(old_page);
-		UnlockPage(old_page);
-		if (!can_reuse)
-			break;
-		/* FallThrough */
-	case 1:
-		if (PageReserved(old_page))
-			break;
-		flush_cache_page(vma, address);
-		establish_pte(vma, address, page_table, pte_mkyoung(pte_mkdirty(pte_mkwrite(pte))));
-		return 1;	/* Minor fault */
+
+	if (!TryLockPage(old_page)) {
+		int reuse = can_share_swap_page(old_page);
+		unlock_page(old_page);
+		if (reuse) {
+			flush_cache_page(vma, address);
+			establish_pte(vma, address, page_table, pte_mkyoung(pte_mkdirty(pte_mkwrite(pte))));
+			spin_unlock(&mm->page_table_lock);
+			return 1;	/* Minor fault */
+		}
 	}
 
 	/*
@@ -963,7 +932,6 @@
 	if (!new_page)
 		goto no_mem;
 	copy_cow_page(old_page,new_page,address);
-	page_cache_release(old_page);
 
 	/*
 	 * Re-check the pte - we dropped the lock
@@ -973,19 +941,22 @@
 		if (PageReserved(old_page))
 			++mm->rss;
 		break_cow(vma, new_page, address, page_table);
+		lru_cache_add(new_page);
 
 		/* Free the old page.. */
 		new_page = old_page;
 	}
+	spin_unlock(&mm->page_table_lock);
 	page_cache_release(new_page);
+	page_cache_release(old_page);
 	return 1;	/* Minor fault */
 
 bad_wp_page:
+	spin_unlock(&mm->page_table_lock);
 	printk("do_wp_page: bogus page at address %08lx (page 0x%lx)\n",address,(unsigned long)old_page);
 	return -1;
 no_mem:
 	page_cache_release(old_page);
-	spin_lock(&mm->page_table_lock);
 	return -1;
 }
 
@@ -1090,10 +1061,6 @@
 	 */
 	num = valid_swaphandles(entry, &offset);
 	for (i = 0; i < num; offset++, i++) {
-		/* Don't block on I/O for read-ahead */
-		if (atomic_read(&nr_async_pages) >=
-		    pager_daemon.swap_cluster << page_cluster)
-			break;
 		/* Ok, do the async read-ahead now */
 		new_page = read_swap_cache_async(SWP_ENTRY(SWP_TYPE(entry), offset));
 		if (!new_page)
@@ -1103,12 +1070,9 @@
 	return;
 }
 
-/* Swap 80% full? Release the pages as they are paged in.. */
-#define vm_swap_full() \
-	(swapper_space.nrpages*5 > total_swap_pages*4)
-
 /*
- * We hold the mm semaphore and the page_table_lock on entry and exit.
+ * We hold the mm semaphore and the page_table_lock on entry and
+ * should release the pagetable lock on exit..
  */
 static int do_swap_page(struct mm_struct * mm,
 	struct vm_area_struct * vma, unsigned long address,
@@ -1125,23 +1089,21 @@
 		swapin_readahead(entry);
 		page = read_swap_cache_async(entry);
 		if (!page) {
-			spin_lock(&mm->page_table_lock);
 			/*
 			 * Back out if somebody else faulted in this pte while
 			 * we released the page table lock.
 			 */
-			return pte_same(*page_table, orig_pte) ? -1 : 1;
+			int retval;
+			spin_lock(&mm->page_table_lock);
+			retval = pte_same(*page_table, orig_pte) ? -1 : 1;
+			spin_unlock(&mm->page_table_lock);
+			return retval;
 		}
 
 		/* Had to read the page from swap area: Major fault */
 		ret = 2;
 	}
 
-	/*
-	 * Freeze the "shared"ness of the page, ie page_count + swap_count.
-	 * Must lock page before transferring our swap count to already
-	 * obtained page count.
-	 */
 	lock_page(page);
 
 	/*
@@ -1150,26 +1112,23 @@
 	 */
 	spin_lock(&mm->page_table_lock);
 	if (!pte_same(*page_table, orig_pte)) {
-		UnlockPage(page);
+		spin_unlock(&mm->page_table_lock);
+		unlock_page(page);
 		page_cache_release(page);
 		return 1;
 	}
-		
+
 	/* The page isn't present yet, go ahead with the fault. */
+		
+	swap_free(entry);
+	if (vm_swap_full())
+		remove_exclusive_swap_page(page);
+
 	mm->rss++;
 	pte = mk_pte(page, vma->vm_page_prot);
-
-	swap_free(entry);
-	mark_page_accessed(page);
-	if (exclusive_swap_page(page)) {
-		if (write_access || vm_swap_full()) {
-			pte = pte_mkdirty(pte);
-			if (vma->vm_flags & VM_WRITE)
-				pte = pte_mkwrite(pte);
-			delete_from_swap_cache(page);
-		}
-	}
-	UnlockPage(page);
+	if (write_access && can_share_swap_page(page))
+		pte = pte_mkdirty(pte_mkwrite(pte));
+	unlock_page(page);
 
 	flush_page_to_ram(page);
 	flush_icache_page(vma, page);
@@ -1177,6 +1136,7 @@
 
 	/* No need to invalidate - it was non-present before */
 	update_mmu_cache(vma, address, pte);
+	spin_unlock(&mm->page_table_lock);
 	return ret;
 }
 
@@ -1207,21 +1167,23 @@
 		spin_lock(&mm->page_table_lock);
 		if (!pte_none(*page_table)) {
 			page_cache_release(page);
+			spin_unlock(&mm->page_table_lock);
 			return 1;
 		}
 		mm->rss++;
 		flush_page_to_ram(page);
 		entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
+		lru_cache_add(page);
 	}
 
 	set_pte(page_table, entry);
 
 	/* No need to invalidate - it was non-present before */
 	update_mmu_cache(vma, addr, entry);
+	spin_unlock(&mm->page_table_lock);
 	return 1;	/* Minor fault */
 
 no_mem:
-	spin_lock(&mm->page_table_lock);
 	return -1;
 }
 
@@ -1235,7 +1197,7 @@
  * do not need to flush old virtual caches or the TLB.
  *
  * This is called with the MM semaphore held and the page table
- * spinlock held.
+ * spinlock held. Exit with the spinlock released.
  */
 static int do_no_page(struct mm_struct * mm, struct vm_area_struct * vma,
 	unsigned long address, int write_access, pte_t *page_table)
@@ -1247,18 +1209,27 @@
 		return do_anonymous_page(mm, vma, page_table, write_access, address);
 	spin_unlock(&mm->page_table_lock);
 
-	/*
-	 * The third argument is "no_share", which tells the low-level code
-	 * to copy, not share the page even if sharing is possible.  It's
-	 * essentially an early COW detection.
-	 */
-	new_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, (vma->vm_flags & VM_SHARED)?0:write_access);
+	new_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, 0);
 
-	spin_lock(&mm->page_table_lock);
 	if (new_page == NULL)	/* no page was available -- SIGBUS */
 		return 0;
 	if (new_page == NOPAGE_OOM)
 		return -1;
+
+	/*
+	 * Should we do an early C-O-W break?
+	 */
+	if (write_access && !(vma->vm_flags & VM_SHARED)) {
+		struct page * page = alloc_page(GFP_HIGHUSER);
+		if (!page)
+			return -1;
+		copy_highpage(page, new_page);
+		page_cache_release(new_page);
+		lru_cache_add(page);
+		new_page = page;
+	}
+
+	spin_lock(&mm->page_table_lock);
 	/*
 	 * This silly early PAGE_DIRTY setting removes a race
 	 * due to the bad i386 page protection. But it's valid
@@ -1275,20 +1246,19 @@
 		flush_page_to_ram(new_page);
 		flush_icache_page(vma, new_page);
 		entry = mk_pte(new_page, vma->vm_page_prot);
-		if (write_access) {
+		if (write_access)
 			entry = pte_mkwrite(pte_mkdirty(entry));
-		} else if (page_count(new_page) > 1 &&
-			   !(vma->vm_flags & VM_SHARED))
-			entry = pte_wrprotect(entry);
 		set_pte(page_table, entry);
 	} else {
 		/* One of our sibling threads was faster, back out. */
 		page_cache_release(new_page);
+		spin_unlock(&mm->page_table_lock);
 		return 1;
 	}
 
 	/* no need to invalidate: a not-present page shouldn't be cached */
 	update_mmu_cache(vma, address, entry);
+	spin_unlock(&mm->page_table_lock);
 	return 2;	/* Major fault */
 }
 
@@ -1309,6 +1279,9 @@
  * The adding of pages is protected by the MM semaphore (which we hold),
  * so we don't need to worry about a page being suddenly been added into
  * our VM.
+ *
+ * We enter with the pagetable spinlock held, we are supposed to
+ * release it when done.
  */
 static inline int handle_pte_fault(struct mm_struct *mm,
 	struct vm_area_struct * vma, unsigned long address,
@@ -1336,6 +1309,7 @@
 	}
 	entry = pte_mkyoung(entry);
 	establish_pte(vma, address, pte, entry);
+	spin_unlock(&mm->page_table_lock);
 	return 1;
 }
 
@@ -1345,7 +1319,6 @@
 int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct * vma,
 	unsigned long address, int write_access)
 {
-	int ret = -1;
 	pgd_t *pgd;
 	pmd_t *pmd;
 
@@ -1362,10 +1335,10 @@
 	if (pmd) {
 		pte_t * pte = pte_alloc(mm, pmd, address);
 		if (pte)
-			ret = handle_pte_fault(mm, vma, address, write_access, pte);
+			return handle_pte_fault(mm, vma, address, write_access, pte);
 	}
 	spin_unlock(&mm->page_table_lock);
-	return ret;
+	return -1;
 }
 
 /*