diff options
Diffstat (limited to 'private/ntos/mm/wslist.c')
| -rw-r--r-- | private/ntos/mm/wslist.c | 2973 |
1 files changed, 2973 insertions, 0 deletions
diff --git a/private/ntos/mm/wslist.c b/private/ntos/mm/wslist.c new file mode 100644 index 000000000..a3d16284d --- /dev/null +++ b/private/ntos/mm/wslist.c @@ -0,0 +1,2973 @@ +/*++ + +Copyright (c) 1989 Microsoft Corporation + +Module Name: + + wslist.c + +Abstract: + + This module contains routines which operate on the working + set list structure. + +Author: + + Lou Perazzoli (loup) 10-Apr-1989 + +Revision History: + +--*/ + +#include "mi.h" + +#define MM_SYSTEM_CACHE_THRESHOLD ((1024*1024) / PAGE_SIZE) + +extern ULONG MmMaximumWorkingSetSize; +ULONG MmFaultsTakenToGoAboveMaxWs = 100; +ULONG MmFaultsTakenToGoAboveMinWs = 16; + +ULONG MmSystemCodePage; +ULONG MmSystemCachePage; +ULONG MmPagedPoolPage; +ULONG MmSystemDriverPage; + +#define MM_RETRY_COUNT 2 + +VOID +MiCheckWsleHash ( + IN PMMWSL WorkingSetList + ); + +VOID +MiEliminateWorkingSetEntry ( + IN ULONG WorkingSetIndex, + IN PMMPTE PointerPte, + IN PMMPFN Pfn, + IN PMMWSLE Wsle + ); + +ULONG +MiAddWorkingSetPage ( + IN PMMSUPPORT WsInfo + ); + +VOID +MiRemoveWorkingSetPages ( + IN PMMWSL WorkingSetList, + IN PMMSUPPORT WsInfo + ); + +VOID +MiCheckNullIndex ( + IN PMMWSL WorkingSetList + ); + +VOID +MiDumpWsleInCacheBlock ( + IN PMMPTE CachePte + ); + +ULONG +MiDumpPteInCacheBlock ( + IN PMMPTE PointerPte + ); + + +#ifdef ALLOC_PRAGMA +#pragma alloc_text(PAGELK, MmAdjustWorkingSetSize) +#pragma alloc_text(PAGELK, MiEmptyWorkingSet) +#endif // ALLOC_PRAGMA + + +ULONG +MiLocateAndReserveWsle ( + PMMSUPPORT WsInfo + ) + +/*++ + +Routine Description: + + This function examines the Working Set List for the current + process and locates an entry to contain a new page. If the + working set is not currently at its quota, the new page is + added without removing a page, if the working set it at its + quota a page is removed from the working set and the new + page added in its place. + +Arguments: + + None. + +Return Value: + + Returns the working set index which is now reserved for the + next page to be added. + +Environment: + + Kernel mode, APC's disabled, working set lock. Pfn lock NOT held. + +--*/ + +{ + ULONG WorkingSetIndex; + ULONG NumberOfCandidates; + PMMWSL WorkingSetList; + PMMWSLE Wsle; + PMMPTE PointerPte; + ULONG CurrentSize; + ULONG AvailablePageThreshold; + ULONG TheNextSlot; + ULONG QuotaIncrement; + LARGE_INTEGER CurrentTime; + KIRQL OldIrql; + + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; + AvailablePageThreshold = 0; + + if (WsInfo == &MmSystemCacheWs) { + MM_SYSTEM_WS_LOCK_ASSERT(); + AvailablePageThreshold = MM_SYSTEM_CACHE_THRESHOLD; + } + + // + // Update page fault counts. + // + + WsInfo->PageFaultCount += 1; + MmInfoCounters.PageFaultCount += 1; + + // + // Determine if a page should be removed from the working set. + // + +recheck: + + CurrentSize = WsInfo->WorkingSetSize; + ASSERT (CurrentSize <= WorkingSetList->LastInitializedWsle); + + if (CurrentSize < WsInfo->MinimumWorkingSetSize) { + + // + // Working set is below minimum, allow it to grow unconditionally. + // + + AvailablePageThreshold = 0; + QuotaIncrement = 1; + + } else if (WsInfo->AllowWorkingSetAdjustment == MM_FORCE_TRIM) { + + // + // The working set manager cannot attach to this process + // to trim it. Force a trim now and update the working + // set managers fields properly to indicate a trim occurred. + // + + MiTrimWorkingSet (20, WsInfo, TRUE); + KeQuerySystemTime (&CurrentTime); + WsInfo->LastTrimTime = CurrentTime; + WsInfo->LastTrimFaultCount = WsInfo->PageFaultCount; + LOCK_EXPANSION_IF_ALPHA (OldIrql); + WsInfo->AllowWorkingSetAdjustment = TRUE; + UNLOCK_EXPANSION_IF_ALPHA (OldIrql); + + // + // Set the quota to the current size. + // + + WorkingSetList->Quota = WsInfo->WorkingSetSize; + if (WorkingSetList->Quota < WsInfo->MinimumWorkingSetSize) { + WorkingSetList->Quota = WsInfo->MinimumWorkingSetSize; + } + goto recheck; + + } else if (CurrentSize < WorkingSetList->Quota) { + + // + // Working set is below quota, allow it to grow with few pages + // available. + // + + AvailablePageThreshold = 10; + QuotaIncrement = 1; + } else if (CurrentSize < WsInfo->MaximumWorkingSetSize) { + + // + // Working set is between min and max. Allow it to grow if enough + // faults have been taken since last adjustment. + // + + if ((WsInfo->PageFaultCount - WsInfo->LastTrimFaultCount) < + MmFaultsTakenToGoAboveMinWs) { + AvailablePageThreshold = MmMoreThanEnoughFreePages + 200; + if (WsInfo->MemoryPriority == MEMORY_PRIORITY_FOREGROUND) { + AvailablePageThreshold -= 250; + } + } else { + AvailablePageThreshold = MmWsAdjustThreshold; + } + QuotaIncrement = MmWorkingSetSizeIncrement; + } else { + + // + // Working set is above max. + // + + if ((WsInfo->PageFaultCount - WsInfo->LastTrimFaultCount) < + (CurrentSize >> 3)) { + AvailablePageThreshold = MmMoreThanEnoughFreePages +200; + if (WsInfo->MemoryPriority == MEMORY_PRIORITY_FOREGROUND) { + AvailablePageThreshold -= 250; + } + } else { + AvailablePageThreshold += MmWsExpandThreshold; + } + QuotaIncrement = MmWorkingSetSizeExpansion; + + if (CurrentSize > MM_MAXIMUM_WORKING_SET) { + AvailablePageThreshold = 0xffffffff; + QuotaIncrement = 1; + } + } + + if ((!WsInfo->AddressSpaceBeingDeleted) && (AvailablePageThreshold != 0)) { + if ((MmAvailablePages <= AvailablePageThreshold) || + (WsInfo->WorkingSetExpansionLinks.Flink == MM_NO_WS_EXPANSION)) { + + // + // Toss a page out of the working set. + // + + WorkingSetIndex = WorkingSetList->NextSlot; + TheNextSlot = WorkingSetIndex; + ASSERT (WorkingSetIndex <= WorkingSetList->LastEntry); + ASSERT (WorkingSetIndex >= WorkingSetList->FirstDynamic); + NumberOfCandidates = 0; + + for (; ; ) { + + // + // Find a valid entry within the set. + // + + WorkingSetIndex += 1; + if (WorkingSetIndex >= WorkingSetList->LastEntry) { + WorkingSetIndex = WorkingSetList->FirstDynamic; + } + + if (Wsle[WorkingSetIndex].u1.e1.Valid != 0) { + PointerPte = MiGetPteAddress ( + Wsle[WorkingSetIndex].u1.VirtualAddress); + if ((MI_GET_ACCESSED_IN_PTE(PointerPte) == 0) || + (NumberOfCandidates > MM_WORKING_SET_LIST_SEARCH)) { + + // + // Don't throw this guy out if he is the same one + // we did last time. + // + + if ((WorkingSetIndex != TheNextSlot) && + MiFreeWsle (WorkingSetIndex, + WsInfo, + PointerPte)) { + + // + // This entry was removed. + // + + WorkingSetList->NextSlot = WorkingSetIndex; + break; + } + } + MI_SET_ACCESSED_IN_PTE (PointerPte, 0); + NumberOfCandidates += 1; + } + + if (WorkingSetIndex == TheNextSlot) { + + // + // Entire working set list has been searched, increase + // the working set size. + // + + break; + } + } + } + } + ASSERT (WsInfo->WorkingSetSize <= WorkingSetList->Quota); + WsInfo->WorkingSetSize += 1; + + if (WsInfo->WorkingSetSize > WorkingSetList->Quota) { + + // + // Add 1 to the quota and check boundary conditions. + // + + WorkingSetList->Quota += QuotaIncrement; + + WsInfo->LastTrimFaultCount = WsInfo->PageFaultCount; + + if (WorkingSetList->Quota > WorkingSetList->LastInitializedWsle) { + + // + // Add more pages to the working set list structure. + // + + MiAddWorkingSetPage (WsInfo); + } + } + + // + // Get the working set entry from the free list. + // + + ASSERT (WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle); + + WorkingSetIndex = WorkingSetList->FirstFree; + WorkingSetList->FirstFree = Wsle[WorkingSetIndex].u1.Long >> MM_FREE_WSLE_SHIFT; + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); + + if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) { + MmPagesAboveWsMinimum += 1; + } + + if (WsInfo->WorkingSetSize >= WsInfo->PeakWorkingSetSize) { + WsInfo->PeakWorkingSetSize = WsInfo->WorkingSetSize; + } + + if (WorkingSetIndex > WorkingSetList->LastEntry) { + WorkingSetList->LastEntry = WorkingSetIndex; + } + + // + // Mark the entry as not valid. + // + + ASSERT (Wsle[WorkingSetIndex].u1.e1.Valid == 0); + + return WorkingSetIndex; +} + +ULONG +MiRemovePageFromWorkingSet ( + IN PMMPTE PointerPte, + IN PMMPFN Pfn1, + IN PMMSUPPORT WsInfo + ) + +/*++ + +Routine Description: + + This function removes the page mapped by the specified PTE from + the process's working set list. + +Arguments: + + PointerPte - Supplies a pointer to the PTE mapping the page to + be removed from the working set list. + + Pfn1 - Supplies a pointer to the PFN database element referred to + by the PointerPte. + +Return Value: + + Returns TRUE if the specified page was locked in the working set, + FALSE otherwise. + +Environment: + + Kernel mode, APC's disabled, working set and pfn mutexes held. + +--*/ + +{ + ULONG WorkingSetIndex; + PVOID VirtualAddress; + ULONG Entry; + PVOID SwapVa; + MMWSLENTRY Locked; + PMMWSL WorkingSetList; + PMMWSLE Wsle; + KIRQL OldIrql; + + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; + + VirtualAddress = MiGetVirtualAddressMappedByPte (PointerPte); + WorkingSetIndex = MiLocateWsle (VirtualAddress, + WorkingSetList, + Pfn1->u1.WsIndex); + + ASSERT (WorkingSetIndex != WSLE_NULL_INDEX); + LOCK_PFN (OldIrql); + MiEliminateWorkingSetEntry (WorkingSetIndex, + PointerPte, + Pfn1, + Wsle); + UNLOCK_PFN (OldIrql); + + // + // Check to see if this entry is locked in the working set + // or locked in memory. + // + + Locked = Wsle[WorkingSetIndex].u1.e1; + MiRemoveWsle (WorkingSetIndex, WorkingSetList); + + // + // Add this entry to the list of free working set entries + // and adjust the working set count. + // + + MiReleaseWsle ((ULONG)WorkingSetIndex, WsInfo); + + if ((Locked.LockedInWs == 1) || (Locked.LockedInMemory == 1)) { + + // + // This entry is locked. + // + + WorkingSetList->FirstDynamic -= 1; + + if (WorkingSetIndex != WorkingSetList->FirstDynamic) { + + SwapVa = Wsle[WorkingSetList->FirstDynamic].u1.VirtualAddress; + SwapVa = PAGE_ALIGN (SwapVa); + + PointerPte = MiGetPteAddress (SwapVa); + Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); +#if 0 + Entry = MiLocateWsleAndParent (SwapVa, + &Parent, + WorkingSetList, + Pfn1->u1.WsIndex); + + // + // Swap the removed entry with the last locked entry + // which is located at first dynamic. + // + + MiSwapWslEntries (Entry, Parent, WorkingSetIndex, WorkingSetList); +#endif //0 + + Entry = MiLocateWsle (SwapVa, WorkingSetList, Pfn1->u1.WsIndex); + + MiSwapWslEntries (Entry, WorkingSetIndex, WsInfo); + + } + return TRUE; + } else { + ASSERT (WorkingSetIndex >= WorkingSetList->FirstDynamic); + } + return FALSE; +} + + +VOID +MiReleaseWsle ( + IN ULONG WorkingSetIndex, + IN PMMSUPPORT WsInfo + ) + +/*++ + +Routine Description: + + This function releases a previously reserved working set entry to + be reused. A release occurs when a page fault is retried due to + changes in PTEs and working sets during an I/O operation. + +Arguments: + + WorkingSetIndex - Supplies the index of the working set entry to + release. + +Return Value: + + None. + +Environment: + + Kernel mode, APC's disabled, working set lock held and PFN lock held. + +--*/ + +{ + PMMWSL WorkingSetList; + PMMWSLE Wsle; + + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; +#if DBG + if (WsInfo == &MmSystemCacheWs) { + MM_SYSTEM_WS_LOCK_ASSERT(); + } +#endif //DBG + + ASSERT (WorkingSetIndex <= WorkingSetList->LastInitializedWsle); + + // + // Put the entry on the free list and decrement the current + // size. + // + + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); + Wsle[WorkingSetIndex].u1.Long = WorkingSetList->FirstFree << MM_FREE_WSLE_SHIFT; + WorkingSetList->FirstFree = WorkingSetIndex; + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); + if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) { + MmPagesAboveWsMinimum -= 1; + } + WsInfo->WorkingSetSize -= 1; + return; + +} + +VOID +MiUpdateWsle ( + IN OUT PULONG DesiredIndex, + IN PVOID VirtualAddress, + PMMWSL WorkingSetList, + IN PMMPFN Pfn + ) + +/*++ + +Routine Description: + + This routine updates a reserved working set entry to place it into + the valid state. + +Arguments: + + DesiredIndex - Supplies the index of the working set entry to update. + + VirtualAddress - Supplies the virtual address which the working set + entry maps. + + WsInfo - Supples a pointer to the working set info block for the + process (or system cache). + + Pfn - Supplies a pointer to the PFN element for the page. + +Return Value: + + None. + +Environment: + + Kernel mode, APC's disabled, working set lock held and PFN lock held. + +--*/ + +{ + PMMWSLE Wsle; + ULONG Index; + ULONG WorkingSetIndex; + + WorkingSetIndex = *DesiredIndex; + Wsle = WorkingSetList->Wsle; + +#if DBG + if (WorkingSetList == MmSystemCacheWorkingSetList) { + ASSERT ((VirtualAddress < (PVOID)PTE_BASE) || + (VirtualAddress >= (PVOID)MM_SYSTEM_SPACE_START)); + } else { + ASSERT (VirtualAddress < (PVOID)MM_SYSTEM_SPACE_START); + } + ASSERT (WorkingSetIndex >= WorkingSetList->FirstDynamic); +#endif //DBG + + if (WorkingSetList == MmSystemCacheWorkingSetList) { + + MM_SYSTEM_WS_LOCK_ASSERT(); + + // + // count system space inserts and removals. + // + + if (VirtualAddress < (PVOID)MM_SYSTEM_CACHE_START) { + MmSystemCodePage += 1; + } else if (VirtualAddress < MM_PAGED_POOL_START) { + MmSystemCachePage += 1; + } else if (VirtualAddress < MmNonPagedSystemStart) { + MmPagedPoolPage += 1; + } else { + MmSystemDriverPage += 1; + } + } + + // + // Make the wsle valid, referring to the corresponding virtual + // page number. + // + + // + // The value 0 is invalid. This is due to the fact that the working + // set lock is a process wide lock and two threads in different + // processes could be adding the same physical page to their working + // sets. Each one could see the WsIndex field in the PFN as 0, and + // set the direct bit. To solve this, the WsIndex field is set to + // the current thread pointer. + // + + ASSERT (Pfn->u1.WsIndex != 0); + +#if DBG + if (Pfn->u1.WsIndex <= WorkingSetList->LastInitializedWsle) { + ASSERT ((PAGE_ALIGN(VirtualAddress) != + PAGE_ALIGN(Wsle[Pfn->u1.WsIndex].u1.VirtualAddress)) || + (Wsle[Pfn->u1.WsIndex].u1.e1.Valid == 0)); + } +#endif //DBG + + Wsle[WorkingSetIndex].u1.VirtualAddress = VirtualAddress; + Wsle[WorkingSetIndex].u1.Long &= ~(PAGE_SIZE - 1); + Wsle[WorkingSetIndex].u1.e1.Valid = 1; + + if (Pfn->u1.WsIndex == (ULONG)PsGetCurrentThread()) { + + // + // Directly index into the WSL for this entry via the PFN database + // element. + // + + Pfn->u1.WsIndex = WorkingSetIndex; + Wsle[WorkingSetIndex].u1.e1.Direct = 1; + return; + + } else if (WorkingSetList->HashTable == NULL) { + + // + // Try to insert at WsIndex. + // + + Index = Pfn->u1.WsIndex; + + if ((Index < WorkingSetList->LastInitializedWsle) && + (Index > WorkingSetList->FirstDynamic) && + (Index != WorkingSetIndex)) { + + if (Wsle[Index].u1.e1.Valid) { + + if (Wsle[Index].u1.e1.Direct) { + + // + // Only move direct indexed entries. + // + + PMMSUPPORT WsInfo; + + if (Wsle == MmSystemCacheWsle) { + WsInfo = &MmSystemCacheWs; + } else { + WsInfo = &PsGetCurrentProcess()->Vm; + } + + MiSwapWslEntries (Index, WorkingSetIndex, WsInfo); + WorkingSetIndex = Index; + } + } else { + + // + // On free list, try to remove quickly without walking + // all the free pages. + // + + ULONG FreeIndex; + MMWSLE Temp; + + FreeIndex = 0; + + if (WorkingSetList->FirstFree == Index) { + WorkingSetList->FirstFree = WorkingSetIndex; + Temp = Wsle[WorkingSetIndex]; + Wsle[WorkingSetIndex] = Wsle[Index]; + Wsle[Index] = Temp; + WorkingSetIndex = Index; + ASSERT (((Wsle[WorkingSetList->FirstFree].u1.Long >> MM_FREE_WSLE_SHIFT) + <= WorkingSetList->LastInitializedWsle) || + ((Wsle[WorkingSetList->FirstFree].u1.Long >> MM_FREE_WSLE_SHIFT) + == WSLE_NULL_INDEX)); + } else if (Wsle[Index - 1].u1.e1.Valid == 0) { + if ((Wsle[Index - 1].u1.Long >> MM_FREE_WSLE_SHIFT) == Index) { + FreeIndex = Index - 1; + } + } else if (Wsle[Index + 1].u1.e1.Valid == 0) { + if ((Wsle[Index + 1].u1.Long >> MM_FREE_WSLE_SHIFT) == Index) { + FreeIndex = Index + 1; + } + } + if (FreeIndex != 0) { + + // + // Link the Wsle into the free list. + // + + Temp = Wsle[WorkingSetIndex]; + Wsle[FreeIndex].u1.Long = WorkingSetIndex << MM_FREE_WSLE_SHIFT; + Wsle[WorkingSetIndex] = Wsle[Index]; + Wsle[Index] = Temp; + WorkingSetIndex = Index; + + ASSERT (((Wsle[FreeIndex].u1.Long >> MM_FREE_WSLE_SHIFT) + <= WorkingSetList->LastInitializedWsle) || + ((Wsle[FreeIndex].u1.Long >> MM_FREE_WSLE_SHIFT) + == WSLE_NULL_INDEX)); + } + + } + *DesiredIndex = WorkingSetIndex; + + if (WorkingSetIndex > WorkingSetList->LastEntry) { + WorkingSetList->LastEntry = WorkingSetIndex; + } + } + } + + // + // Insert the valid WSLE into the working set list tree. + // + + MiInsertWsle (WorkingSetIndex, WorkingSetList); + return; +} + + +#if 0 //COMMENTED OUT!!! +ULONG +MiGetFirstFreeWsle ( + IN PMMSUPPORT WsInfo + ) + +/*++ + +Routine Description: + + This function removes the first entry from the WSLE free list and + updates the WSLIST structures. + + NOTE: There must be an element on the free list! + +Arguments: + + WsInfo - Supples a pointer to the working set info block for the + process (or system cache). + +Return Value: + + Free WSLE. + +Environment: + + Kernel mode, APC's disabled, working set lock held. + +--*/ + +{ + PMMWSL WorkingSetList; + PMMWSLE Wsle; + ULONG WorkingSetIndex; + + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; + + // + // Get the working set entry from the free list. + // + + ASSERT (WorkingSetList->FirstFree != WSLE_NULL_INDEX); + + WorkingSetIndex = WorkingSetList->FirstFree; + WorkingSetList->FirstFree = Wsle[WorkingSetIndex].u1.Long >> MM_FREE_WSLE_SHIFT; + + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); + + WsInfo->WorkingSetSize += 1; + + if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) { + MmPagesAboveWsMinimum += 1; + } + + if (WsInfo->WorkingSetSize >= WsInfo->PeakWorkingSetSize) { + WsInfo->PeakWorkingSetSize = WsInfo->WorkingSetSize; + } + + if (WorkingSetIndex > WorkingSetList->LastEntry) { + WorkingSetList->LastEntry = WorkingSetIndex; + } + + if (WsInfo->WorkingSetSize > WorkingSetList->Quota) { + WorkingSetList->Quota = WsInfo->WorkingSetSize; + } + + // + // Mark the entry as not valid. + // + + ASSERT (Wsle[WorkingSetIndex].u1.e1.Valid == 0); + + return WorkingSetIndex; +} +#endif //0 COMMENTED OUT!!! + +VOID +MiTakePageFromWorkingSet ( + IN ULONG Entry, + IN PMMSUPPORT WsInfo, + IN PMMPTE PointerPte + ) + +/*++ + +Routine Description: + + This routine is a wrapper for MiFreeWsle that acquires the pfn + lock. Used by pagable code. + +Arguments: + + same as free wsle. + +Return Value: + + same as free wsle. + +Environment: + + Kernel mode, PFN lock NOT held, working set lock held. + +--*/ + +{ + KIRQL OldIrql; +//fixfix is this still needed? + MiFreeWsle (Entry, WsInfo, PointerPte); + return; +} + +ULONG +MiFreeWsle ( + IN ULONG WorkingSetIndex, + IN PMMSUPPORT WsInfo, + IN PMMPTE PointerPte + ) + +/*++ + +Routine Description: + + This routine frees the specified WSLE and decrements the share + count for the corresponding page, putting the PTE into a transition + state if the share count goes to 0. + +Arguments: + + WorkingSetIndex - Supplies the index of the working set entry to free. + + WsInfo - Supplies a pointer to the working set structure (process or + system cache). + + PointerPte - Supplies a pointer to the PTE for the working set entry. + +Return Value: + + Returns TRUE if the WSLE was removed, FALSE if it was not removed. + Pages with valid PTEs are not removed (i.e. page table pages + that contain valid or transition PTEs). + +Environment: + + Kernel mode, APC's disabled, working set lock. Pfn lock NOT held. + +--*/ + +{ + PMMPFN Pfn1; + ULONG NumberOfCandidates = 0; + PMMWSL WorkingSetList; + PMMWSLE Wsle; + KIRQL OldIrql; + + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; + +#if DBG + if (WsInfo == &MmSystemCacheWs) { + MM_SYSTEM_WS_LOCK_ASSERT(); + } +#endif //DBG + + ASSERT (Wsle[WorkingSetIndex].u1.e1.Valid == 1); + + // + // Check to see the located entry is elgible for removal. + // + + ASSERT (PointerPte->u.Hard.Valid == 1); + + // + // Check to see if this is a page table with valid PTEs. + // + // Note, don't clear the access bit for page table pages + // with valid PTEs as this could cause an access trap fault which + // would not be handled (it is only handled for PTEs not PDEs). + // + + Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); + + LOCK_PFN (OldIrql); + + // + // If the PTE is page table page with non-zero share count or + // within the system cache with its reference count greater + // than 0, don't remove it. + // + + if (WsInfo == &MmSystemCacheWs) { + if (Pfn1->u3.e2.ReferenceCount > 1) { + UNLOCK_PFN (OldIrql); + return FALSE; + } + } else { + if ((Pfn1->u2.ShareCount > 1) && + (Pfn1->u3.e1.PrototypePte == 0)) { + + ASSERT ((Wsle[WorkingSetIndex].u1.VirtualAddress >= (PVOID)PTE_BASE) && + (Wsle[WorkingSetIndex].u1.VirtualAddress<= (PVOID)PDE_TOP)); + + + // + // Don't remove page table pages from the working set until + // all transition pages have exited. + // + + UNLOCK_PFN (OldIrql); + return FALSE; + } + } + + // + // Found a candidate, remove the page from the working set. + // + + MiEliminateWorkingSetEntry (WorkingSetIndex, + PointerPte, + Pfn1, + Wsle); + UNLOCK_PFN (OldIrql); + + // + // Remove the working set entry from the working set tree. + // + + MiRemoveWsle (WorkingSetIndex, WorkingSetList); + + // + // Put the entry on the free list and decrement the current + // size. + // + + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); + Wsle[WorkingSetIndex].u1.Long = WorkingSetList->FirstFree << MM_FREE_WSLE_SHIFT; + WorkingSetList->FirstFree = WorkingSetIndex; + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); + + if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) { + MmPagesAboveWsMinimum -= 1; + } + WsInfo->WorkingSetSize -= 1; + +#if 0 + if ((WsInfo == &MmSystemCacheWs) && + (Pfn1->u3.e1.Modified == 1)) { + MiDumpWsleInCacheBlock (PointerPte); + } +#endif //0 + return TRUE; +} + +VOID +MiInitializeWorkingSetList ( + IN PEPROCESS CurrentProcess + ) + +/*++ + +Routine Description: + + This routine initializes a process's working set to the empty + state. + +Arguments: + + CurrentProcess - Supplies a pointer to the process to initialize. + +Return Value: + + None. + +Environment: + + Kernel mode, APC's disabled. + +--*/ + +{ + ULONG i; + PMMWSLE WslEntry; + ULONG CurrentEntry; + PMMPTE PointerPte; + PMMPFN Pfn1; + ULONG NumberOfEntriesMapped; + ULONG CurrentVa; + ULONG WorkingSetPage; + MMPTE TempPte; + KIRQL OldIrql; + + WslEntry = MmWsle; + + // + // Initialize the temporary double mapping portion of hyperspace, if + // it has not already been done. + // + // Initialize the working set list control cells. + // + + MmWorkingSetList->LastEntry = CurrentProcess->Vm.MinimumWorkingSetSize; + MmWorkingSetList->Quota = MmWorkingSetList->LastEntry; + MmWorkingSetList->WaitingForImageMapping = (PKEVENT)NULL; + MmWorkingSetList->HashTable = NULL; + MmWorkingSetList->HashTableSize = 0; + MmWorkingSetList->Wsle = MmWsle; + + // + // Fill in the reserved slots. + // + + WslEntry->u1.Long = PDE_BASE; + WslEntry->u1.e1.Valid = 1; + WslEntry->u1.e1.LockedInWs = 1; + WslEntry->u1.e1.Direct = 1; + + PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress); + Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); + + Pfn1->u1.WsIndex = (ULONG)CurrentProcess; + + // + // As this index is 0, don't set another zero into the WsIndex field. + // + + // don't put it in the list. MiInsertWsle(0, MmWorkingSetList); + + // + // Fill in page table page which maps hyper space. + // + + WslEntry += 1; + + WslEntry->u1.VirtualAddress = (PVOID)MiGetPteAddress (HYPER_SPACE); + WslEntry->u1.e1.Valid = 1; + WslEntry->u1.e1.LockedInWs = 1; + WslEntry->u1.e1.Direct = 1; + + PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress); + Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); + + ASSERT (Pfn1->u1.WsIndex == 0); + Pfn1->u1.WsIndex = 1; + + // MiInsertWsle(1, MmWorkingSetList); + + // + // Fill in page which contains the working set list. + // + + WslEntry += 1; + + WslEntry->u1.VirtualAddress = (PVOID)MmWorkingSetList; + WslEntry->u1.e1.Valid = 1; + WslEntry->u1.e1.LockedInWs = 1; + WslEntry->u1.e1.Direct = 1; + + PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress); + Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); + + ASSERT (Pfn1->u1.WsIndex == 0); + Pfn1->u1.WsIndex = 2; + + // MiInsertWsle(2, MmWorkingSetList); + + CurrentEntry = 3; + + // + // Check to see if more pages are required in the working set list + // to map the current maximum working set size. + // + + NumberOfEntriesMapped = ((PMMWSLE)((ULONG)WORKING_SET_LIST + PAGE_SIZE)) - + MmWsle; + + if (CurrentProcess->Vm.MaximumWorkingSetSize >= NumberOfEntriesMapped) { + + PointerPte = MiGetPteAddress (&MmWsle[0]); + + CurrentVa = (ULONG)MmWorkingSetList + PAGE_SIZE; + + // + // The working set requires more than a single page. + // + + LOCK_PFN (OldIrql); + + do { + + MiEnsureAvailablePageOrWait (NULL, NULL); + + PointerPte += 1; + WorkingSetPage = MiRemoveZeroPage ( + MI_PAGE_COLOR_PTE_PROCESS (PointerPte, + &CurrentProcess->NextPageColor)); + PointerPte->u.Long = MM_DEMAND_ZERO_WRITE_PTE; + + MiInitializePfn (WorkingSetPage, PointerPte, 1); + + MI_MAKE_VALID_PTE (TempPte, + WorkingSetPage, + MM_READWRITE, + PointerPte ); + + MI_SET_PTE_DIRTY (TempPte); + *PointerPte = TempPte; + + WslEntry += 1; + + WslEntry->u1.Long = CurrentVa; + WslEntry->u1.e1.Valid = 1; + WslEntry->u1.e1.LockedInWs = 1; + WslEntry->u1.e1.Direct = 1; + + Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); + + ASSERT (Pfn1->u1.WsIndex == 0); + Pfn1->u1.WsIndex = CurrentEntry; + + // MiInsertWsle(CurrentEntry, MmWorkingSetList); + + CurrentEntry += 1; + CurrentVa += PAGE_SIZE; + + NumberOfEntriesMapped += PAGE_SIZE / sizeof(MMWSLE); + + } while (CurrentProcess->Vm.MaximumWorkingSetSize >= NumberOfEntriesMapped); + + UNLOCK_PFN (OldIrql); + } + + CurrentProcess->Vm.WorkingSetSize = CurrentEntry; + MmWorkingSetList->FirstFree = CurrentEntry; + MmWorkingSetList->FirstDynamic = CurrentEntry; + MmWorkingSetList->NextSlot = CurrentEntry; + + // + // Initialize the following slots as free. + // + + i = CurrentEntry + 1; + do { + + // + // Build the free list, note that the first working + // set entries (CurrentEntry) are not on the free list. + // These entries are reserved for the pages which + // map the working set and the page which contains the PDE. + // + + WslEntry += 1; + WslEntry->u1.Long = i << MM_FREE_WSLE_SHIFT; + i++; + } while (i <= NumberOfEntriesMapped); + + WslEntry->u1.Long = WSLE_NULL_INDEX << MM_FREE_WSLE_SHIFT; // End of list. + + MmWorkingSetList->LastInitializedWsle = + NumberOfEntriesMapped - 1; + + if (CurrentProcess->Vm.MaximumWorkingSetSize > ((1536*1024) >> PAGE_SHIFT)) { + + // + // The working set list consists of more than a single page. + // + + MiGrowWsleHash (&CurrentProcess->Vm, FALSE); + } + + return; +} + +NTSTATUS +MmAdjustWorkingSetSize ( + IN ULONG WorkingSetMinimum, + IN ULONG WorkingSetMaximum, + IN ULONG SystemCache + ) + +/*++ + +Routine Description: + + This routine adjusts the current size of a process's working set + list. If the maximum value is above the current maximum, pages + are removed from the working set list. + + An exception is raised if the limit cannot be granted. This + could occur if too many pages were locked in the process's + working set. + + Note: if the minimum and maximum are both 0xffffffff, the working set + is purged, but the default sizes are not changed. + +Arguments: + + WorkingSetMinimum - Supplies the new minimum working set size in bytes. + + WorkingSetMaximum - Supplies the new maximum working set size in bytes. + +Return Value: + + None. + +Environment: + + Kernel mode, IRQL 0 or APC_LEVEL. + +--*/ + + +{ + PEPROCESS CurrentProcess; + ULONG Entry; + ULONG SwapEntry; + ULONG CurrentEntry; + ULONG LastFreed; + PMMWSLE WslEntry; + PMMWSLE Wsle; + KIRQL OldIrql; + KIRQL OldIrql2; + LONG i; + PMMPTE PointerPte; + PMMPTE Va; + ULONG NumberOfEntriesMapped; + NTSTATUS ReturnStatus; + PMMPFN Pfn1; + LONG PagesAbove; + LONG NewPagesAbove; + ULONG FreeTryCount = 0; + PMMSUPPORT WsInfo; + IN PMMWSL WorkingSetList; + + // + // Get the working set lock and disable APCs. + // + + if (SystemCache) { + WsInfo = &MmSystemCacheWs; + } else { + CurrentProcess = PsGetCurrentProcess (); + WsInfo = &CurrentProcess->Vm; + } + + if (WorkingSetMinimum == 0) { + WorkingSetMinimum = WsInfo->MinimumWorkingSetSize; + } + + if (WorkingSetMaximum == 0) { + WorkingSetMaximum = WsInfo->MaximumWorkingSetSize; + } + + if ((WorkingSetMinimum == 0xFFFFFFFF) && + (WorkingSetMaximum == 0xFFFFFFFF)) { + return MiEmptyWorkingSet (WsInfo); + } + + WorkingSetMinimum = WorkingSetMinimum >> PAGE_SHIFT; + WorkingSetMaximum = WorkingSetMaximum >> PAGE_SHIFT; + + if (WorkingSetMinimum > WorkingSetMaximum) { + return STATUS_BAD_WORKING_SET_LIMIT; + } + + MmLockPagableSectionByHandle(ExPageLockHandle); + + ReturnStatus = STATUS_SUCCESS; + + if (SystemCache) { + LOCK_SYSTEM_WS (OldIrql2); + } else { + LOCK_WS (CurrentProcess); + } + + if (WorkingSetMaximum > MmMaximumWorkingSetSize) { + WorkingSetMaximum = MmMaximumWorkingSetSize; + ReturnStatus = STATUS_WORKING_SET_LIMIT_RANGE; + } + + if (WorkingSetMinimum > MmMaximumWorkingSetSize) { + WorkingSetMinimum = MmMaximumWorkingSetSize; + ReturnStatus = STATUS_WORKING_SET_LIMIT_RANGE; + } + + if (WorkingSetMinimum < MmMinimumWorkingSetSize) { + WorkingSetMinimum = MmMinimumWorkingSetSize; + ReturnStatus = STATUS_WORKING_SET_LIMIT_RANGE; + } + + // + // Make sure that the number of locked pages will not + // make the working set not fluid. + // + + if ((WsInfo->VmWorkingSetList->FirstDynamic + MM_FLUID_WORKING_SET) >= + WorkingSetMaximum) { + ReturnStatus = STATUS_BAD_WORKING_SET_LIMIT; + goto Returns; + } + + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; + + // + // Check to make sure ample resident phyiscal pages exist for + // this operation. + // + + LOCK_PFN (OldIrql); + + i = WorkingSetMinimum - WsInfo->MinimumWorkingSetSize; + + if (i > 0) { + + // + // New minimum working set is greater than the old one. + // + + if ((MmResidentAvailablePages < i) || + (MmAvailablePages < (20 + (i / (PAGE_SIZE / sizeof (MMWSLE)))))) { + UNLOCK_PFN (OldIrql); + ReturnStatus = STATUS_INSUFFICIENT_RESOURCES; + goto Returns; + } + } + + // + // Adjust the number of resident pages up or down dependent on + // the size of the new minimum working set size verus the previous + // minimum size. + // + + MmResidentAvailablePages -= i; + + UNLOCK_PFN (OldIrql); + + if (WsInfo->AllowWorkingSetAdjustment == FALSE) { + MmAllowWorkingSetExpansion (); + } + + if (WorkingSetMaximum > WorkingSetList->LastInitializedWsle) { + + do { + + // + // The maximum size of the working set is being increased, check + // to ensure the proper number of pages are mapped to cover + // the complete working set list. + // + + if (!MiAddWorkingSetPage (WsInfo)) { + WorkingSetMaximum = WorkingSetList->LastInitializedWsle - 1; + break; + } + } while (WorkingSetMaximum > WorkingSetList->LastInitializedWsle); + + } else { + + // + // The new working set maximum is less than the current working set + // maximum. + // + + if (WsInfo->WorkingSetSize > WorkingSetMaximum) { + + // + // Remove some pages from the working set. + // + + // + // Make sure that the number of locked pages will not + // make the working set not fluid. + // + + if ((WorkingSetList->FirstDynamic + MM_FLUID_WORKING_SET) >= + WorkingSetMaximum) { + + ReturnStatus = STATUS_BAD_WORKING_SET_LIMIT; + goto Returns; + } + + // + // Attempt to remove the pages from the Maximum downward. + // + + LastFreed = WorkingSetList->LastEntry; + if (WorkingSetList->LastEntry > WorkingSetMaximum) { + + while (LastFreed >= WorkingSetMaximum) { + + PointerPte = MiGetPteAddress( + Wsle[LastFreed].u1.VirtualAddress); + + if ((Wsle[LastFreed].u1.e1.Valid != 0) && + (!MiFreeWsle (LastFreed, + WsInfo, + PointerPte))) { + + // + // This LastFreed could not be removed. + // + + break; + } + LastFreed -= 1; + } + WorkingSetList->LastEntry = LastFreed; + if (WorkingSetList->NextSlot >= LastFreed) { + WorkingSetList->NextSlot = WorkingSetList->FirstDynamic; + } + } + + // + // Remove pages. + // + + Entry = WorkingSetList->FirstDynamic; + + while (WsInfo->WorkingSetSize > WorkingSetMaximum) { + if (Wsle[Entry].u1.e1.Valid != 0) { + PointerPte = MiGetPteAddress ( + Wsle[Entry].u1.VirtualAddress); + MiFreeWsle (Entry, WsInfo, PointerPte); + } + Entry += 1; + if (Entry > LastFreed) { + FreeTryCount += 1; + if (FreeTryCount > MM_RETRY_COUNT) { + + // + // Page table pages are not becoming free, give up + // and return an error. + // + + ReturnStatus = STATUS_BAD_WORKING_SET_LIMIT; + + break; + } + Entry = WorkingSetList->FirstDynamic; + } + } + + if (FreeTryCount <= MM_RETRY_COUNT) { + WorkingSetList->Quota = WorkingSetMaximum; + } + } + } + + // + // Adjust the number of pages above the working set minimum. + // + + PagesAbove = (LONG)WsInfo->WorkingSetSize - + (LONG)WsInfo->MinimumWorkingSetSize; + NewPagesAbove = (LONG)WsInfo->WorkingSetSize - + (LONG)WorkingSetMinimum; + + LOCK_PFN (OldIrql); + if (PagesAbove > 0) { + MmPagesAboveWsMinimum -= (ULONG)PagesAbove; + } + if (NewPagesAbove > 0) { + MmPagesAboveWsMinimum += (ULONG)NewPagesAbove; + } + UNLOCK_PFN (OldIrql); + + if (FreeTryCount <= MM_RETRY_COUNT) { + WsInfo->MaximumWorkingSetSize = WorkingSetMaximum; + WsInfo->MinimumWorkingSetSize = WorkingSetMinimum; + + if (WorkingSetMinimum >= WorkingSetList->Quota) { + WorkingSetList->Quota = WorkingSetMinimum; + } + } + + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); + + if ((WorkingSetList->HashTable == NULL) && + (WsInfo->MaximumWorkingSetSize > ((1536*1024) >> PAGE_SHIFT))) { + + // + // The working set list consists of more than a single page. + // + + MiGrowWsleHash (WsInfo, FALSE); + } + +Returns: + + if (SystemCache) { + UNLOCK_SYSTEM_WS (OldIrql2); + } else { + UNLOCK_WS (CurrentProcess); + } + + MmUnlockPagableImageSection(ExPageLockHandle); + + return ReturnStatus; +} + +ULONG +MiAddWorkingSetPage ( + IN PMMSUPPORT WsInfo + ) + +/*++ + +Routine Description: + + This function grows the working set list above working set + maximum during working set adjustment. At most one page + can be added at a time. + +Arguments: + + None. + +Return Value: + + Returns FALSE if no working set page could be added. + +Environment: + + Kernel mode, APC's disabled, working set mutexes held. + +--*/ + +{ + ULONG SwapEntry; + ULONG CurrentEntry; + PMMWSLE WslEntry; + ULONG i; + PMMPTE PointerPte; + PMMPTE Va; + MMPTE TempPte; + ULONG NumberOfEntriesMapped; + ULONG WorkingSetPage; + ULONG WorkingSetIndex; + PMMWSL WorkingSetList; + PMMWSLE Wsle; + PMMPFN Pfn1; + KIRQL OldIrql; + + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; + +#if DBG + if (WsInfo == &MmSystemCacheWs) { + MM_SYSTEM_WS_LOCK_ASSERT(); + } +#endif //DBG + + // + // The maximum size of the working set is being increased, check + // to ensure the proper number of pages are mapped to cover + // the complete working set list. + // + + PointerPte = MiGetPteAddress (&Wsle[WorkingSetList->LastInitializedWsle]); + + ASSERT (PointerPte->u.Hard.Valid == 1); + PointerPte += 1; + ASSERT (PointerPte->u.Hard.Valid == 0); + + Va = (PMMPTE)MiGetVirtualAddressMappedByPte (PointerPte); + + NumberOfEntriesMapped = ((PMMWSLE)((ULONG)Va + PAGE_SIZE)) - Wsle; + + // + // Map in a new working set page. + // + + LOCK_PFN (OldIrql); + if (MmAvailablePages < 20) { + + // + // No pages are available, set the quota to the last + // initialized WSLE and return. + + WorkingSetList->Quota = WorkingSetList->LastInitializedWsle; + UNLOCK_PFN (OldIrql); + return FALSE; + } + + WorkingSetPage = MiRemoveZeroPage (MI_GET_PAGE_COLOR_FROM_PTE (PointerPte)); + PointerPte->u.Long = MM_DEMAND_ZERO_WRITE_PTE; + MiInitializePfn (WorkingSetPage, PointerPte, 1); + UNLOCK_PFN (OldIrql); + + MI_MAKE_VALID_PTE (TempPte, + WorkingSetPage, + MM_READWRITE, + PointerPte ); + + MI_SET_PTE_DIRTY (TempPte); + *PointerPte = TempPte; + + CurrentEntry = WorkingSetList->LastInitializedWsle + 1; + + ASSERT (NumberOfEntriesMapped > CurrentEntry); + + WslEntry = &Wsle[CurrentEntry - 1]; + + for (i = CurrentEntry; i < NumberOfEntriesMapped; i++) { + + // + // Build the free list, note that the first working + // set entries (CurrentEntry) are not on the free list. + // These entries are reserved for the pages which + // map the working set and the page which contains the PDE. + // + + WslEntry += 1; + WslEntry->u1.Long = (i + 1) << MM_FREE_WSLE_SHIFT; + } + + WslEntry->u1.Long = WorkingSetList->FirstFree << MM_FREE_WSLE_SHIFT; + + WorkingSetList->FirstFree = CurrentEntry; + + WorkingSetList->LastInitializedWsle = + (NumberOfEntriesMapped - 1); + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); + + // + // As we are growing the working set, make sure the quota is + // above the working set size by adding 1 to the quota. + // + + WorkingSetList->Quota += 1; + + Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); + Pfn1->u1.WsIndex = (ULONG)PsGetCurrentThread(); + + // + // Get a working set entry. + // + + WsInfo->WorkingSetSize += 1; + ASSERT (WorkingSetList->FirstFree != WSLE_NULL_INDEX); + WorkingSetIndex = WorkingSetList->FirstFree; + WorkingSetList->FirstFree = Wsle[WorkingSetIndex].u1.Long >> MM_FREE_WSLE_SHIFT; + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); + + if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) { + MmPagesAboveWsMinimum += 1; + } + if (WorkingSetIndex > WorkingSetList->LastEntry) { + WorkingSetList->LastEntry = WorkingSetIndex; + } + + MiUpdateWsle ( &WorkingSetIndex, Va, WorkingSetList, Pfn1); + + // + // Lock any created page table pages into the working set. + // + + if (WorkingSetIndex >= WorkingSetList->FirstDynamic) { + + SwapEntry = WorkingSetList->FirstDynamic; + + if (WorkingSetIndex != WorkingSetList->FirstDynamic) { + + // + // Swap this entry with the one at first dynamic. + // + + MiSwapWslEntries (WorkingSetIndex, SwapEntry, WsInfo); + } + + WorkingSetList->FirstDynamic += 1; + WorkingSetList->NextSlot = WorkingSetList->FirstDynamic; + + Wsle[SwapEntry].u1.e1.LockedInWs = 1; + ASSERT (Wsle[SwapEntry].u1.e1.Valid == 1); + } + + ASSERT ((MiGetPteAddress(&Wsle[WorkingSetList->LastInitializedWsle]))->u.Hard.Valid == 1); + + if ((WorkingSetList->HashTable == NULL) && + (MmAvailablePages > 20)) { + + // + // Add a hash table to support shared pages in the working set to + // eliminate costly lookups. + // + + LOCK_EXPANSION_IF_ALPHA (OldIrql); + ASSERT (WsInfo->AllowWorkingSetAdjustment != FALSE); + WsInfo->AllowWorkingSetAdjustment = MM_GROW_WSLE_HASH; + UNLOCK_EXPANSION_IF_ALPHA (OldIrql); + } + + return TRUE; +} +VOID +MiGrowWsleHash ( + IN PMMSUPPORT WsInfo, + IN ULONG PfnLockHeld + ) + +/*++ + +Routine Description: + + This function grows (or adds) a hash table to the working set list + to allow direct indexing for WSLEs than cannot be located via the + PFN database WSINDEX field. + + The hash table is located AFTER the WSLE array and the pages are + locked into the working set just like standard WSLEs. + + Note, that the hash table is expanded by setting the hash table + field in the working set to NULL, but leaving the size as non-zero. + This indicates that the hash should be expanded and the initial + portion of the table zeroed. + +Arguments: + + WsInfo - Supples a pointer to the working set info block for the + process (or system cache). + + PfnLockHeld - Supplies TRUE if the PFN lock is already held. + +Return Value: + + None. + +Environment: + + Kernel mode, APC's disabled, working set lock held. + +--*/ +{ + LONG Size; + PMMWSLE Wsle; + PMMPFN Pfn1; + PMMPTE PointerPte; + MMPTE TempPte; + ULONG First; + PVOID Va; + ULONG SwapEntry; + ULONG WorkingSetPage; + ULONG Hash; + ULONG HashValue; + ULONG NewSize; + ULONG WorkingSetIndex; + PMMWSLE_HASH Table; + ULONG j; + PMMWSL WorkingSetList; + KIRQL OldIrql; + ULONG Count; + + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; + + Table = WorkingSetList->HashTable; + if (Table == NULL) { + NewSize = (ULONG)PAGE_ALIGN (((1 + WorkingSetList->NonDirectCount) * + 2 * sizeof(MMWSLE_HASH)) + PAGE_SIZE - 1); + + Table = (PMMWSLE_HASH) + ((PCHAR)PAGE_ALIGN (&Wsle[MM_MAXIMUM_WORKING_SET]) + PAGE_SIZE); + First = WorkingSetList->HashTableSize; + ASSERT (MiGetPteAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0); + WorkingSetList->HashTableSize = 0; + + j = First * sizeof(MMWSLE_HASH); + if (j > NewSize) { + NewSize = j; + } + + } else { + + // + // Attempt to add 4 pages, make sure the working set list has + // 4 free entries. + // + + ASSERT (MiGetPteAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0); + if ((WorkingSetList->LastInitializedWsle + 5) > WsInfo->WorkingSetSize) { + NewSize = PAGE_SIZE * 4; + } else { + NewSize = PAGE_SIZE; + } + First = WorkingSetList->HashTableSize; + } + + Size = NewSize; + + PointerPte = MiGetPteAddress (&Table[WorkingSetList->HashTableSize]); + + do { + + if (PointerPte->u.Hard.Valid == 0) { + + LOCK_PFN (OldIrql); + WorkingSetPage = MiRemoveZeroPage ( + MI_GET_PAGE_COLOR_FROM_PTE (PointerPte)); + + PointerPte->u.Long = MM_DEMAND_ZERO_WRITE_PTE; + MiInitializePfn (WorkingSetPage, PointerPte, 1); + + MI_MAKE_VALID_PTE (TempPte, + WorkingSetPage, + MM_READWRITE, + PointerPte ); + + MI_SET_PTE_DIRTY (TempPte); + *PointerPte = TempPte; + + UNLOCK_PFN (OldIrql); + + // + // As we are growing the working set, we know that quota + // is above the current working set size. Just take the + // next free WSLE from the list and use it. + // + + Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); + Pfn1->u1.WsIndex = (ULONG)PsGetCurrentThread(); + + Va = (PMMPTE)MiGetVirtualAddressMappedByPte (PointerPte); + + WorkingSetIndex = MiLocateAndReserveWsle (WsInfo); + MiUpdateWsle (&WorkingSetIndex , Va, WorkingSetList, Pfn1); + + // + // Lock any created page table pages into the working set. + // + + if (WorkingSetIndex >= WorkingSetList->FirstDynamic) { + + SwapEntry = WorkingSetList->FirstDynamic; + + if (WorkingSetIndex != WorkingSetList->FirstDynamic) { + + // + // Swap this entry with the one at first dynamic. + // + + MiSwapWslEntries (WorkingSetIndex, SwapEntry, WsInfo); + } + + WorkingSetList->FirstDynamic += 1; + WorkingSetList->NextSlot = WorkingSetList->FirstDynamic; + + Wsle[SwapEntry].u1.e1.LockedInWs = 1; + ASSERT (Wsle[SwapEntry].u1.e1.Valid == 1); + } + } + PointerPte += 1; + Size -= PAGE_SIZE; + } while (Size > 0); + + ASSERT (PointerPte->u.Hard.Valid == 0); + + WorkingSetList->HashTableSize += NewSize / sizeof (MMWSLE_HASH); + WorkingSetList->HashTable = Table; + ASSERT (MiGetPteAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0); + + if (First != 0) { + RtlZeroMemory (Table, First * sizeof(MMWSLE_HASH)); + } + + // + // Fill hash table + // + + j = 0; + Count = WorkingSetList->NonDirectCount; + + Size = WorkingSetList->HashTableSize; + HashValue = Size - 1; + + do { + if ((Wsle[j].u1.e1.Valid == 1) && + (Wsle[j].u1.e1.Direct == 0)) { + + // + // Hash this. + // + + Count -= 1; + + Hash = (Wsle[j].u1.Long >> (PAGE_SHIFT - 2)) % HashValue; + + while (Table[Hash].Key != 0) { + Hash += 1; + if (Hash >= (ULONG)Size) { + Hash = 0; + } + } + + Table[Hash].Key = Wsle[j].u1.Long & ~(PAGE_SIZE - 1); + Table[Hash].Index = j; +#if DBG + { + PMMPTE PointerPte; + PMMPFN Pfn; + + PointerPte = MiGetPteAddress(Wsle[j].u1.VirtualAddress); + ASSERT (PointerPte->u.Hard.Valid); + Pfn = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); + } +#endif //DBG + + } + ASSERT (j <= WorkingSetList->LastEntry); + j += 1; + } while (Count); + +#if DBG + MiCheckWsleHash (WorkingSetList); +#endif //DBG + return; +} + + +ULONG +MiTrimWorkingSet ( + ULONG Reduction, + IN PMMSUPPORT WsInfo, + IN ULONG ForcedReduction + ) + +/*++ + +Routine Description: + + This function reduces the working set by the specified amount. + +Arguments: + + Reduction - Supplies the number of pages to remove from the working + set. + + WsInfo - Supplies a pointer to the working set information for the + process (or system cache) to trim. + + ForcedReduction - Set TRUE if the reduction is being done to free up + pages in which case we should try to reduce + working set pages as well. Set to FALSE when the + reduction is trying to increase the fault rates + in which case the policy should be more like + locate and reserve. + +Return Value: + + Returns the actual number of pages removed. + +Environment: + + Kernel mode, APC's disabled, working set lock. Pfn lock NOT held. + +--*/ + +{ + ULONG TryToFree; + ULONG LastEntry; + PMMWSL WorkingSetList; + PMMWSLE Wsle; + PMMPTE PointerPte; + ULONG NumberLeftToRemove; + ULONG LoopCount; + ULONG EndCount; + + NumberLeftToRemove = Reduction; + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; + +#if DBG + if (WsInfo == &MmSystemCacheWs) { + MM_SYSTEM_WS_LOCK_ASSERT(); + } +#endif //DBG + + TryToFree = WorkingSetList->NextSlot; + LastEntry = WorkingSetList->LastEntry; + LoopCount = 0; + + if (ForcedReduction) { + EndCount = 5; + } else { + EndCount = 2; + } + + while ((NumberLeftToRemove != 0) && (LoopCount != EndCount)) { + while ((NumberLeftToRemove != 0) && (TryToFree <= LastEntry)) { + + if (Wsle[TryToFree].u1.e1.Valid == 1) { + PointerPte = MiGetPteAddress (Wsle[TryToFree].u1.VirtualAddress); + if (MI_GET_ACCESSED_IN_PTE (PointerPte)) { + + // + // If accessed bit is set, clear it. If accessed + // bit is clear, remove from working set. + // + + MI_SET_ACCESSED_IN_PTE (PointerPte, 0); + } else { + if (MiFreeWsle (TryToFree, WsInfo, PointerPte)) { + NumberLeftToRemove -= 1; + } + } + } + TryToFree += 1; + } + TryToFree = WorkingSetList->FirstDynamic; + LoopCount += 1; + } + WorkingSetList->NextSlot = TryToFree; + + // + // If this is not the system cache working set, see if the working + // set list can be contracted. + // + + if (WsInfo != &MmSystemCacheWs) { + + // + // Make sure we are at least a page above the working set maximum. + // + + if (WorkingSetList->FirstDynamic == WsInfo->WorkingSetSize) { + MiRemoveWorkingSetPages (WorkingSetList, WsInfo); + } else { + + if ((WorkingSetList->Quota + 15 + (PAGE_SIZE / sizeof(MMWSLE))) < + WorkingSetList->LastEntry) { + if ((WsInfo->MaximumWorkingSetSize + 15 + (PAGE_SIZE / sizeof(MMWSLE))) < + WorkingSetList->LastEntry ) { + MiRemoveWorkingSetPages (WorkingSetList, WsInfo); + } + } + } + } + return (Reduction - NumberLeftToRemove); +} + +#if 0 //COMMENTED OUT. +VOID +MmPurgeWorkingSet ( + IN PEPROCESS Process, + IN PVOID BaseAddress, + IN ULONG RegionSize + ) + +/*++ + +Routine Description: + + This function removes any valid pages with a reference count + of 1 within the specified address range of the specified process. + + If the address range is within the system cache, the process + paramater is ignored. + +Arguments: + + Process - Supplies a pointer to the process to operate upon. + + BaseAddress - Supplies the base address of the range to operate upon. + + RegionSize - Supplies the size of the region to operate upon. + +Return Value: + + None. + +Environment: + + Kernel mode, APC_LEVEL or below. + +--*/ + +{ + PMMSUPPORT WsInfo; + PMMPTE PointerPte; + PMMPTE PointerPde; + PMMPTE LastPte; + PMMPFN Pfn1; + MMPTE PteContents; + PEPROCESS CurrentProcess; + PVOID EndingAddress; + ULONG SystemCache; + KIRQL OldIrql; + + // + // Determine if the specified base address is within the system + // cache and if so, don't attach, the working set lock is still + // required to "lock" paged pool pages (proto PTEs) into the + // working set. + // + + CurrentProcess = PsGetCurrentProcess (); + + ASSERT (RegionSize != 0); + + EndingAddress = (PVOID)((ULONG)BaseAddress + RegionSize - 1); + + if ((BaseAddress <= MM_HIGHEST_USER_ADDRESS) || + ((BaseAddress >= (PVOID)PTE_BASE) && + (BaseAddress < (PVOID)MM_SYSTEM_SPACE_START)) || + ((BaseAddress >= MM_PAGED_POOL_START) && + (BaseAddress <= MmPagedPoolEnd))) { + + SystemCache = FALSE; + + // + // Attach to the specified process. + // + + KeAttachProcess (&Process->Pcb); + + WsInfo = &Process->Vm, + + LOCK_WS (Process); + } else { + + SystemCache = TRUE; + Process = CurrentProcess; + WsInfo = &MmSystemCacheWs; + } + + PointerPde = MiGetPdeAddress (BaseAddress); + PointerPte = MiGetPteAddress (BaseAddress); + LastPte = MiGetPteAddress (EndingAddress); + + while (!MiDoesPdeExistAndMakeValid(PointerPde, Process, FALSE)) { + + // + // No page table page exists for this address. + // + + PointerPde += 1; + + PointerPte = MiGetVirtualAddressMappedByPte (PointerPde); + + if (PointerPte > LastPte) { + break; + } + } + + LOCK_PFN (OldIrql); + + while (PointerPte <= LastPte) { + + PteContents = *PointerPte; + + if (PteContents.u.Hard.Valid == 1) { + + // + // Remove this page from the working set. + // + + Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber); + + if (Pfn1->u3.e2.ReferenceCount == 1) { + MiRemovePageFromWorkingSet (PointerPte, Pfn1, WsInfo); + } + } + + PointerPte += 1; + + if (((ULONG)PointerPte & (PAGE_SIZE - 1)) == 0) { + + PointerPde = MiGetPteAddress (PointerPte); + + while ((PointerPte <= LastPte) && + (!MiDoesPdeExistAndMakeValid(PointerPde, Process, TRUE))) { + + // + // No page table page exists for this address. + // + + PointerPde += 1; + + PointerPte = MiGetVirtualAddressMappedByPte (PointerPde); + } + } + } + + UNLOCK_PFN (OldIrql); + + if (!SystemCache) { + + UNLOCK_WS (Process); + KeDetachProcess(); + } + return; +} +#endif //0 + +VOID +MiEliminateWorkingSetEntry ( + IN ULONG WorkingSetIndex, + IN PMMPTE PointerPte, + IN PMMPFN Pfn, + IN PMMWSLE Wsle + ) + +/*++ + +Routine Description: + + This routine removes the specified working set list entry + form the working set, flushes the TB for the page, decrements + the share count for the physical page, and, if necessary turns + the PTE into a transition PTE. + +Arguments: + + WorkingSetIndex - Supplies the working set index to remove. + + PointerPte - Supplies a pointer to the PTE corresonding to the virtual + address in the working set. + + Pfn - Supplies a pointer to the PFN element corresponding to the PTE. + + Wsle - Supplies a pointer to the first working set list entry for this + working set. + +Return Value: + + None. + +Environment: + + Kernel mode, Working set lock and PFN lock held, APC's disabled. + +--*/ + +{ + PMMPTE ContainingPageTablePage; + MMPTE TempPte; + MMPTE PreviousPte; + ULONG PageFrameIndex; + KIRQL OldIrql; + + // + // Remove the page from the working set. + // + + MM_PFN_LOCK_ASSERT (); + + TempPte = *PointerPte; + PageFrameIndex = TempPte.u.Hard.PageFrameNumber; + +#ifdef _X86_ +#if DBG +#if !defined(NT_UP) + if (TempPte.u.Hard.Writable == 1) { + ASSERT (TempPte.u.Hard.Dirty == 1); + } + ASSERT (TempPte.u.Hard.Accessed == 1); +#endif //NTUP +#endif //DBG +#endif //X86 + + MI_MAKING_VALID_PTE_INVALID (FALSE); + + if (Pfn->u3.e1.PrototypePte) { + + // + // This is a prototype PTE. The PFN database does not contain + // the contents of this PTE it contains the contents of the + // prototype PTE. This PTE must be reconstructed to contain + // a pointer to the prototype PTE. + // + // The working set list entry contains information about + // how to reconstruct the PTE. + // + + if (Wsle[WorkingSetIndex].u1.e1.SameProtectAsProto == 0) { + + // + // The protection for the prototype PTE is in the + // WSLE. + // + + ASSERT (Wsle[WorkingSetIndex].u1.e1.Protection != 0); + TempPte.u.Long = 0; + TempPte.u.Soft.Protection = + Wsle[WorkingSetIndex].u1.e1.Protection; + TempPte.u.Soft.PageFileHigh = 0xFFFFF; + + } else { + + // + // The protection is in the prototype PTE. + // + + TempPte.u.Long = MiProtoAddressForPte (Pfn->PteAddress); + MI_SET_GLOBAL_BIT_IF_SYSTEM (TempPte, PointerPte); + } + + TempPte.u.Proto.Prototype = 1; + + // + // Decrement the share count of the containing page table + // page as the PTE for the removed page is no longer valid + // or in transition + // + + ContainingPageTablePage = MiGetPteAddress (PointerPte); + if (ContainingPageTablePage->u.Hard.Valid == 0) { + MiCheckPdeForPagedPool (PointerPte); + } + MiDecrementShareAndValidCount (ContainingPageTablePage->u.Hard.PageFrameNumber); + + } else { + + // + // This is a private page, make it transition. + // + + // + // Assert that the share count is 1 for all user mode pages. + // + + ASSERT ((Pfn->u2.ShareCount == 1) || + (Wsle[WorkingSetIndex].u1.VirtualAddress > + (PVOID)MM_HIGHEST_USER_ADDRESS)); + + // + // Set the working set index to zero. This allows page table + // pages to be brough back in with the proper WSINDEX. + // + + ASSERT (Pfn->u1.WsIndex != 0); + Pfn->u1.WsIndex = 0; + MI_MAKE_VALID_PTE_TRANSITION (TempPte, + Pfn->OriginalPte.u.Soft.Protection); + + } + + PreviousPte.u.Flush = KeFlushSingleTb (Wsle[WorkingSetIndex].u1.VirtualAddress, + TRUE, + (BOOLEAN)(Wsle == MmSystemCacheWsle), + (PHARDWARE_PTE)PointerPte, + TempPte.u.Flush); + + ASSERT (PreviousPte.u.Hard.Valid == 1); + + // + // A page is being removed from the working set, on certain + // hardware the dirty bit should be ORed into the modify bit in + // the PFN element. + // + + MI_CAPTURE_DIRTY_BIT_TO_PFN (&PreviousPte, Pfn); + + // + // Flush the translation buffer and decrement the number of valid + // PTEs within the containing page table page. Note that for a + // private page, the page table page is still needed because the + // page is in transiton. + // + + MiDecrementShareCount (PageFrameIndex); + + return; +} + +VOID +MiRemoveWorkingSetPages ( + IN PMMWSL WorkingSetList, + IN PMMSUPPORT WsInfo + ) + +/*++ + +Routine Description: + + This routine compresses the WSLEs into the front of the working set + and frees the pages for unneeded working set entries. + +Arguments: + + WorkingSetList - Supplies a pointer to the working set list to compress. + +Return Value: + + None. + +Environment: + + Kernel mode, Working set lock held, APC's disabled. + +--*/ + +{ + PMMWSLE FreeEntry; + PMMWSLE LastEntry; + PMMWSLE Wsle; + ULONG FreeIndex; + ULONG LastIndex; + ULONG LastInvalid; + PMMPTE PointerPte; + PMMPTE WsPte; + PMMPFN Pfn1; + PEPROCESS CurrentProcess; + MMPTE_FLUSH_LIST PteFlushList; + ULONG NewSize; + PMMWSLE_HASH Table; + KIRQL OldIrql; + + PteFlushList.Count = 0; + CurrentProcess = PsGetCurrentProcess(); + +#if DBG + MiCheckNullIndex (WorkingSetList); +#endif //DBG + + // + // Check to see if the wsle hash table should be contracted. + // + + if (WorkingSetList->HashTable) { + + Table = WorkingSetList->HashTable; + ASSERT (MiGetPteAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0); + + NewSize = (ULONG)PAGE_ALIGN ((WorkingSetList->NonDirectCount * 2 * + sizeof(MMWSLE_HASH)) + PAGE_SIZE - 1); + + NewSize = NewSize / sizeof(MMWSLE_HASH); + + if (WsInfo->WorkingSetSize < 200) { + NewSize = 0; + } + + if (NewSize < WorkingSetList->HashTableSize) { + + LOCK_EXPANSION_IF_ALPHA (OldIrql); + if (NewSize && WsInfo->AllowWorkingSetAdjustment) { + WsInfo->AllowWorkingSetAdjustment = MM_GROW_WSLE_HASH; + } + UNLOCK_EXPANSION_IF_ALPHA (OldIrql); + + // + // Remove pages from hash table. + // + + ASSERT (((ULONG)&WorkingSetList->HashTable[NewSize] & + (PAGE_SIZE - 1)) == 0); + + PointerPte = MiGetPteAddress (&WorkingSetList->HashTable[NewSize]); + + // + // Set the hash table to null indicating that no hashing + // is going on. + // + + WorkingSetList->HashTable = NULL; + WorkingSetList->HashTableSize = NewSize; + + LOCK_PFN (OldIrql); + while (PointerPte->u.Hard.Valid == 1) { + + MiDeletePte (PointerPte, + MiGetVirtualAddressMappedByPte (PointerPte), + FALSE, + CurrentProcess, + NULL, + &PteFlushList); + + PointerPte += 1; + + // + // Add back in the private page MiDeletePte subtracted. + // + + CurrentProcess->NumberOfPrivatePages += 1; + } + MiFlushPteList (&PteFlushList, FALSE, ZeroPte); + UNLOCK_PFN (OldIrql); + } + ASSERT (MiGetPteAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0); + } + + // + // If the only pages in the working set are locked pages (that + // is all pages are BEFORE first dynamic, just reorganize the + // free list.) + // + + Wsle = WorkingSetList->Wsle; + if (WorkingSetList->FirstDynamic == WsInfo->WorkingSetSize) { + + LastIndex = WorkingSetList->FirstDynamic; + LastEntry = &Wsle[LastIndex]; + + } else { + + // + // Start from the first dynamic and move towards the end looking + // for free entries. At the same time start from the end and + // move towards first dynamic looking for valid entries. + // + + LastInvalid = 0; + FreeIndex = WorkingSetList->FirstDynamic; + FreeEntry = &Wsle[FreeIndex]; + LastIndex = WorkingSetList->LastEntry; + LastEntry = &Wsle[LastIndex]; + + while (FreeEntry < LastEntry) { + if (FreeEntry->u1.e1.Valid == 1) { + FreeEntry += 1; + FreeIndex += 1; + } else if (LastEntry->u1.e1.Valid == 0) { + LastEntry -= 1; + LastIndex -= 1; + } else { + + // + // Move the WSLE at LastEntry to the free slot at FreeEntry. + // + + LastInvalid = 1; + *FreeEntry = *LastEntry; + if (LastEntry->u1.e1.Direct) { + + PointerPte = MiGetPteAddress (LastEntry->u1.VirtualAddress); + Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber); + Pfn1->u1.WsIndex = FreeIndex; + + } else { + + // + // This entry is in the working set tree. Remove it + // and then add the entry add the free slot. + // + + MiRemoveWsle (LastIndex, WorkingSetList); + MiInsertWsle (FreeIndex, WorkingSetList); + } + LastEntry->u1.Long = 0; + LastEntry -= 1; + LastIndex -= 1; + FreeEntry += 1; + FreeIndex += 1; + } + } + + // + // If no entries were freed, just return. + // + + if (LastInvalid == 0) { +#if DBG + MiCheckNullIndex (WorkingSetList); +#endif //DBG + return; + } + } + + // + // Reorganize the free list. Make last entry the first free. + // + + ASSERT ((LastEntry - 1)->u1.e1.Valid == 1); + + if (LastEntry->u1.e1.Valid == 1) { + LastEntry += 1; + LastIndex += 1; + } + + WorkingSetList->LastEntry = LastIndex - 1; + WorkingSetList->FirstFree = LastIndex; + + ASSERT ((LastEntry - 1)->u1.e1.Valid == 1); + ASSERT ((LastEntry)->u1.e1.Valid == 0); + + // + // Point free entry to the first invalid page. + // + + FreeEntry = LastEntry; + + while (LastIndex < WorkingSetList->LastInitializedWsle) { + + // + // Put the remainer of the WSLEs on the free list. + // + + ASSERT (LastEntry->u1.e1.Valid == 0); + LastIndex += 1; + LastEntry->u1.Long = LastIndex << MM_FREE_WSLE_SHIFT; + LastEntry += 1; + } + + //LastEntry->u1.Long = WSLE_NULL_INDEX << MM_FREE_WSLE_SHIFT; // End of list. + + // + // Delete the working set pages at the end. + // + + PointerPte = MiGetPteAddress (&Wsle[WorkingSetList->LastInitializedWsle]); + if (&Wsle[WsInfo->MinimumWorkingSetSize] > FreeEntry) { + FreeEntry = &Wsle[WsInfo->MinimumWorkingSetSize]; + } + + WsPte = MiGetPteAddress (FreeEntry); + + LOCK_PFN (OldIrql); + while (PointerPte > WsPte) { + ASSERT (PointerPte->u.Hard.Valid == 1); + + MiDeletePte (PointerPte, + MiGetVirtualAddressMappedByPte (PointerPte), + FALSE, + CurrentProcess, + NULL, + &PteFlushList); + + PointerPte -= 1; + + // + // Add back in the private page MiDeletePte subtracted. + // + + CurrentProcess->NumberOfPrivatePages += 1; + } + + MiFlushPteList (&PteFlushList, FALSE, ZeroPte); + + UNLOCK_PFN (OldIrql); + + // + // Mark the last pte in the list as free. + // + + LastEntry = (PMMWSLE)((ULONG)(PAGE_ALIGN(FreeEntry)) + PAGE_SIZE); + LastEntry -= 1; + + ASSERT (LastEntry->u1.e1.Valid == 0); + LastEntry->u1.Long = WSLE_NULL_INDEX << MM_FREE_WSLE_SHIFT; //End of List. + ASSERT (LastEntry > &Wsle[0]); + WorkingSetList->LastInitializedWsle = LastEntry - &Wsle[0]; + WorkingSetList->NextSlot = WorkingSetList->FirstDynamic; + + ASSERT (WorkingSetList->LastEntry <= WorkingSetList->LastInitializedWsle); + + if (WorkingSetList->Quota < WorkingSetList->LastInitializedWsle) { + WorkingSetList->Quota = WorkingSetList->LastInitializedWsle; + } + + ASSERT ((MiGetPteAddress(&Wsle[WorkingSetList->LastInitializedWsle]))->u.Hard.Valid == 1); + ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) || + (WorkingSetList->FirstFree == WSLE_NULL_INDEX)); +#if DBG + MiCheckNullIndex (WorkingSetList); +#endif //DBG + return; +} + + +NTSTATUS +MiEmptyWorkingSet ( + IN PMMSUPPORT WsInfo + ) + +/*++ + +Routine Description: + + This routine frees all pages from the working set. + +Arguments: + + None. + +Return Value: + + Status of operation. + +Environment: + + Kernel mode. No locks. + +--*/ + +{ + PEPROCESS Process; + KIRQL OldIrql; + KIRQL OldIrql2; + PMMPTE PointerPte; + ULONG Entry; + ULONG LastFreed; + PMMWSL WorkingSetList; + PMMWSLE Wsle; + ULONG Last = 0; + NTSTATUS Status; + + MmLockPagableSectionByHandle(ExPageLockHandle); + + if (WsInfo == &MmSystemCacheWs) { + LOCK_SYSTEM_WS (OldIrql); + } else { + Process = PsGetCurrentProcess (); + LOCK_WS (Process); + if (Process->AddressSpaceDeleted != 0) { + Status = STATUS_PROCESS_IS_TERMINATING; + goto Deleted; + } + } + + WorkingSetList = WsInfo->VmWorkingSetList; + Wsle = WorkingSetList->Wsle; + + // + // Attempt to remove the pages from the Maximum downward. + // + + Entry = WorkingSetList->FirstDynamic; + LastFreed = WorkingSetList->LastEntry; + while (Entry <= LastFreed) { + if (Wsle[Entry].u1.e1.Valid != 0) { + PointerPte = MiGetPteAddress (Wsle[Entry].u1.VirtualAddress); + MiFreeWsle (Entry, WsInfo, PointerPte); + } + Entry += 1; + } + + if (WsInfo != &MmSystemCacheWs) { + MiRemoveWorkingSetPages (WorkingSetList,WsInfo); + } + WorkingSetList->Quota = WsInfo->WorkingSetSize; + WorkingSetList->NextSlot = WorkingSetList->FirstDynamic; + + // + // Attempt to remove the pages from the front to the end. + // + + // + // Reorder the free list. + // + + Entry = WorkingSetList->FirstDynamic; + LastFreed = WorkingSetList->LastInitializedWsle; + while (Entry <= LastFreed) { + if (Wsle[Entry].u1.e1.Valid == 0) { + if (Last == 0) { + WorkingSetList->FirstFree = Entry; + } else { + Wsle[Last].u1.Long = Entry << MM_FREE_WSLE_SHIFT; + } + Last = Entry; + } + Entry += 1; + } + if (Last != 0) { + Wsle[Last].u1.Long = WSLE_NULL_INDEX << MM_FREE_WSLE_SHIFT; // End of list. + } + + Status = STATUS_SUCCESS; + +Deleted: + + if (WsInfo == &MmSystemCacheWs) { + UNLOCK_SYSTEM_WS (OldIrql); + } else { + UNLOCK_WS (Process); + } + MmUnlockPagableImageSection(ExPageLockHandle); + return Status; +} + +#if 0 + +#define x256k_pte_mask (((256*1024) >> (PAGE_SHIFT - PTE_SHIFT)) - (sizeof(MMPTE))) + +VOID +MiDumpWsleInCacheBlock ( + IN PMMPTE CachePte + ) + +/*++ + +Routine Description: + + The routine checks the prototypte PTEs adjacent to the supplied + PTE and if they are modified, in the system cache working set, + and have a reference count of 1, removes it from the system + cache working set. + +Arguments: + + CachePte - Supplies a pointer to the cache pte. + +Return Value: + + None. + +Environment: + + Kernel mode, Working set lock and PFN lock held, APC's disabled. + +--*/ + +{ + PMMPTE LoopPte; + PMMPTE PointerPte; + + LoopPte = (PMMPTE)((ULONG)CachePte & ~x256k_pte_mask); + PointerPte = CachePte - 1; + + while (PointerPte >= LoopPte ) { + + if (MiDumpPteInCacheBlock (PointerPte) == FALSE) { + break; + } + PointerPte -= 1; + } + + PointerPte = CachePte + 1; + LoopPte = (PMMPTE)((ULONG)CachePte | x256k_pte_mask); + + while (PointerPte <= LoopPte ) { + + if (MiDumpPteInCacheBlock (PointerPte) == FALSE) { + break; + } + PointerPte += 1; + } + return; +} + +ULONG +MiDumpPteInCacheBlock ( + IN PMMPTE PointerPte + ) + +{ + PMMPFN Pfn1; + MMPTE PteContents; + ULONG WorkingSetIndex; + + PteContents = *PointerPte; + + if (PteContents.u.Hard.Valid == 1) { + + Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber); + + // + // If the PTE is valid and dirty (or pfn indicates dirty) + // and the Wsle is direct index via the pfn wsindex element + // and the reference count is one, then remove this page from + // the cache manager's working set list. + // + + if ((Pfn1->u3.e2.ReferenceCount == 1) && + ((Pfn1->u3.e1.Modified == 1) || + (MI_IS_PTE_DIRTY (PteContents))) && + (MiGetPteAddress ( + MmSystemCacheWsle[Pfn1->u1.WsIndex].u1.VirtualAddress) == + PointerPte)) { + + // + // Found a candidate, remove the page from the working set. + // + + WorkingSetIndex = Pfn1->u1.WsIndex; + LOCK_PFN (OldIrql); + MiEliminateWorkingSetEntry (WorkingSetIndex, + PointerPte, + Pfn1, + MmSystemCacheWsle); + UNLOCK_PFN (OldIrql); + + // + // Remove the working set entry from the working set tree. + // + + MiRemoveWsle (WorkingSetIndex, MmSystemCacheWorkingSetList); + + // + // Put the entry on the free list and decrement the current + // size. + // + + MmSystemCacheWsle[WorkingSetIndex].u1.Long = + MmSystemCacheWorkingSetList->FirstFree << MM_FREE_WSLE_SHIFT; + MmSystemCacheWorkingSetList->FirstFree = WorkingSetIndex; + + if (MmSystemCacheWs.WorkingSetSize > MmSystemCacheWs.MinimumWorkingSetSize) { + MmPagesAboveWsMinimum -= 1; + } + MmSystemCacheWs.WorkingSetSize -= 1; + return TRUE; + } + } + return FALSE; +} +#endif //0 + +#if DBG +VOID +MiCheckNullIndex ( + IN PMMWSL WorkingSetList + ) + +{ + PMMWSLE Wsle; + ULONG j; + ULONG Nulls = 0; + + Wsle = WorkingSetList->Wsle; + for (j = 0;j <= WorkingSetList->LastInitializedWsle; j++) { + if ((Wsle[j].u1.Long >> MM_FREE_WSLE_SHIFT) == WSLE_NULL_INDEX ) { + Nulls += 1; + } + } + ASSERT (Nulls == 1); + return; +} + +#endif //DBG + + |