/*++ Copyright (c) 1989-1994 Microsoft Corporation Module Name: pool.c Abstract: Thie module implements the NT executive pool allocator. Author: Mark Lucovsky 16-feb-1989 Lou Perazzoli 31-Aug-1991 (change from binary buddy) David N. Cutler (davec) 27-May-1994 Environment: kernel mode only Revision History: --*/ #include "exp.h" #pragma hdrstop #undef ExAllocatePoolWithTag #undef ExAllocatePool #undef ExAllocatePoolWithQuota #undef ExAllocatePoolWithQuotaTag #undef ExFreePoolWithTag #if DBG BOOLEAN ExpEchoPoolCalls; #endif //DBG // // Define forward referenced funtion prototypes. // VOID ExpInitializePoolDescriptor( IN PPOOL_DESCRIPTOR PoolDescriptor, IN POOL_TYPE PoolType, IN ULONG PoolIndex, IN ULONG Threshold, IN PVOID PoolLock ); NTSTATUS ExpSnapShotPoolPages( IN PVOID Address, IN ULONG Size, IN OUT PSYSTEM_POOL_INFORMATION PoolInformation, IN OUT PSYSTEM_POOL_ENTRY *PoolEntryInfo, IN ULONG Length, IN OUT PULONG RequiredLength ); #ifdef ALLOC_PRAGMA #pragma alloc_text(INIT, InitializePool) #pragma alloc_text(INIT, ExpInitializePoolDescriptor) #if DBG #pragma alloc_text(PAGELK, ExSnapShotPool) #pragma alloc_text(PAGELK, ExpSnapShotPoolPages) #endif // DBG #endif ULONG FirstPrint; PPOOL_TRACKER_TABLE PoolTrackTable; PPOOL_TRACKER_BIG_PAGES PoolBigPageTable; ULONG PoolHitTag = 0xffffff0f; USHORT ExpInsertPoolTracker ( ULONG Key, ULONG Size, POOL_TYPE PoolType ); VOID ExpRemovePoolTracker ( ULONG Key, ULONG Size, POOL_TYPE PoolType ); PPOOL_TRACKER_BIG_PAGES ExpAddTagForBigPages ( IN PVOID Va, IN ULONG Key ); ULONG ExpFindAndRemoveTagBigPages ( IN PVOID Va ); PVOID ExpAllocateStringRoutine( IN ULONG NumberOfBytes ) { return ExAllocatePoolWithTag(PagedPool,NumberOfBytes,'grtS'); } BOOLEAN ExOkayToLockRoutine( IN PVOID Lock ); BOOLEAN ExOkayToLockRoutine( IN PVOID Lock ) { if (KeIsExecutingDpc()) { return FALSE; } else { return TRUE; } } PRTL_ALLOCATE_STRING_ROUTINE RtlAllocateStringRoutine = ExpAllocateStringRoutine; PRTL_FREE_STRING_ROUTINE RtlFreeStringRoutine = (PRTL_FREE_STRING_ROUTINE)ExFreePool; // // Define macros to pack and unpack a pool index. // #define PACK_POOL_INDEX(Index) ((UCHAR)(((Index) << 4) | (Index))) #define UNPACK_POOL_INDEX(Index) ((ULONG)((Index) & 0xf)) // // This structure exists in the pool descriptor structure. There is one of // these for each pool block size // // The check list macro comes in three flavors there is one in the checked // build that will assert if the doubly linked list is ill-formed. There // is one that is noop for the shipping version and then there is one that // can be enabled to check the list in the free build. // #if DBG #define CHECK_LIST(LINE,LIST,ENTRY) \ ASSERT((LIST)->Flink->Blink == (LIST)); \ ASSERT((LIST)->Blink->Flink == (LIST)); #elif 1 #define CHECK_LIST(LINE,LIST,ENTRY) {NOTHING;} #else #define CHECK_LIST(LINE,LIST,ENTRY) \ if (((LIST)->Flink->Blink != (LIST)) || \ ((LIST)->Blink->Flink != (LIST))) { \ KeBugCheckEx (BAD_POOL_HEADER,3,(ULONG)LIST,LINE,(ULONG)ENTRY);\ } #endif //DBG #define CHECK_POOL_PAGE(PAGE) \ { \ PPOOL_HEADER P = (PPOOL_HEADER)(((ULONG)(PAGE)) & ~(PAGE_SIZE-1)); \ ULONG SIZE, LSIZE; \ BOOLEAN FOUND=FALSE; \ LSIZE = 0; \ SIZE = 0; \ do { \ if (P == (PPOOL_HEADER)PAGE) { \ FOUND = TRUE; \ } \ if (P->PreviousSize != LSIZE) { \ DbgPrint("POOL: Inconsistent size: ( %lx ) - %lx->%u != %u\n",\ PAGE, P, P->PreviousSize, LSIZE); \ DbgBreakPoint(); \ } \ LSIZE = P->BlockSize; \ SIZE += LSIZE; \ P = (PPOOL_HEADER)((PPOOL_BLOCK)P + LSIZE); \ } while ((SIZE < (PAGE_SIZE / POOL_SMALLEST_BLOCK)) && \ (PAGE_END(P) == FALSE)); \ if ((PAGE_END(P) == FALSE) || (FOUND == FALSE)) { \ DbgPrint("POOL: Inconsistent page: %lx\n",P); \ DbgBreakPoint(); \ } \ } // // Define the number of paged pools. This value may be overridden at boot // time. // ULONG ExpNumberOfPagedPools = NUMBER_OF_PAGED_POOLS; // // Pool descriptors for nonpaged pool and nonpaged pool must succeed are // static. The pool descriptors for paged pool are dynamically allocated // since there can be more than one paged pool. There is always one more // paged pool descriptor than there are paged pools. This descriptor is // used when a page allocation is done for a paged pool and is the first // descriptor in the paged ppol descriptor array. // POOL_DESCRIPTOR NonPagedPoolDescriptor; POOL_DESCRIPTOR NonPagedPoolDescriptorMS; // // The pool vector contains an array of pointers to pool descriptors. For // nonpaged pool and nonpaged pool must success, this is a pointer to a // single descriptor. For page pool, this is a pointer to an array of pool // descriptors. The pointer to the paged pool descriptor is duplicated so // if can be found easily by the kernel debugger. // PPOOL_DESCRIPTOR PoolVector[NUMBER_OF_POOLS]; PPOOL_DESCRIPTOR ExpPagedPoolDescriptor; extern KSPIN_LOCK NonPagedPoolLock; extern KSPIN_LOCK PoolTraceLock; volatile ULONG ExpPoolIndex = 1; KSPIN_LOCK ExpTaggedPoolLock; #if DBG PSZ PoolTypeNames[MaxPoolType] = { "NonPaged", "Paged", "NonPagedMustSucceed", "NotUsed", "NonPagedCacheAligned", "PagedCacheAligned", "NonPagedCacheAlignedMustS" }; #endif //DBG // // Define paged and nonpaged pool lookaside descriptors. // SMALL_POOL_LOOKASIDE ExpSmallNPagedPoolLookasideLists[POOL_SMALL_LISTS]; #if !defined(_PPC_) SMALL_POOL_LOOKASIDE ExpSmallPagedPoolLookasideLists[POOL_SMALL_LISTS]; #endif // // Two routines to check for pool that has been altered after it was freed. // #if DEADBEEF _inline VOID ExFillFreedPool ( IN PVOID Buffer, IN ULONG Size, IN ULONG Tag ) { RtlFillMemoryUlong( Buffer, Size, Tag ); return; } VOID ExCheckFreedPool ( IN ULONG LineNumber, IN PPOOL_HEADER PoolHeader ) { ULONG MatchBytes; ULONG i; MatchBytes = RtlCompareMemoryUlong( (PCHAR)PoolHeader + 0x10, 0x20 - 0x10, PoolHeader->PoolTag ); if (MatchBytes != 0x20 - 0x10) { DbgPrint("EX(%d): Freed pool block %lx modified at %lx after it was freed\n", LineNumber, ((PCHAR)PoolHeader), ((PCHAR)PoolHeader) + 0x10 + MatchBytes); DbgBreakPoint(); } for (i = 1; i < PoolHeader->BlockSize; i += 1) { MatchBytes = RtlCompareMemoryUlong( (((PCHAR)PoolHeader) + (0x20 * i)), 0x20, *((PULONG)(((PCHAR)PoolHeader) + (0x20 * i))) ); if (MatchBytes != 0x20) { DbgPrint("EX(%d): Freed pool block %lx modified at %lx after it was freed\n", LineNumber, ((PCHAR)PoolHeader), ((PCHAR)PoolHeader) + (0x20 * i) + MatchBytes); DbgBreakPoint(); } } return; } #endif // // LOCK_POOL and LOCK_IF_PAGED_POOL are only used within this module. // #define LOCK_POOL(PoolDesc, LockHandle) { \ if ((PoolDesc->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { \ ExAcquireSpinLock(&NonPagedPoolLock, &LockHandle); \ } else { \ ExAcquireFastMutex((PFAST_MUTEX)PoolDesc->LockAddress); \ } \ } #define LOCK_IF_PAGED_POOL(CheckType) \ if (CheckType == PagedPool) { \ ExAcquireFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); \ } KIRQL ExLockPool( IN POOL_TYPE PoolType ) /*++ Routine Description: This function locks the pool specified by pool type. Arguments: PoolType - Specifies the pool that should be locked. Return Value: The previous IRQL is returned as the function value. --*/ { KIRQL OldIrql; // // If the pool type is nonpaged, then use a spinlock to lock the // pool. Otherwise, use a fast mutex to lock the pool. // if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { ExAcquireSpinLock(NonPagedPoolDescriptor.LockAddress, &OldIrql); } else { ExAcquireFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); OldIrql = (KIRQL)((PFAST_MUTEX)(PoolVector[PagedPool]->LockAddress))->OldIrql; } return OldIrql; } // // UNLOCK_POOL and UNLOCK_IF_PAGED_POOL are only used within this module. // #define UNLOCK_POOL(PoolDesc, LockHandle) { \ if ((PoolDesc->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { \ ExReleaseSpinLock(&NonPagedPoolLock, (KIRQL)LockHandle); \ } else { \ ExReleaseFastMutex((PFAST_MUTEX)PoolDesc->LockAddress); \ } \ } #define UNLOCK_IF_PAGED_POOL(CheckType) \ if (CheckType == PagedPool) { \ ExReleaseFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); \ } VOID ExUnlockPool( IN POOL_TYPE PoolType, IN KIRQL LockHandle ) /*++ Routine Description: This function unlocks the pool specified by pool type. Arguments: PoolType - Specifies the pool that should be unlocked. LockHandle - Specifies the lock handle from a previous call to ExLockPool. Return Value: None. --*/ { // // If the pool type is nonpaged, then use a spinlock to unlock the // pool. Otherwise, use a fast mutex to unlock the pool. // if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { ExReleaseSpinLock(&NonPagedPoolLock, LockHandle); } else { ExReleaseFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); } return; } VOID ExpInitializePoolDescriptor( IN PPOOL_DESCRIPTOR PoolDescriptor, IN POOL_TYPE PoolType, IN ULONG PoolIndex, IN ULONG Threshold, IN PVOID PoolLock ) /*++ Routine Description: This function initializes a pool descriptor. Arguments: PoolDescriptor - Supplies a pointer to the pool descriptor. PoolType - Supplies the type of the pool. PoolIndex - Supplies the pool descriptor index. Threshold - Supplies the threshold value for the specified pool. PoolLock - Supplies a point to the lock for the specified pool. Return Value: None. --*/ { ULONG Index; // // Initialize statistics fields, the pool type, the threshold value, // and the lock address // PoolDescriptor->PoolType = PoolType; PoolDescriptor->PoolIndex = PoolIndex; PoolDescriptor->RunningAllocs = 0; PoolDescriptor->RunningDeAllocs = 0; PoolDescriptor->TotalPages = 0; PoolDescriptor->TotalBigPages = 0; PoolDescriptor->Threshold = Threshold; PoolDescriptor->LockAddress = PoolLock; // // Initialize the allocation listheads. // for (Index = 0; Index < POOL_LIST_HEADS; Index += 1) { InitializeListHead(&PoolDescriptor->ListHeads[Index]); } return; } VOID InitializePool( IN POOL_TYPE PoolType, IN ULONG Threshold ) /*++ Routine Description: This procedure initializes a pool descriptor for the specified pool type. Once initialized, the pool may be used for allocation and deallocation. This function should be called once for each base pool type during system initialization. Each pool descriptor contains an array of list heads for free blocks. Each list head holds blocks which are a multiple of the POOL_BLOCK_SIZE. The first element on the list [0] links together free entries of size POOL_BLOCK_SIZE, the second element [1] links together entries of POOL_BLOCK_SIZE * 2, the third POOL_BLOCK_SIZE * 3, etc, up to the number of blocks which fit into a page. Arguments: PoolType - Supplies the type of pool being initialized (e.g. nonpaged pool, paged pool...). Threshold - Supplies the threshold value for the specified pool. Return Value: None. --*/ { PPOOL_DESCRIPTOR Descriptor; ULONG Index; PFAST_MUTEX FastMutex; ULONG Size; ASSERT((PoolType & MUST_SUCCEED_POOL_TYPE_MASK) == 0); if (PoolType == NonPagedPool) { // // Initialize nonpaged pools. // #if !DBG if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) { #endif //!DBG PoolTrackTable = MiAllocatePoolPages(NonPagedPool, MAX_TRACKER_TABLE * sizeof(POOL_TRACKER_TABLE)); RtlZeroMemory(PoolTrackTable, MAX_TRACKER_TABLE * sizeof(POOL_TRACKER_TABLE)); PoolBigPageTable = MiAllocatePoolPages(NonPagedPool, MAX_BIGPAGE_TABLE * sizeof(POOL_TRACKER_BIG_PAGES)); RtlZeroMemory(PoolBigPageTable, MAX_BIGPAGE_TABLE * sizeof(POOL_TRACKER_BIG_PAGES)); #if !DBG } #endif //!DBG // // Initialize the spinlocks for nonpaged pool. // KeInitializeSpinLock (&ExpTaggedPoolLock); KeInitializeSpinLock(&NonPagedPoolLock); KeInitializeSpinLock(&PoolTraceLock); // // Initialize the nonpaged pool descriptor. // PoolVector[NonPagedPool] = &NonPagedPoolDescriptor; ExpInitializePoolDescriptor(&NonPagedPoolDescriptor, NonPagedPool, 0, Threshold, (PVOID)&NonPagedPoolLock); // // Initialize the nonpaged must succeed pool descriptor. // PoolVector[NonPagedPoolMustSucceed] = &NonPagedPoolDescriptorMS; ExpInitializePoolDescriptor(&NonPagedPoolDescriptorMS, NonPagedPoolMustSucceed, 0, 0, (PVOID)&NonPagedPoolLock); #if DBG if (MmSpecialPoolTag != 0) { MmInitializeSpecialPool(); } #endif //DBG } else { // // Allocate memory for the paged pool descriptors and fast mutexes. // Size = (ExpNumberOfPagedPools + 1) * (sizeof(FAST_MUTEX) + sizeof(POOL_DESCRIPTOR)); Descriptor = (PPOOL_DESCRIPTOR)ExAllocatePoolWithTag (NonPagedPoolMustSucceed, Size, 'looP'); if (PoolTrackTable) { ExpInsertPoolTracker('looP', MAX_TRACKER_TABLE * sizeof(POOL_TRACKER_TABLE), NonPagedPool); ExpInsertPoolTracker('looP', MAX_BIGPAGE_TABLE * sizeof(POOL_TRACKER_BIG_PAGES), NonPagedPool); } FastMutex = (PFAST_MUTEX)(Descriptor + ExpNumberOfPagedPools + 1); PoolVector[PagedPool] = Descriptor; ExpPagedPoolDescriptor = Descriptor; for (Index = 0; Index < (ExpNumberOfPagedPools + 1); Index += 1) { ExInitializeFastMutex(FastMutex); ExpInitializePoolDescriptor(Descriptor, PagedPool, Index, Threshold, (PVOID)FastMutex); Descriptor += 1; FastMutex += 1; } } // // The maximum cache alignment must be less than the size of the // smallest pool block because the lower bits are being cleared // in ExFreePool to find the entry's address. // #if POOL_CACHE_SUPPORTED // // Compute pool cache information. // PoolCacheSize = HalGetDmaAlignmentRequirement(); ASSERT(PoolCacheSize >= POOL_OVERHEAD); PoolCacheOverhead = PoolCacheSize + PoolCacheSize - (sizeof(POOL_HEADER) + 1); #ifndef CHECK_POOL_TAIL PoolBuddyMax = (POOL_PAGE_SIZE - (POOL_OVERHEAD + (3*POOL_SMALLEST_BLOCK) + 2*PoolCacheSize)); #else PoolBuddyMax = (POOL_PAGE_SIZE - (POOL_OVERHEAD + 2*PoolCacheSize + (4*POOL_SMALLEST_BLOCK))); #endif // CHECK_POOL_TAIL #endif //POOL_CACHE_SUPPORTED return; } #if DBG VOID ExpVerifyPool( PPOOL_DESCRIPTOR PoolDescriptor ) /*++ Routine Description: This function verifies the specified pool Arguments: PoolDesc - Supplies a pointer to a pool descriptor. Return Value: None. --*/ { PLIST_ENTRY Entry; ULONG Index; PLIST_ENTRY ListHead; ULONG Number; PPOOL_HEADER PoolHeader; // // Scan each of the allocation lists and perform the following checks: // // 1. Make sure each free block is in the correct list. // // 2. Make sure each free block is really free. // // 3. Make sure all the blocks in each page add up to a page. // // for (Index = 0; Index < POOL_LIST_HEADS; Index += 1) { ListHead = &PoolDescriptor->ListHeads[Index]; Entry = ListHead->Flink; while (Entry != ListHead) { PoolHeader = (PPOOL_HEADER)((PCHAR)Entry - POOL_OVERHEAD); // // Assert that the pool block is not allocated. // ASSERT(PoolHeader->PoolType == 0); ASSERT(PoolHeader->PoolIndex == PACK_POOL_INDEX(PoolDescriptor->PoolIndex)); // // Assert that the pool block is in the correct list. // Number = PoolHeader->BlockSize; if (Number > POOL_SMALL_LISTS) { Number = (Number >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1; } ASSERT(Index == (Number - 1)); // // Check to make sure the pool block is properly filled. // #if DEADBEEF if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { // // Make sure the pool block has not been modified after it was freed. // ExCheckFreedPool( __LINE__, PoolHeader ); CHECK_POOL_PAGE(PoolHeader); } #endif //DEADBEEF Entry = Entry->Flink; } } return; } #else #define ExpVerifyPool(PoolDesc) #endif PVOID ExAllocatePool( IN POOL_TYPE PoolType, IN ULONG NumberOfBytes ) /*++ Routine Description: This function allocates a block of pool of the specified type and returns a pointer to the allocated block. This function is used to access both the page-aligned pools, and the list head entries (less than a page) pools. If the number of bytes specifies a size that is too large to be satisfied by the appropriate list, then the page-aligned pool allocator is used. The allocated block will be page-aligned and a page-sized multiple. Otherwise, the appropriate pool list entry is used. The allocated block will be 64-bit aligned, but will not be page aligned. The pool allocator calculates the smallest number of POOL_BLOCK_SIZE that can be used to satisfy the request. If there are no blocks available of this size, then a block of the next larger block size is allocated and split. One piece is placed back into the pool, and the other piece is used to satisfy the request. If the allocator reaches the paged-sized block list, and nothing is there, the page-aligned pool allocator is called. The page is split and added to the pool... Arguments: PoolType - Supplies the type of pool to allocate. If the pool type is one of the "MustSucceed" pool types, then this call will always succeed and return a pointer to allocated pool. Otherwise, if the system can not allocate the requested amount of memory a NULL is returned. Valid pool types: NonPagedPool PagedPool NonPagedPoolMustSucceed, NonPagedPoolCacheAligned PagedPoolCacheAligned NonPagedPoolCacheAlignedMustS NumberOfBytes - Supplies the number of bytes to allocate. Return Value: NULL - The PoolType is not one of the "MustSucceed" pool types, and not enough pool exists to satisfy the request. NON-NULL - Returns a pointer to the allocated pool. --*/ { return ExAllocatePoolWithTag(PoolType, NumberOfBytes, 'enoN'); } PVOID ExAllocatePoolWithTag( IN POOL_TYPE PoolType, IN ULONG NumberOfBytes, IN ULONG Tag ) /*++ Routine Description: This function allocates a block of pool of the specified type and returns a pointer to the allocated block. This function is used to access both the page-aligned pools and the list head entries (less than a page) pools. If the number of bytes specifies a size that is too large to be satisfied by the appropriate list, then the page-aligned pool allocator is used. The allocated block will be page-aligned and a page-sized multiple. Otherwise, the appropriate pool list entry is used. The allocated block will be 64-bit aligned, but will not be page aligned. The pool allocator calculates the smallest number of POOL_BLOCK_SIZE that can be used to satisfy the request. If there are no blocks available of this size, then a block of the next larger block size is allocated and split. One piece is placed back into the pool, and the other piece is used to satisfy the request. If the allocator reaches the paged-sized block list, and nothing is there, the page-aligned pool allocator is called. The page is split and added to the pool. Arguments: PoolType - Supplies the type of pool to allocate. If the pool type is one of the "MustSucceed" pool types, then this call will always succeed and return a pointer to allocated pool. Otherwise, if the system can not allocate the requested amount of memory a NULL is returned. Valid pool types: NonPagedPool PagedPool NonPagedPoolMustSucceed, NonPagedPoolCacheAligned PagedPoolCacheAligned NonPagedPoolCacheAlignedMustS NumberOfBytes - Supplies the number of bytes to allocate. Return Value: NULL - The PoolType is not one of the "MustSucceed" pool types, and not enough pool exists to satisfy the request. NON-NULL - Returns a pointer to the allocated pool. --*/ { PVOID Block; PPOOL_HEADER Entry; PSMALL_POOL_LOOKASIDE LookasideList; PPOOL_HEADER NextEntry; PPOOL_HEADER SplitEntry; KIRQL LockHandle; PPOOL_DESCRIPTOR PoolDesc; PVOID Lock; ULONG Index; ULONG ListNumber; ULONG NeededSize; ULONG PoolIndex; POOL_TYPE CheckType; PLIST_ENTRY ListHead; USHORT PoolTagHash; PKPRCB Prcb; POOL_TYPE NewPoolType; #if POOL_CACHE_SUPPORTED ULONG CacheOverhead; #else #define CacheOverhead POOL_OVERHEAD #endif #if DBG VOID CalculatePoolUtilization( IN PPOOL_DESCRIPTOR PoolDesc, IN ULONG BytesWanted ); #endif ASSERT(NumberOfBytes != 0); // // Isolate the base pool type and select a pool from which to allocate // the specified block size. // CheckType = PoolType & BASE_POOL_TYPE_MASK; PoolDesc = PoolVector[CheckType]; // // Check to determine if the requested block can be allocated from one // of the pool lists or must be directed allocated from virtual memory. // if (NumberOfBytes > POOL_BUDDY_MAX) { // // The requested size is greater than the largest block maintained // by allocation lists. // ASSERT((PoolType & MUST_SUCCEED_POOL_TYPE_MASK) == 0); LOCK_POOL(PoolDesc, LockHandle); PoolDesc->RunningAllocs++; Entry = (PPOOL_HEADER)MiAllocatePoolPages(CheckType, NumberOfBytes); UNLOCK_POOL(PoolDesc, LockHandle); if (Entry != NULL) { PPOOL_TRACKER_BIG_PAGES p; PoolDesc->TotalBigPages += BYTES_TO_PAGES(NumberOfBytes); if (PoolBigPageTable != FALSE) { if (!(p = ExpAddTagForBigPages((PVOID)Entry, Tag))) { Tag = ' GIB'; } #if DBG || (i386 && !FPO) else if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) && Entry) { p->NumberOfPages = (USHORT)BYTES_TO_PAGES(NumberOfBytes); #if i386 && !FPO p->AllocatorBackTraceIndex = RtlLogStackBackTrace(); #endif // i386 && !FPO } #endif // DBG || (i386 && !FPO) ExpInsertPoolTracker(Tag, ROUND_TO_PAGES(NumberOfBytes), PoolType); } } else { KdPrint(("EX: ExAllocatePool( %d ) returning NULL\n",NumberOfBytes)); if ((PoolType & POOL_RAISE_IF_ALLOCATION_FAILURE) != 0) { ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); } } #if DBG if (ExpEchoPoolCalls){ PVOID CallingAddress; PVOID CallersCaller; DbgPrint("0x%lx EXALLOC: from %s size %d", Entry, PoolTypeNames[PoolType & MaxPoolType], NumberOfBytes ); RtlGetCallersAddress(&CallingAddress, &CallersCaller); DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller); } #endif //DBG return Entry; } else { // // The requested size is less than of equal to the size of the // maximum block maintained by the allocation lists. // #if DBG if ((MmSpecialPoolTag != 0) && (Tag == MmSpecialPoolTag)) { return MmAllocateSpecialPool(NumberOfBytes, Tag); } #endif //DBG // // If the request is for cache aligned memory adjust the number of // bytes. // #if POOL_CACHE_SUPPORTED //only compile for machines which have a nonzero value. CacheOverhead = POOL_OVERHEAD; if (PoolType & CACHE_ALIGNED_POOL_TYPE_MASK) { NumberOfBytes += PoolCacheOverhead; CacheOverhead = PoolCacheSize; } #endif //POOL_CACHE_SUPPORTED // // Compute the Index of the listhead for blocks of the requested // size. // ListNumber = ((NumberOfBytes + POOL_OVERHEAD + (POOL_SMALLEST_BLOCK - 1)) >> POOL_BLOCK_SHIFT); #ifdef CHECK_POOL_TAIL ListNumber += 1; #endif // CHECK_POOL_TAIL NeededSize = ListNumber; if (ListNumber > POOL_SMALL_LISTS) { ListNumber = (ListNumber >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 2; } // // If the pool type is paged, then pick a starting pool number and // attempt to lock each paged pool in circular succession. Otherwise, // lock the nonpaged pool as the same lock is used for both nonpaged // and nonpage must succeed. // // N.B. The paged pool is selected in a round robin fashion using a // simple counter. Note that the counter is incremented using a // a noninterlocked sequence, but the pool index is never allowed // to get out of range. // Prcb = KeGetCurrentPrcb(); if (CheckType == PagedPool) { // // If the requested pool block is a small block, then attempt to // allocate the requested pool from the per processor single entry // lookaside list. If the allocation attempt fails, then select a // pool to alocate from and allocate the block normally. // #if !defined(CHECK_POOL_TAIL) && (DEADBEEF == 0) if (NeededSize <= POOL_SMALL_LISTS) { #if defined(_PPC_) if ((Entry = (PPOOL_HEADER)InterlockedExchange((PLONG)&Prcb->PagedFreeEntry[NeededSize - 1], (LONG)NULL)) != NULL) { PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex); Prcb->PagedPoolLookasideHits += 1; #else LookasideList = &ExpSmallPagedPoolLookasideLists[NeededSize - 1]; LookasideList->TotalAllocates += 1; if ((Isx86FeaturePresent(KF_CMPXCHG8B)) && (Entry = (PPOOL_HEADER)ExInterlockedPopEntrySList(&LookasideList->SListHead, &LookasideList->Lock)) != NULL) { Entry -= 1; LookasideList->AllocateHits += 1; #endif NewPoolType = (PoolType & (BASE_POOL_TYPE_MASK | POOL_QUOTA_MASK)) + 1; #if defined (_ALPHA_) // // On Alpha, Entry->PoolType cannot be updated without synchronizing with // updates to Entry->PreviousSize. Otherwise, the lack of byte granularity // can cause one update to get lost. In order to avoid an expensive interlocked // operation, check PoolType to see if it really needs to be updated. // if (Entry->PoolType != NewPoolType) { ULONG NewHeader; ULONG OldHeader; NewHeader = Entry->Ulong1; do { OldHeader = NewHeader; ((PPOOL_HEADER)(&NewHeader))->PoolType = NewPoolType; NewHeader = (ULONG)InterlockedCompareExchange((PVOID *)&Entry->Ulong1, (PVOID)NewHeader, (PVOID)OldHeader); } while ( NewHeader != OldHeader ); } #else Entry->PoolType = NewPoolType; #endif if (PoolTrackTable != NULL) { ExpInsertPoolTracker(Tag, Entry->BlockSize << POOL_BLOCK_SHIFT, PoolType); } Entry->PoolTag = Tag; return (PUCHAR)Entry + CacheOverhead; } } #endif // // If there is more than one paged pool, then attempt to find // one that can be immediately locked. // PoolIndex = 1; if (ExpNumberOfPagedPools != PoolIndex) { ExpPoolIndex += 1; PoolIndex = ExpPoolIndex; if (PoolIndex > ExpNumberOfPagedPools) { PoolIndex = 1; ExpPoolIndex = 1; } Index = PoolIndex; do { Lock = PoolDesc[PoolIndex].LockAddress; if (ExTryToAcquireFastMutex((PFAST_MUTEX)Lock) != FALSE) { goto PoolLocked; } PoolIndex += 1; if (PoolIndex > ExpNumberOfPagedPools) { PoolIndex = 1; } } while (PoolIndex != Index); } // // None of the paged pools could be conditionally locked or there // is only one paged pool. The first pool considered is picked as // the victim to wait on. // Lock = PoolDesc[PoolIndex].LockAddress; ExAcquireFastMutex((PFAST_MUTEX)Lock); PoolLocked: PoolDesc = &PoolDesc[PoolIndex]; } else { // // If the requested pool block is a small block, then attempt to // allocate the requested pool from the per processor single entry // lookaside list. If the allocation attempt fails, then allocate // the pool normally. // #if !defined(CHECK_POOL_TAIL) && (DEADBEEF == 0) if (NeededSize <= POOL_SMALL_LISTS) { LookasideList = &ExpSmallNPagedPoolLookasideLists[NeededSize - 1]; LookasideList->TotalAllocates += 1; if ((Entry = (PPOOL_HEADER)ExInterlockedPopEntrySList(&LookasideList->SListHead, &LookasideList->Lock)) != NULL) { Entry -= 1; LookasideList->AllocateHits += 1; NewPoolType = (PoolType & (BASE_POOL_TYPE_MASK | POOL_QUOTA_MASK)) + 1; #if defined (_ALPHA_) // // On Alpha, Entry->PoolType cannot be updated without synchronizing with // updates to Entry->PreviousSize. Otherwise, the lack of byte granularity // can cause one update to get lost. In order to avoid an expensive interlocked // operation, check PoolType to see if it really needs to be updated. // if (Entry->PoolType != NewPoolType) { ULONG NewHeader; ULONG OldHeader; NewHeader = Entry->Ulong1; do { OldHeader = NewHeader; ((PPOOL_HEADER)(&NewHeader))->PoolType = NewPoolType; NewHeader = (ULONG)InterlockedCompareExchange((PVOID *)&Entry->Ulong1, (PVOID)NewHeader, (PVOID)OldHeader); } while ( NewHeader != OldHeader ); } #else Entry->PoolType = NewPoolType; #endif if (PoolTrackTable != NULL) { ExpInsertPoolTracker(Tag, Entry->BlockSize << POOL_BLOCK_SHIFT, PoolType); } Entry->PoolTag = Tag; return (PUCHAR)Entry + CacheOverhead; } } #endif PoolIndex = 0; ExAcquireSpinLock(&NonPagedPoolLock, &LockHandle); } ASSERT(PoolIndex == PoolDesc->PoolIndex); // // The following code has an outer loop and an inner loop. // // The outer loop is utilized to repeat a nonpaged must succeed // allocation if necessary. // // The inner loop is used to repeat an allocation attempt if there // are no entries in any of the pool lists. // PoolDesc->RunningAllocs += 1; ListHead = &PoolDesc->ListHeads[ListNumber - 1]; do { // // Attempt to allocate the requested block from the current free // blocks. // do { // // If the list is not empty, then allocate a block from the // selected list. // if (IsListEmpty(ListHead) == FALSE) { CHECK_LIST( __LINE__, ListHead, 0 ); Block = RemoveHeadList(ListHead); CHECK_LIST( __LINE__, ListHead, 0 ); Entry = (PPOOL_HEADER)((PCHAR)Block - POOL_OVERHEAD); ASSERT(Entry->BlockSize >= NeededSize); ASSERT(Entry->PoolIndex == PACK_POOL_INDEX(PoolIndex)); ASSERT(Entry->PoolType == 0); if (Entry->BlockSize != NeededSize) { // // The selected block is larger than the allocation // request. Split the block and insert the remaining // fragment in the appropriate list. // // If the entry is at the start of a page, then take // the allocation from the front of the block so as // to minimize fragmentation. Otherwise, take the // allocation from the end of the block which may // also reduce fragmentation is the block is at the // end of a page. // if (Entry->PreviousSize == 0) { // // The entry is at the start of a page. // SplitEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + NeededSize); SplitEntry->BlockSize = Entry->BlockSize - (UCHAR)NeededSize; SplitEntry->PreviousSize = (UCHAR)NeededSize; // // If the allocated block is not at the end of a // page, then adjust the size of the next block. // NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)SplitEntry + SplitEntry->BlockSize); if (PAGE_END(NextEntry) == FALSE) { NextEntry->PreviousSize = SplitEntry->BlockSize; } } else { // // The entry is not at the start of a page. // SplitEntry = Entry; Entry->BlockSize -= (UCHAR)NeededSize; Entry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + Entry->BlockSize); Entry->PreviousSize = SplitEntry->BlockSize; // // If the allocated block is not at the end of a // page, then adjust the size of the next block. // NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + NeededSize); if (PAGE_END(NextEntry) == FALSE) { NextEntry->PreviousSize = (UCHAR)NeededSize; } } // // Set the size of the allocated entry, clear the pool // type of the split entry, set the index of the split // entry, and insert the split entry in the appropriate // free list. // Entry->BlockSize = (UCHAR)NeededSize; Entry->PoolIndex = PACK_POOL_INDEX(PoolIndex); SplitEntry->PoolType = 0; SplitEntry->PoolIndex = PACK_POOL_INDEX(PoolIndex); Index = SplitEntry->BlockSize; if (Index > POOL_SMALL_LISTS) { Index = (Index >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1; } InsertTailList(&PoolDesc->ListHeads[Index - 1], ((PLIST_ENTRY)((PCHAR)SplitEntry + POOL_OVERHEAD))); } Entry->PoolType = (PoolType & (BASE_POOL_TYPE_MASK | POOL_QUOTA_MASK)) + 1; #if DEADBEEF if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { // // Make sure the pool block has not been modified after it was freed // and then fill it in with allocated tags. // ExCheckFreedPool( __LINE__, Entry ); RtlFillMemoryUlong( (PCHAR)Entry + POOL_OVERHEAD, (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD, ALLOCATED_POOL ); CHECK_POOL_PAGE(Entry); } #endif //DEADBEEF if (PoolTrackTable != NULL) { PoolTagHash = ExpInsertPoolTracker(Tag, Entry->BlockSize << POOL_BLOCK_SHIFT, PoolType); } UNLOCK_POOL(PoolDesc, LockHandle); Entry->PoolTag = Tag; #if i386 && !FPO if (NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) { USHORT AllocatorBackTraceIndex; if (AllocatorBackTraceIndex = RtlLogStackBackTrace()) { Entry->AllocatorBackTraceIndex = AllocatorBackTraceIndex | POOL_BACKTRACEINDEX_PRESENT; Entry->PoolTagHash = PoolTagHash; } } #endif // i386 && !FPO #if DBG if (ExpEchoPoolCalls){ PVOID CallingAddress; PVOID CallersCaller; DbgPrint("0x%lx EXALLOC: from %s size %d", ((PCH)Entry + POOL_OVERHEAD), PoolTypeNames[PoolType & MaxPoolType], NumberOfBytes ); RtlGetCallersAddress(&CallingAddress, &CallersCaller); DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller); } #endif return (PCHAR)Entry + CacheOverhead; } else { ListHead += 1; } } while (ListHead != &PoolDesc->ListHeads[POOL_LIST_HEADS]); // // A block of the desired size does not exist and there are // no large blocks that can be split to satify the allocation. // Attempt to expand the pool by allocating another page to be // added to the pool. // // If the pool type is paged pool, then the paged pool page lock // must be held during the allocation of the pool pages. // LOCK_IF_PAGED_POOL(CheckType); Entry = (PPOOL_HEADER)MiAllocatePoolPages(CheckType, PAGE_SIZE); UNLOCK_IF_PAGED_POOL(CheckType); if (Entry == NULL) { if ((PoolType & MUST_SUCCEED_POOL_TYPE_MASK) != 0) { // // Must succeed pool was requested. Reset the the type, // the pool descriptor address, and continue the search. // CheckType = NonPagedPoolMustSucceed; PoolDesc = PoolVector[NonPagedPoolMustSucceed]; ListHead = &PoolDesc->ListHeads[ListNumber - 1]; continue; } else { // // No more pool of the specified type is available. // KdPrint(("EX: ExAllocatePool( %d ) returning NULL\n", NumberOfBytes)); UNLOCK_POOL(PoolDesc, LockHandle); if ((PoolType & POOL_RAISE_IF_ALLOCATION_FAILURE) != 0) { ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); } return NULL; } } #if DEADBEEF if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { // // Fill in the entire page with freed pool tags. // ExFillFreedPool( (PCHAR)Entry + POOL_OVERHEAD, PAGE_SIZE - POOL_OVERHEAD, FREED_POOL ); Entry->PoolTag = FREED_POOL; } #endif // // Insert the allocate page in the last allocation list. // PoolDesc->TotalPages += 1; Entry->PoolType = 0; Entry->PoolIndex = PACK_POOL_INDEX(PoolIndex); // // N.B. A byte is used to store the block size in units of the // smallest block size. Therefore, if the number of small // blocks in the page is greater than 255, the block size // is set to 255. // if ((PAGE_SIZE / POOL_SMALLEST_BLOCK) > 255) { Entry->BlockSize = 255; } else { Entry->BlockSize = PAGE_SIZE / POOL_SMALLEST_BLOCK; } Entry->PreviousSize = 0; ListHead = &PoolDesc->ListHeads[POOL_LIST_HEADS - 1]; InsertHeadList(ListHead, ((PLIST_ENTRY)((PCHAR)Entry + POOL_OVERHEAD))); } while (TRUE); } } USHORT ExpInsertPoolTracker ( ULONG Key, ULONG Size, POOL_TYPE PoolType ) /*++ Routine Description: This function insert a pool tag in the tag table and increments the number of allocates and updates the total allocation size. Arguments: Key - Supplies the key value used to locate a matching entry in the tag table. Size - Supplies the allocation size. PoolType - Supplies the pool type. Return Value: The tag index is returned as the function value. --*/ { USHORT Result; ULONG Hash; ULONG Index; KIRQL OldIrql; // // Ignore protected pool bit except for returned hash index // if (Key & PROTECTED_POOL) { Key &= ~PROTECTED_POOL; Result = (USHORT)(PROTECTED_POOL >> 16); } else { Result = 0; } if (Key == PoolHitTag) { DbgBreakPoint(); } // // Compute hash index and search for pool tag. // Hash = ((40543*((((((((PUCHAR)&Key)[0]<<2)^((PUCHAR)&Key)[1])<<2)^((PUCHAR)&Key)[2])<<2)^((PUCHAR)&Key)[3]))>>2) & TRACKER_TABLE_MASK; Index = Hash; ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql); do { if ((PoolTrackTable[Hash].Key == Key) || (PoolTrackTable[Hash].Key == 0)) { goto EntryFound; } Hash = (Hash + 1) & TRACKER_TABLE_MASK; } while (Hash != Index); // // No matching entry and no free entry was found. // Hash = MAX_TRACKER_TABLE - 1; // // Update pool tracker table entry. // EntryFound: PoolTrackTable[Hash].Key = Key; if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) { PoolTrackTable[Hash].PagedAllocs += 1; PoolTrackTable[Hash].PagedBytes += Size; } else { PoolTrackTable[Hash].NonPagedAllocs += 1; PoolTrackTable[Hash].NonPagedBytes += Size; } ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); return (USHORT)Hash | Result; } VOID ExpRemovePoolTracker ( ULONG Key, ULONG Size, POOL_TYPE PoolType ) /*++ Routine Description: This function increments the number of frees and updates the total allocation size. Arguments: Key - Supplies the key value used to locate a matching entry in the tag table. Size - Supplies the allocation size. PoolType - Supplies the pool type. Return Value: None. --*/ { ULONG Hash; ULONG Index; KIRQL OldIrql; // // Ignore protected pool bit // Key &= ~PROTECTED_POOL; if (Key == PoolHitTag) { DbgBreakPoint(); } // // Compute hash index and search for pool tag. // Hash = ((40543*((((((((PUCHAR)&Key)[0]<<2)^((PUCHAR)&Key)[1])<<2)^((PUCHAR)&Key)[2])<<2)^((PUCHAR)&Key)[3]))>>2) & TRACKER_TABLE_MASK; Index = Hash; ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql); do { if (PoolTrackTable[Hash].Key == Key) { goto EntryFound; } if (PoolTrackTable[Hash].Key == 0) { KdPrint(("POOL: Unable to find tracker %lx, table corrupted\n", Key)); goto ExitRoutine; } Hash = (Hash + 1) & TRACKER_TABLE_MASK; } while (Hash != Index); // // No matching entry and no free entry was found. // Hash = MAX_TRACKER_TABLE - 1; // // Update pool tracker table entry. // EntryFound: if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) { PoolTrackTable[Hash].PagedBytes -= Size; PoolTrackTable[Hash].PagedFrees += 1; } else { PoolTrackTable[Hash].NonPagedBytes -= Size; PoolTrackTable[Hash].NonPagedFrees += 1; } ExitRoutine: ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); return; } PPOOL_TRACKER_BIG_PAGES ExpAddTagForBigPages ( IN PVOID Va, IN ULONG Key ) { ULONG Hash; BOOLEAN Inserted = TRUE; KIRQL OldIrql; Hash = ((ULONG)Va >> PAGE_SHIFT) & BIGPAGE_TABLE_MASK; ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql); while (PoolBigPageTable[Hash].Va != NULL) { Hash += 1; if (Hash > MAX_BIGPAGE_TABLE) { if (!Inserted) { if (!FirstPrint) { KdPrint(("POOL:unable to insert big page slot %lx\n",Key)); FirstPrint = TRUE; } ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); return NULL; } Hash = 0; Inserted = FALSE; } } PoolBigPageTable[Hash].Va = Va; PoolBigPageTable[Hash].Key = Key; ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); return &PoolBigPageTable[Hash]; } ULONG ExpFindAndRemoveTagBigPages ( IN PVOID Va ) { ULONG Hash; BOOLEAN Inserted = TRUE; KIRQL OldIrql; ULONG ReturnKey; Hash = ((ULONG)Va >> PAGE_SHIFT) & BIGPAGE_TABLE_MASK; ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql); while (PoolBigPageTable[Hash].Va != Va) { Hash += 1; if (Hash > MAX_BIGPAGE_TABLE) { if (!Inserted) { if (!FirstPrint) { KdPrint(("POOL:unable to find big page slot %lx\n",Va)); FirstPrint = TRUE; } ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); return ' GIB'; } Hash = 0; Inserted = FALSE; } } PoolBigPageTable[Hash].Va = NULL; ReturnKey = PoolBigPageTable[Hash].Key; ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); return ReturnKey; } ULONG ExpAllocatePoolWithQuotaHandler( IN NTSTATUS ExceptionCode, IN PVOID PoolAddress, IN BOOLEAN ContinueSearch ) /*++ Routine Description: This function is called when an exception occurs in ExFreePool while quota is being charged to a process. Its function is to deallocate the pool block and continue the search for an exception handler. Arguments: ExceptionCode - Supplies the exception code that caused this function to be entered. PoolAddress - Supplies the address of a pool block that needs to be deallocated. ContinueSearch - Supplies a value that if TRUE causes the exception search to continue. This is used in allocate pool with quota calls that do not contain the pool quota mask bit set. Return Value: EXCEPTION_CONTINUE_SEARCH - The exception should be propagated to the caller of ExAllocatePoolWithQuota. --*/ { if ( PoolAddress ) { ASSERT(ExceptionCode == STATUS_QUOTA_EXCEEDED); ExFreePool(PoolAddress); } else { ASSERT(ExceptionCode == STATUS_INSUFFICIENT_RESOURCES); } return ContinueSearch ? EXCEPTION_CONTINUE_SEARCH : EXCEPTION_EXECUTE_HANDLER; } PVOID ExAllocatePoolWithQuota( IN POOL_TYPE PoolType, IN ULONG NumberOfBytes ) /*++ Routine Description: This function allocates a block of pool of the specified type, returns a pointer to the allocated block, and if the binary buddy allocator was used to satisfy the request, charges pool quota to the current process. This function is used to access both the page-aligned pools, and the binary buddy. If the number of bytes specifies a size that is too large to be satisfied by the appropriate binary buddy pool, then the page-aligned pool allocator is used. The allocated block will be page-aligned and a page-sized multiple. No quota is charged to the current process if this is the case. Otherwise, the appropriate binary buddy pool is used. The allocated block will be 64-bit aligned, but will not be page aligned. After the allocation completes, an attempt will be made to charge pool quota (of the appropriate type) to the current process object. If the quota charge succeeds, then the pool block's header is adjusted to point to the current process. The process object is not dereferenced until the pool is deallocated and the appropriate amount of quota is returned to the process. Otherwise, the pool is deallocated, a "quota exceeded" condition is raised. Arguments: PoolType - Supplies the type of pool to allocate. If the pool type is one of the "MustSucceed" pool types and sufficient quota exists, then this call will always succeed and return a pointer to allocated pool. Otherwise, if the system can not allocate the requested amount of memory a STATUS_INSUFFICIENT_RESOURCES status is raised. NumberOfBytes - Supplies the number of bytes to allocate. Return Value: NON-NULL - Returns a pointer to the allocated pool. Unspecified - If insuffient quota exists to complete the pool allocation, the return value is unspecified. --*/ { return (ExAllocatePoolWithQuotaTag (PoolType, NumberOfBytes, 'enoN')); } PVOID ExAllocatePoolWithQuotaTag( IN POOL_TYPE PoolType, IN ULONG NumberOfBytes, IN ULONG Tag ) /*++ Routine Description: This function allocates a block of pool of the specified type, returns a pointer to the allocated block, and if the binary buddy allocator was used to satisfy the request, charges pool quota to the current process. This function is used to access both the page-aligned pools, and the binary buddy. If the number of bytes specifies a size that is too large to be satisfied by the appropriate binary buddy pool, then the page-aligned pool allocator is used. The allocated block will be page-aligned and a page-sized multiple. No quota is charged to the current process if this is the case. Otherwise, the appropriate binary buddy pool is used. The allocated block will be 64-bit aligned, but will not be page aligned. After the allocation completes, an attempt will be made to charge pool quota (of the appropriate type) to the current process object. If the quota charge succeeds, then the pool block's header is adjusted to point to the current process. The process object is not dereferenced until the pool is deallocated and the appropriate amount of quota is returned to the process. Otherwise, the pool is deallocated, a "quota exceeded" condition is raised. Arguments: PoolType - Supplies the type of pool to allocate. If the pool type is one of the "MustSucceed" pool types and sufficient quota exists, then this call will always succeed and return a pointer to allocated pool. Otherwise, if the system can not allocate the requested amount of memory a STATUS_INSUFFICIENT_RESOURCES status is raised. NumberOfBytes - Supplies the number of bytes to allocate. Return Value: NON-NULL - Returns a pointer to the allocated pool. Unspecified - If insuffient quota exists to complete the pool allocation, the return value is unspecified. --*/ { PVOID p; PEPROCESS Process; PPOOL_HEADER Entry; BOOLEAN IgnoreQuota = FALSE; BOOLEAN RaiseOnQuotaFailure = TRUE; if ( PoolType & POOL_QUOTA_FAIL_INSTEAD_OF_RAISE ) { RaiseOnQuotaFailure = FALSE; PoolType &= ~POOL_QUOTA_FAIL_INSTEAD_OF_RAISE; } if (PoolTrackTable #if i386 && !FPO || (NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) #endif // i386 && !FPO ) { IgnoreQuota = TRUE; } else { PoolType = (POOL_TYPE)((UCHAR)PoolType + POOL_QUOTA_MASK); } p = ExAllocatePoolWithTag(PoolType, NumberOfBytes, Tag); // // Note - NULL is page aligned. // if (!PAGE_ALIGNED(p) && !IgnoreQuota ) { #if POOL_CACHE_SUPPORTED // // Align entry on pool allocation boundary. // if (((ULONG)p & POOL_CACHE_CHECK) == 0) { Entry = (PPOOL_HEADER)((ULONG)p - PoolCacheSize); } else { Entry = (PPOOL_HEADER)((PCH)p - POOL_OVERHEAD); } #else Entry = (PPOOL_HEADER)((PCH)p - POOL_OVERHEAD); #endif //POOL_CACHE_SUPPORTED Process = PsGetCurrentProcess(); // // Catch exception and back out allocation if necessary // try { Entry->ProcessBilled = NULL; if ( Process != PsInitialSystemProcess ) { PsChargePoolQuota(Process, PoolType & BASE_POOL_TYPE_MASK, (ULONG)(Entry->BlockSize << POOL_BLOCK_SHIFT)); ObReferenceObject(Process); Entry->ProcessBilled = Process; } } except ( ExpAllocatePoolWithQuotaHandler(GetExceptionCode(),p,RaiseOnQuotaFailure)) { if ( RaiseOnQuotaFailure ) { KeBugCheck(GetExceptionCode()); } else { p = NULL; } } } else { if ( !p && RaiseOnQuotaFailure ) { ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); } } return p; } VOID ExFreePool( IN PVOID P ) { #ifdef POOL_TAGGING ExFreePoolWithTag(P, 0); return; } VOID ExFreePoolWithTag( IN PVOID P, IN ULONG TagToFree ) { #else ULONG TagToFree = 0; #endif /*++ Routine Description: This function deallocates a block of pool. This function is used to deallocate to both the page aligned pools, and the buddy (less than a page) pools. If the address of the block being deallocated is page-aligned, then the page-aliged pool deallocator is used. Otherwise, the binary buddy pool deallocator is used. Deallocation looks at the allocated block's pool header to determine the pool type and block size being deallocated. If the pool was allocated using ExAllocatePoolWithQuota, then after the deallocation is complete, the appropriate process's pool quota is adjusted to reflect the deallocation, and the process object is dereferenced. Arguments: P - Supplies the address of the block of pool being deallocated. Return Value: None. --*/ POOL_TYPE CheckType; PPOOL_HEADER Entry; ULONG Index; KIRQL LockHandle; PSMALL_POOL_LOOKASIDE LookasideList; PPOOL_HEADER NextEntry; ULONG PoolIndex; POOL_TYPE PoolType; PPOOL_DESCRIPTOR PoolDesc; PEPROCESS ProcessBilled = NULL; PKPRCB Prcb; BOOLEAN Combined; ULONG BigPages; ULONG Tag; #if DBG BOOLEAN ExpCheckForResource( IN PVOID p, IN ULONG Size ); #endif //DBG #if DBG if ((P > MmSpecialPoolStart) && (P < MmSpecialPoolEnd)) { MmFreeSpecialPool (P); return; } #endif //DBG // // If entry is page aligned, then call free block to the page aligned // pool. Otherwise, free the block to the allocation lists. // if (PAGE_ALIGNED(P)) { PoolType = MmDeterminePoolType(P); CheckType = PoolType & BASE_POOL_TYPE_MASK; PoolDesc = PoolVector[PoolType]; LOCK_POOL(PoolDesc, LockHandle); PoolDesc->RunningDeAllocs++; BigPages = MiFreePoolPages(P); PoolDesc->TotalBigPages -= BigPages; #if DBG // // Check is an ERESOURCE is current active in this memory block. // ExpCheckForResource(P, BigPages * PAGE_SIZE); #endif // DBG UNLOCK_POOL(PoolDesc, LockHandle); if (PoolTrackTable != NULL) { Tag = ExpFindAndRemoveTagBigPages(P); if (Tag & PROTECTED_POOL) { Tag &= ~PROTECTED_POOL; TagToFree &= ~PROTECTED_POOL; if (Tag != TagToFree) { DbgPrint( "EX: Invalid attempt to free protected pool block %x (%c%c%c%c)\n", P, Tag, Tag >> 8, Tag >> 16, Tag >> 24 ); DbgBreakPoint(); } } ExpRemovePoolTracker(Tag, BigPages * PAGE_SIZE, PoolType); } #if DBG if (ExpEchoPoolCalls){ PVOID CallingAddress; PVOID CallersCaller; DbgPrint("0x%lx EXDEALLOC: from %s", P, PoolTypeNames[PoolType]); RtlGetCallersAddress(&CallingAddress, &CallersCaller); DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller); } #endif //DBG } else { // // Align the entry address to a pool allocation boundary. // #if POOL_CACHE_SUPPORTED if (((ULONG)P & POOL_CACHE_CHECK) == 0) { Entry = (PPOOL_HEADER)((ULONG)P - PoolCacheSize); } else { Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD); } #else Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD); #endif //POOL_CACHE_SUPPORTED PoolType = (Entry->PoolType & POOL_TYPE_MASK) - 1; CheckType = PoolType & BASE_POOL_TYPE_MASK; #if DBG // // Check if an ERESOURCE is currently active in this memory block. // ExpCheckForResource(Entry, (ULONG)(Entry->BlockSize << POOL_BLOCK_SHIFT)); // // Check if the pool type field is defined correctly. // if (Entry->PoolType == 0) { DbgPrint("EX: Invalid pool header 0x%lx 0x%lx\n",P,*(PULONG)P); KeBugCheckEx(BAD_POOL_HEADER, 1, (ULONG)Entry, *(PULONG)Entry, 0); } // // Check if the pool index field is defined correctly. // if ((CheckType == NonPagedPool) && (Entry->PoolIndex != 0)) { DbgPrint("EX: Invalid pool header 0x%lx 0x%lx\n",Entry,*(PULONG)Entry); KeBugCheckEx(BAD_POOL_HEADER, 2, (ULONG)Entry, *(PULONG)Entry, 0); } else if (((CheckType == PagedPool) && (Entry->PoolIndex == 0)) || (((Entry->PoolIndex >> 4) & 0xf) != (Entry->PoolIndex & 0xf))) { DbgPrint("EX: Invalid pool header 0x%lx 0x%lx\n",Entry,*(PULONG)Entry); KeBugCheckEx(BAD_POOL_HEADER, 4, (ULONG)Entry, *(PULONG)Entry, 0); } #endif // DBG if (Entry->PoolType & POOL_QUOTA_MASK) { if (PoolTrackTable == NULL) { ProcessBilled = Entry->ProcessBilled; Entry->PoolTag = 'atoQ'; } } #if DBG if (ExpEchoPoolCalls){ PVOID CallingAddress; PVOID CallersCaller; DbgPrint("0x%lx EXDEALLOC: from %s", P, PoolTypeNames[PoolType]); RtlGetCallersAddress(&CallingAddress, &CallersCaller); DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller); } #endif #ifdef CHECK_POOL_TAIL if (NtGlobalFlag & FLG_POOL_ENABLE_TAIL_CHECK) { PCHAR PoolBlock; ULONG CountBytes; ULONG CountBytesEqual; PoolBlock = (PCHAR)(((PPOOL_BLOCK)Entry + Entry->BlockSize)) - POOL_SMALLEST_BLOCK; CountBytes = POOL_SMALLEST_BLOCK; CountBytesEqual = RtlCompareMemoryUlong(PoolBlock, CountBytes, ALLOCATED_POOL); if (CountBytesEqual != CountBytes) { DbgPrint("EX: Pool block at %lx modified at %lx past requested size of %lx\n", PoolBlock, PoolBlock + CountBytesEqual, (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_SMALLEST_BLOCK - POOL_OVERHEAD); DbgBreakPoint(); } } #endif //CHECK_POOL_TAIL #if i386 && !FPO if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) && PoolTrackTable != NULL && Entry->AllocatorBackTraceIndex != 0 && Entry->AllocatorBackTraceIndex & POOL_BACKTRACEINDEX_PRESENT) { if (Entry->PoolTagHash & (PROTECTED_POOL >> 16)) { Entry->PoolTagHash &= ~(PROTECTED_POOL >> 16); Tag = PROTECTED_POOL; } else { Tag = 0; } Entry->PoolTag = Tag | PoolTrackTable[Entry->PoolTagHash].Key; } #endif // i386 && !FPO // // If pool tagging is enabled, then update the pool tracking database. // Otherwise, check to determine if quota was charged when the pool // block was allocated. // if (PoolTrackTable != NULL) { Tag = Entry->PoolTag; if (Tag & PROTECTED_POOL) { Tag &= ~PROTECTED_POOL; TagToFree &= ~PROTECTED_POOL; if (Tag != TagToFree) { DbgPrint( "EX: Invalid attempt to free protected pool block %x (%c%c%c%c)\n", P, Tag, Tag >> 8, Tag >> 16, Tag >> 24 ); DbgBreakPoint(); } } ExpRemovePoolTracker(Tag, Entry->BlockSize << POOL_BLOCK_SHIFT , PoolType); } else if (ProcessBilled != NULL) { PsReturnPoolQuota(ProcessBilled, PoolType & BASE_POOL_TYPE_MASK, (ULONG)Entry->BlockSize << POOL_BLOCK_SHIFT); ObDereferenceObject(ProcessBilled); } // // If the pool block is a small block, then attempt to free the block // to the single entry lookaside list. If the free atempts fails, then // free the block by merging it back into the pool data structures. // PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex); PoolDesc = PoolVector[PoolType]; Index = Entry->BlockSize; if (Index > POOL_SMALL_LISTS) { Index = (Index >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1; } else { // // Attempt to free the small block to the per rpocessor single // entry lookaside list. // #if !defined(CHECK_POOL_TAIL) && (DEADBEEF == 0) Prcb = KeGetCurrentPrcb(); if (CheckType == PagedPool) { #if defined(_PPC_) if ((Entry = (PPOOL_HEADER)InterlockedExchange((PLONG)&Prcb->PagedFreeEntry[Index - 1], (LONG)Entry)) == NULL) { #else LookasideList = &ExpSmallPagedPoolLookasideLists[Index - 1]; LookasideList->TotalFrees += 1; if ((Isx86FeaturePresent(KF_CMPXCHG8B)) && (ExQueryDepthSList(&LookasideList->SListHead) < LookasideList->Depth)) { LookasideList->FreeHits += 1; Entry += 1; ExInterlockedPushEntrySList(&LookasideList->SListHead, (PSINGLE_LIST_ENTRY)Entry, &LookasideList->Lock); #endif return; } PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex); } else { LookasideList = &ExpSmallNPagedPoolLookasideLists[Index - 1]; LookasideList->TotalFrees += 1; if (ExQueryDepthSList(&LookasideList->SListHead) < LookasideList->Depth) { LookasideList->FreeHits += 1; Entry += 1; ExInterlockedPushEntrySList(&LookasideList->SListHead, (PSINGLE_LIST_ENTRY)Entry, &LookasideList->Lock); return; } PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex); } #endif } // // If the pool type is paged pool, then get the appropriate pool // descriptor address. // if (CheckType == PagedPool) { PoolDesc = &PoolDesc[PoolIndex]; } ASSERT(PoolIndex == PoolDesc->PoolIndex); LOCK_POOL(PoolDesc, LockHandle); #if DEADBEEF if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { CHECK_POOL_PAGE(Entry); } #endif PoolDesc->RunningDeAllocs += 1; #if DEADBEEF if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { // // Fill in the block with its previous pool tags. // ExFillFreedPool( (PCHAR)Entry + POOL_OVERHEAD, (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD, Entry->PoolTag ); } #endif // // Free the specified pool block. // // Check to see if the next entry is free. // Combined = FALSE; NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + Entry->BlockSize); if (PAGE_END(NextEntry) == FALSE) { if (NextEntry->PoolType == 0) { // // This block is free, combine with the released block. // CHECK_LIST(__LINE__, ((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD)), P); Combined = TRUE; RemoveEntryList(((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD))); Entry->BlockSize += NextEntry->BlockSize; #if DEADBEEF if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { // // Overwrite the poolheader with its tag because we're being combined // an the previous block. // ExFillFreedPool( (PCHAR)NextEntry, POOL_OVERHEAD + sizeof(LIST_ENTRY), NextEntry->PoolTag ); } #endif } } // // Check to see if the previous entry is free. // if (Entry->PreviousSize != 0) { NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry - Entry->PreviousSize); if (NextEntry->PoolType == 0) { // // This block is free, combine with the released block. // CHECK_LIST(__LINE__, ((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD)), P); Combined = TRUE; RemoveEntryList(((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD))); NextEntry->BlockSize += Entry->BlockSize; #if DEADBEEF if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { // // Overwrite the poolheader with its tag because we're being combined // an the previous block. // ExFillFreedPool( (PCHAR)Entry, POOL_OVERHEAD + sizeof(LIST_ENTRY), Entry->PoolTag ); } #endif Entry = NextEntry; } } // // If the block being freed has been combined into a full page, // then return the free the page to memory management. // if (PAGE_ALIGNED(Entry) && (PAGE_END((PPOOL_BLOCK)Entry + Entry->BlockSize) != FALSE)) { // // If the pool type is paged pool, then the paged pool page lock // must be held during the free of the pool pages. // LOCK_IF_PAGED_POOL(CheckType); MiFreePoolPages(Entry); UNLOCK_IF_PAGED_POOL(CheckType); PoolDesc->TotalPages -= 1; } else { // // Insert this element into the list. // Entry->PoolType = 0; Entry->PoolIndex = PACK_POOL_INDEX(PoolIndex); Index = Entry->BlockSize; if (Index > POOL_SMALL_LISTS) { Index = (Index >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1; } // // If the freed block was combined with any other block, then // adjust the size of the next block if necessary. // if (Combined != FALSE) { // // The size of this entry has changed, if this entry is // not the last one in the page, update the pool block // after this block to have a new previous allocation size. // NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + Entry->BlockSize); if (PAGE_END(NextEntry) == FALSE) { NextEntry->PreviousSize = Entry->BlockSize; } // // Reduce fragmentation and insert at the tail in hopes // neighbors for this will be freed before this is reallocated. // CHECK_LIST(__LINE__, &PoolDesc->ListHeads[Index - 1], P); InsertTailList(&PoolDesc->ListHeads[Index - 1], ((PLIST_ENTRY)((PCHAR)Entry + POOL_OVERHEAD))); } else { CHECK_LIST(__LINE__, &PoolDesc->ListHeads[Index - 1], P); InsertHeadList(&PoolDesc->ListHeads[Index - 1], ((PLIST_ENTRY)((PCHAR)Entry + POOL_OVERHEAD))); } } UNLOCK_POOL(PoolDesc, LockHandle); } return; } ULONG ExQueryPoolBlockSize ( IN PVOID PoolBlock, OUT PBOOLEAN QuotaCharged ) /*++ Routine Description: This function returns the size of the pool block. Arguments: PoolBlock - Supplies the address of the block of pool. QuotaCharged - Supplies a BOOLEAN variable to receive whether or not the pool block had quota charged. NOTE: If the entry is bigger than a page, the value PAGE_SIZE is returned rather than the correct number of bytes. Return Value: Size of pool block. --*/ { PPOOL_HEADER Entry; ULONG size; if (PAGE_ALIGNED(PoolBlock)) { *QuotaCharged = FALSE; return PAGE_SIZE; } #if POOL_CACHE_SUPPORTED // // Align entry on pool allocation boundary. // if (((ULONG)PoolBlock & POOL_CACHE_CHECK) == 0) { Entry = (PPOOL_HEADER)((ULONG)PoolBlock - PoolCacheSize); size = (Entry->BlockSize << POOL_BLOCK_SHIFT) - PoolCacheSize; } else { Entry = (PPOOL_HEADER)((PCHAR)PoolBlock - POOL_OVERHEAD); size = (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD; } #else Entry = (PPOOL_HEADER)((PCHAR)PoolBlock - POOL_OVERHEAD); size = (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD; #endif //POOL_CACHE_SUPPORTED #ifdef CHECK_POOL_TAIL size = size - POOL_SMALLEST_BLOCK; #endif if ( PoolTrackTable ) { *QuotaCharged = FALSE; } else { *QuotaCharged = (BOOLEAN) (Entry->ProcessBilled != NULL); } return size; } VOID ExQueryPoolUsage( OUT PULONG PagedPoolPages, OUT PULONG NonPagedPoolPages, OUT PULONG PagedPoolAllocs, OUT PULONG PagedPoolFrees, OUT PULONG PagedPoolLookasideHits, OUT PULONG NonPagedPoolAllocs, OUT PULONG NonPagedPoolFrees, OUT PULONG NonPagedPoolLookasideHits ) { ULONG Count; ULONG Index; PPOOL_DESCRIPTOR pd; PKPRCB Prcb; // // Sum all the paged pool usage. // pd = PoolVector[PagedPool]; *PagedPoolPages = 0; *PagedPoolAllocs = 0; *PagedPoolFrees = 0; for (Index = 0; Index < ExpNumberOfPagedPools + 1; Index += 1) { *PagedPoolPages += pd[Index].TotalPages + pd[Index].TotalBigPages; *PagedPoolAllocs += pd[Index].RunningAllocs; *PagedPoolFrees += pd[Index].RunningDeAllocs; } // // Sum all the nonpaged pool usage. // pd = PoolVector[NonPagedPool]; *NonPagedPoolPages = pd->TotalPages + pd->TotalBigPages; *NonPagedPoolAllocs = pd->RunningAllocs; *NonPagedPoolFrees = pd->RunningDeAllocs; // // Sum all the nonpaged must succeed usage. // pd = PoolVector[NonPagedPoolMustSucceed]; *NonPagedPoolPages += pd->TotalPages + pd->TotalBigPages; *NonPagedPoolAllocs += pd->RunningAllocs; *NonPagedPoolFrees += pd->RunningDeAllocs; // // Sum all the lookaside hits for paged and nonpaged pool. // for (Index = 0; Index < (ULONG)KeNumberProcessors; Index += 1) { Prcb = KiProcessorBlock[Index]; if (Prcb != NULL) { #if defined(_PPC_) *PagedPoolLookasideHits += Prcb->PagedPoolLookasideHits; for (Count = 0; Count < POOL_SMALL_LISTS; Count +=1) { *NonPagedPoolLookasideHits += ExpSmallNPagedPoolLookasideLists[Count].AllocateHits; } #else for (Count = 0; Count < POOL_SMALL_LISTS; Count +=1) { *PagedPoolLookasideHits += ExpSmallPagedPoolLookasideLists[Count].AllocateHits; *NonPagedPoolLookasideHits += ExpSmallNPagedPoolLookasideLists[Count].AllocateHits; } #endif } } return; } VOID ExReturnPoolQuota( IN PVOID P ) /*++ Routine Description: This function returns quota charged to a subject process when the specified pool block was allocated. Arguments: P - Supplies the address of the block of pool being deallocated. Return Value: None. --*/ { PPOOL_HEADER Entry; POOL_TYPE PoolType; PEPROCESS Process; // // Align the entry address to a pool allocation boundary. // #if POOL_CACHE_SUPPORTED if (((ULONG)P & POOL_CACHE_CHECK) == 0) { Entry = (PPOOL_HEADER)((ULONG)P - PoolCacheSize); } else { Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD); } #else Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD); #endif //POOL_CACHE_SUPPORTED // // If quota was charged, then return the appropriate quota to the // subject process. // PoolType = (Entry->PoolType & POOL_TYPE_MASK) - 1; if ((Entry->PoolType & POOL_QUOTA_MASK) && (PoolTrackTable == NULL)) { Process = Entry->ProcessBilled; Entry->PoolTag = 'atoQ'; Entry->PoolType &= ~POOL_QUOTA_MASK; if (Process != NULL) { PsReturnPoolQuota(Process, PoolType & BASE_POOL_TYPE_MASK, (ULONG)Entry->BlockSize << POOL_BLOCK_SHIFT); ObDereferenceObject(Process); } } return; } #if DBG || (i386 && !FPO) // // Only works on checked builds or free x86 builds with FPO turned off // See comment in mm\allocpag.c // NTSTATUS ExpSnapShotPoolPages( IN PVOID Address, IN ULONG Size, IN OUT PSYSTEM_POOL_INFORMATION PoolInformation, IN OUT PSYSTEM_POOL_ENTRY *PoolEntryInfo, IN ULONG Length, IN OUT PULONG RequiredLength ) { NTSTATUS Status; CLONG i; PPOOL_HEADER p; if (PAGE_ALIGNED(Address)) { for (i = 0; i < MAX_BIGPAGE_TABLE; i++) { if (PoolBigPageTable[i].NumberOfPages != 0 && PoolBigPageTable[i].Va == Address ) { PoolInformation->NumberOfEntries += 1; *RequiredLength += sizeof(SYSTEM_POOL_ENTRY); if (Length < *RequiredLength) { Status = STATUS_INFO_LENGTH_MISMATCH; } else { (*PoolEntryInfo)->Allocated = TRUE; (*PoolEntryInfo)->Size = (PoolBigPageTable[i].NumberOfPages * PAGE_SIZE); (*PoolEntryInfo)->AllocatorBackTraceIndex = 0; #if i386 && !FPO if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB)) { (*PoolEntryInfo)->AllocatorBackTraceIndex = PoolBigPageTable[i].AllocatorBackTraceIndex; } #endif // i386 && !FPO (*PoolEntryInfo)->ProcessChargedQuota = 0; #if !DBG if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) #endif //!DBG (*PoolEntryInfo)->TagUlong = PoolBigPageTable[i].Key; (*PoolEntryInfo)++; Status = STATUS_SUCCESS; } return Status; } } } p = (PPOOL_HEADER)Address; if ((Size == PAGE_SIZE) && (p->PreviousSize == 0)) { ULONG EntrySize; do { EntrySize = p->BlockSize << POOL_BLOCK_SHIFT; if (EntrySize == 0) { return STATUS_COMMITMENT_LIMIT; } PoolInformation->NumberOfEntries += 1; *RequiredLength += sizeof(SYSTEM_POOL_ENTRY); if (Length < *RequiredLength) { Status = STATUS_INFO_LENGTH_MISMATCH; } else { (*PoolEntryInfo)->Size = EntrySize; if (p->PoolType != 0) { (*PoolEntryInfo)->Allocated = TRUE; (*PoolEntryInfo)->AllocatorBackTraceIndex = 0; (*PoolEntryInfo)->ProcessChargedQuota = 0; #if !DBG if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) #endif //!DBG (*PoolEntryInfo)->TagUlong = p->PoolTag; #if i386 && !FPO if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) && p->AllocatorBackTraceIndex != 0 && p->AllocatorBackTraceIndex & POOL_BACKTRACEINDEX_PRESENT ) { (*PoolEntryInfo)->AllocatorBackTraceIndex = p->AllocatorBackTraceIndex ^ POOL_BACKTRACEINDEX_PRESENT; #if !DBG if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) { #else if (TRUE) { #endif //!DBG if (p->PoolTagHash & (PROTECTED_POOL >> 16)) { (*PoolEntryInfo)->TagUlong = PROTECTED_POOL; } else { (*PoolEntryInfo)->TagUlong = 0; } (*PoolEntryInfo)->TagUlong |= PoolTrackTable[p->PoolTagHash&~(PROTECTED_POOL >> 16)].Key; } } #endif // i386 && !FPO } else { (*PoolEntryInfo)->Allocated = FALSE; (*PoolEntryInfo)->AllocatorBackTraceIndex = 0; (*PoolEntryInfo)->ProcessChargedQuota = 0; #if !DBG if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) #endif //!DBG (*PoolEntryInfo)->TagUlong = p->PoolTag; } (*PoolEntryInfo)++; Status = STATUS_SUCCESS; } p = (PPOOL_HEADER)((PCHAR)p + EntrySize); } while (PAGE_END(p) == FALSE); } else { PoolInformation->NumberOfEntries += 1; *RequiredLength += sizeof(SYSTEM_POOL_ENTRY); if (Length < *RequiredLength) { Status = STATUS_INFO_LENGTH_MISMATCH; } else { (*PoolEntryInfo)->Allocated = TRUE; (*PoolEntryInfo)->Size = Size; (*PoolEntryInfo)->AllocatorBackTraceIndex = 0; (*PoolEntryInfo)->ProcessChargedQuota = 0; (*PoolEntryInfo)++; Status = STATUS_SUCCESS; } } return Status; } NTSTATUS ExSnapShotPool( IN POOL_TYPE PoolType, IN PSYSTEM_POOL_INFORMATION PoolInformation, IN ULONG Length, OUT PULONG ReturnLength OPTIONAL ) { ULONG Index; PVOID Lock; KIRQL LockHandle; PPOOL_DESCRIPTOR PoolDesc; ULONG RequiredLength; NTSTATUS Status; RequiredLength = FIELD_OFFSET(SYSTEM_POOL_INFORMATION, Entries); if (Length < RequiredLength) { return STATUS_INFO_LENGTH_MISMATCH; } try { // // If the pool type is paged, then lock all of the paged pools. // Otherwise, lock the nonpaged pool. // PoolDesc = PoolVector[PoolType]; if (PoolType == PagedPool) { Index = 0; KeRaiseIrql(APC_LEVEL, &LockHandle); \ do { Lock = PoolDesc[Index].LockAddress; ExAcquireFastMutex((PFAST_MUTEX)Lock); Index += 1; } while (Index < ExpNumberOfPagedPools); } else { ExAcquireSpinLock(&NonPagedPoolLock, &LockHandle); } PoolInformation->EntryOverhead = POOL_OVERHEAD; PoolInformation->NumberOfEntries = 0; #if POOL_CACHE_SUPPORTED //only compile for machines which have a nonzero value. if (PoolType & CACHE_ALIGNED_POOL_TYPE_MASK) { PoolInformation->EntryOverhead = (USHORT)PoolCacheSize; } #endif //POOL_CACHE_SUPPORTED Status = MmSnapShotPool(PoolType, ExpSnapShotPoolPages, PoolInformation, Length, &RequiredLength); } finally { // // If the pool type is paged, then unlock all of the paged pools. // Otherwise, unlock the nonpaged pool. // if (PoolType == PagedPool) { Index = 0; do { Lock = PoolDesc[Index].LockAddress; ExReleaseFastMutex((PFAST_MUTEX)Lock); Index += 1; } while (Index < ExpNumberOfPagedPools); KeLowerIrql(LockHandle); } else { ExReleaseSpinLock(&NonPagedPoolLock, LockHandle); } } if (ARGUMENT_PRESENT(ReturnLength)) { *ReturnLength = RequiredLength; } return Status; } #endif // DBG || (i386 && !FPO) #if i386 && !FPO USHORT ExGetPoolBackTraceIndex( IN PVOID P ) { CLONG i; PPOOL_HEADER Entry; if (NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) { if ( PAGE_ALIGNED(P) ) { for (i = 0; i < MAX_BIGPAGE_TABLE; i++) { if (PoolBigPageTable[i].NumberOfPages != 0 && PoolBigPageTable[i].Va == P ) { return PoolBigPageTable[i].AllocatorBackTraceIndex; } } } else { #if POOL_CACHE_SUPPORTED // // Align entry on pool allocation boundary. // if (((ULONG)P & POOL_CACHE_CHECK) == 0) { Entry = (PPOOL_HEADER)((ULONG)P - PoolCacheSize); } else { Entry = (PPOOL_HEADER)((PCH)P - POOL_OVERHEAD); } #else Entry = (PPOOL_HEADER)((PCH)P - POOL_OVERHEAD); #endif //POOL_CACHE_SUPPORTED if (Entry->AllocatorBackTraceIndex != 0 && Entry->AllocatorBackTraceIndex & POOL_BACKTRACEINDEX_PRESENT ) { return (Entry->AllocatorBackTraceIndex ^ POOL_BACKTRACEINDEX_PRESENT); } } } return 0; } #endif // i386 && !FPO