diff options
Diffstat (limited to 'private/ntos/ex/pool.c')
| -rw-r--r-- | private/ntos/ex/pool.c | 3074 |
1 files changed, 3074 insertions, 0 deletions
diff --git a/private/ntos/ex/pool.c b/private/ntos/ex/pool.c new file mode 100644 index 000000000..0d1f8aafd --- /dev/null +++ b/private/ntos/ex/pool.c @@ -0,0 +1,3074 @@ +/*++ + +Copyright (c) 1989-1994 Microsoft Corporation + +Module Name: + + pool.c + +Abstract: + + Thie module implements the NT executive pool allocator. + +Author: + + Mark Lucovsky 16-feb-1989 + Lou Perazzoli 31-Aug-1991 (change from binary buddy) + David N. Cutler (davec) 27-May-1994 + +Environment: + + kernel mode only + +Revision History: + +--*/ + +#include "exp.h" +#pragma hdrstop + +#undef ExAllocatePoolWithTag +#undef ExAllocatePool +#undef ExAllocatePoolWithQuota +#undef ExAllocatePoolWithQuotaTag +#undef ExFreePoolWithTag + +#if DBG +BOOLEAN ExpEchoPoolCalls; +#endif //DBG + +// +// Define forward referenced funtion prototypes. +// + +VOID +ExpInitializePoolDescriptor( + IN PPOOL_DESCRIPTOR PoolDescriptor, + IN POOL_TYPE PoolType, + IN ULONG PoolIndex, + IN ULONG Threshold, + IN PVOID PoolLock + ); + +NTSTATUS +ExpSnapShotPoolPages( + IN PVOID Address, + IN ULONG Size, + IN OUT PSYSTEM_POOL_INFORMATION PoolInformation, + IN OUT PSYSTEM_POOL_ENTRY *PoolEntryInfo, + IN ULONG Length, + IN OUT PULONG RequiredLength + ); + +#ifdef ALLOC_PRAGMA +#pragma alloc_text(INIT, InitializePool) +#pragma alloc_text(INIT, ExpInitializePoolDescriptor) +#if DBG +#pragma alloc_text(PAGELK, ExSnapShotPool) +#pragma alloc_text(PAGELK, ExpSnapShotPoolPages) +#endif // DBG +#endif + + +ULONG FirstPrint; +PPOOL_TRACKER_TABLE PoolTrackTable; +PPOOL_TRACKER_BIG_PAGES PoolBigPageTable; + +ULONG PoolHitTag = 0xffffff0f; + +USHORT +ExpInsertPoolTracker ( + ULONG Key, + ULONG Size, + POOL_TYPE PoolType + ); + +VOID +ExpRemovePoolTracker ( + ULONG Key, + ULONG Size, + POOL_TYPE PoolType + ); + +PPOOL_TRACKER_BIG_PAGES +ExpAddTagForBigPages ( + IN PVOID Va, + IN ULONG Key + ); + +ULONG +ExpFindAndRemoveTagBigPages ( + IN PVOID Va + ); + +PVOID +ExpAllocateStringRoutine( + IN ULONG NumberOfBytes + ) +{ + return ExAllocatePoolWithTag(PagedPool,NumberOfBytes,'grtS'); +} + +BOOLEAN +ExOkayToLockRoutine( + IN PVOID Lock + ); + +BOOLEAN +ExOkayToLockRoutine( + IN PVOID Lock + ) +{ + if (KeIsExecutingDpc()) { + return FALSE; + } else { + return TRUE; + } +} + +PRTL_ALLOCATE_STRING_ROUTINE RtlAllocateStringRoutine = ExpAllocateStringRoutine; +PRTL_FREE_STRING_ROUTINE RtlFreeStringRoutine = (PRTL_FREE_STRING_ROUTINE)ExFreePool; + +// +// Define macros to pack and unpack a pool index. +// + +#define PACK_POOL_INDEX(Index) ((UCHAR)(((Index) << 4) | (Index))) +#define UNPACK_POOL_INDEX(Index) ((ULONG)((Index) & 0xf)) + +// +// This structure exists in the pool descriptor structure. There is one of +// these for each pool block size +// +// The check list macro comes in three flavors there is one in the checked +// build that will assert if the doubly linked list is ill-formed. There +// is one that is noop for the shipping version and then there is one that +// can be enabled to check the list in the free build. +// + +#if DBG +#define CHECK_LIST(LINE,LIST,ENTRY) \ + ASSERT((LIST)->Flink->Blink == (LIST)); \ + ASSERT((LIST)->Blink->Flink == (LIST)); +#elif 1 +#define CHECK_LIST(LINE,LIST,ENTRY) {NOTHING;} +#else +#define CHECK_LIST(LINE,LIST,ENTRY) \ + if (((LIST)->Flink->Blink != (LIST)) || \ + ((LIST)->Blink->Flink != (LIST))) { \ + KeBugCheckEx (BAD_POOL_HEADER,3,(ULONG)LIST,LINE,(ULONG)ENTRY);\ + } +#endif //DBG + +#define CHECK_POOL_PAGE(PAGE) \ + { \ + PPOOL_HEADER P = (PPOOL_HEADER)(((ULONG)(PAGE)) & ~(PAGE_SIZE-1)); \ + ULONG SIZE, LSIZE; \ + BOOLEAN FOUND=FALSE; \ + LSIZE = 0; \ + SIZE = 0; \ + do { \ + if (P == (PPOOL_HEADER)PAGE) { \ + FOUND = TRUE; \ + } \ + if (P->PreviousSize != LSIZE) { \ + DbgPrint("POOL: Inconsistent size: ( %lx ) - %lx->%u != %u\n",\ + PAGE, P, P->PreviousSize, LSIZE); \ + DbgBreakPoint(); \ + } \ + LSIZE = P->BlockSize; \ + SIZE += LSIZE; \ + P = (PPOOL_HEADER)((PPOOL_BLOCK)P + LSIZE); \ + } while ((SIZE < (PAGE_SIZE / POOL_SMALLEST_BLOCK)) && \ + (PAGE_END(P) == FALSE)); \ + if ((PAGE_END(P) == FALSE) || (FOUND == FALSE)) { \ + DbgPrint("POOL: Inconsistent page: %lx\n",P); \ + DbgBreakPoint(); \ + } \ + } + + +// +// Define the number of paged pools. This value may be overridden at boot +// time. +// + +ULONG ExpNumberOfPagedPools = NUMBER_OF_PAGED_POOLS; + +// +// Pool descriptors for nonpaged pool and nonpaged pool must succeed are +// static. The pool descriptors for paged pool are dynamically allocated +// since there can be more than one paged pool. There is always one more +// paged pool descriptor than there are paged pools. This descriptor is +// used when a page allocation is done for a paged pool and is the first +// descriptor in the paged ppol descriptor array. +// + +POOL_DESCRIPTOR NonPagedPoolDescriptor; +POOL_DESCRIPTOR NonPagedPoolDescriptorMS; + +// +// The pool vector contains an array of pointers to pool descriptors. For +// nonpaged pool and nonpaged pool must success, this is a pointer to a +// single descriptor. For page pool, this is a pointer to an array of pool +// descriptors. The pointer to the paged pool descriptor is duplicated so +// if can be found easily by the kernel debugger. +// + +PPOOL_DESCRIPTOR PoolVector[NUMBER_OF_POOLS]; +PPOOL_DESCRIPTOR ExpPagedPoolDescriptor; + +extern KSPIN_LOCK NonPagedPoolLock; +extern KSPIN_LOCK PoolTraceLock; + +volatile ULONG ExpPoolIndex = 1; +KSPIN_LOCK ExpTaggedPoolLock; + +#if DBG +PSZ PoolTypeNames[MaxPoolType] = { + "NonPaged", + "Paged", + "NonPagedMustSucceed", + "NotUsed", + "NonPagedCacheAligned", + "PagedCacheAligned", + "NonPagedCacheAlignedMustS" + }; +#endif //DBG + +// +// Define paged and nonpaged pool lookaside descriptors. +// + +SMALL_POOL_LOOKASIDE ExpSmallNPagedPoolLookasideLists[POOL_SMALL_LISTS]; + +#if !defined(_PPC_) + +SMALL_POOL_LOOKASIDE ExpSmallPagedPoolLookasideLists[POOL_SMALL_LISTS]; + +#endif + +// +// Two routines to check for pool that has been altered after it was freed. +// + +#if DEADBEEF + +_inline +VOID +ExFillFreedPool ( + IN PVOID Buffer, + IN ULONG Size, + IN ULONG Tag + ) +{ + RtlFillMemoryUlong( Buffer, Size, Tag ); + + return; +} + +VOID +ExCheckFreedPool ( + IN ULONG LineNumber, + IN PPOOL_HEADER PoolHeader + ) +{ + ULONG MatchBytes; + ULONG i; + + MatchBytes = RtlCompareMemoryUlong( (PCHAR)PoolHeader + 0x10, + 0x20 - 0x10, + PoolHeader->PoolTag ); + if (MatchBytes != 0x20 - 0x10) { + DbgPrint("EX(%d): Freed pool block %lx modified at %lx after it was freed\n", + LineNumber, + ((PCHAR)PoolHeader), + ((PCHAR)PoolHeader) + 0x10 + MatchBytes); + + DbgBreakPoint(); + } + + for (i = 1; i < PoolHeader->BlockSize; i += 1) { + + MatchBytes = RtlCompareMemoryUlong( (((PCHAR)PoolHeader) + (0x20 * i)), + 0x20, + *((PULONG)(((PCHAR)PoolHeader) + (0x20 * i))) ); + + if (MatchBytes != 0x20) { + DbgPrint("EX(%d): Freed pool block %lx modified at %lx after it was freed\n", + LineNumber, + ((PCHAR)PoolHeader), + ((PCHAR)PoolHeader) + (0x20 * i) + MatchBytes); + + DbgBreakPoint(); + } + } + + return; +} + +#endif + + +// +// LOCK_POOL and LOCK_IF_PAGED_POOL are only used within this module. +// + +#define LOCK_POOL(PoolDesc, LockHandle) { \ + if ((PoolDesc->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { \ + ExAcquireSpinLock(&NonPagedPoolLock, &LockHandle); \ + } else { \ + ExAcquireFastMutex((PFAST_MUTEX)PoolDesc->LockAddress); \ + } \ +} + +#define LOCK_IF_PAGED_POOL(CheckType) \ + if (CheckType == PagedPool) { \ + ExAcquireFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); \ + } + +KIRQL +ExLockPool( + IN POOL_TYPE PoolType + ) + +/*++ + +Routine Description: + + This function locks the pool specified by pool type. + +Arguments: + + PoolType - Specifies the pool that should be locked. + +Return Value: + + The previous IRQL is returned as the function value. + +--*/ + +{ + + KIRQL OldIrql; + + // + // If the pool type is nonpaged, then use a spinlock to lock the + // pool. Otherwise, use a fast mutex to lock the pool. + // + + if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { + ExAcquireSpinLock(NonPagedPoolDescriptor.LockAddress, &OldIrql); + + } else { + ExAcquireFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); + OldIrql = (KIRQL)((PFAST_MUTEX)(PoolVector[PagedPool]->LockAddress))->OldIrql; + } + + return OldIrql; +} + + +// +// UNLOCK_POOL and UNLOCK_IF_PAGED_POOL are only used within this module. +// + +#define UNLOCK_POOL(PoolDesc, LockHandle) { \ + if ((PoolDesc->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { \ + ExReleaseSpinLock(&NonPagedPoolLock, (KIRQL)LockHandle); \ + } else { \ + ExReleaseFastMutex((PFAST_MUTEX)PoolDesc->LockAddress); \ + } \ +} + +#define UNLOCK_IF_PAGED_POOL(CheckType) \ + if (CheckType == PagedPool) { \ + ExReleaseFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); \ + } + +VOID +ExUnlockPool( + IN POOL_TYPE PoolType, + IN KIRQL LockHandle + ) + +/*++ + +Routine Description: + + This function unlocks the pool specified by pool type. + + +Arguments: + + PoolType - Specifies the pool that should be unlocked. + + LockHandle - Specifies the lock handle from a previous call to + ExLockPool. + +Return Value: + + None. + +--*/ + +{ + + // + // If the pool type is nonpaged, then use a spinlock to unlock the + // pool. Otherwise, use a fast mutex to unlock the pool. + // + + if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { + ExReleaseSpinLock(&NonPagedPoolLock, LockHandle); + + } else { + ExReleaseFastMutex((PFAST_MUTEX)PoolVector[PagedPool]->LockAddress); + } + + return; +} + +VOID +ExpInitializePoolDescriptor( + IN PPOOL_DESCRIPTOR PoolDescriptor, + IN POOL_TYPE PoolType, + IN ULONG PoolIndex, + IN ULONG Threshold, + IN PVOID PoolLock + ) + +/*++ + +Routine Description: + + This function initializes a pool descriptor. + +Arguments: + + PoolDescriptor - Supplies a pointer to the pool descriptor. + + PoolType - Supplies the type of the pool. + + PoolIndex - Supplies the pool descriptor index. + + Threshold - Supplies the threshold value for the specified pool. + + PoolLock - Supplies a point to the lock for the specified pool. + +Return Value: + + None. + +--*/ + +{ + + ULONG Index; + + // + // Initialize statistics fields, the pool type, the threshold value, + // and the lock address + // + + PoolDescriptor->PoolType = PoolType; + PoolDescriptor->PoolIndex = PoolIndex; + PoolDescriptor->RunningAllocs = 0; + PoolDescriptor->RunningDeAllocs = 0; + PoolDescriptor->TotalPages = 0; + PoolDescriptor->TotalBigPages = 0; + PoolDescriptor->Threshold = Threshold; + PoolDescriptor->LockAddress = PoolLock; + + // + // Initialize the allocation listheads. + // + + for (Index = 0; Index < POOL_LIST_HEADS; Index += 1) { + InitializeListHead(&PoolDescriptor->ListHeads[Index]); + } + + return; +} + +VOID +InitializePool( + IN POOL_TYPE PoolType, + IN ULONG Threshold + ) + +/*++ + +Routine Description: + + This procedure initializes a pool descriptor for the specified pool + type. Once initialized, the pool may be used for allocation and + deallocation. + + This function should be called once for each base pool type during + system initialization. + + Each pool descriptor contains an array of list heads for free + blocks. Each list head holds blocks which are a multiple of + the POOL_BLOCK_SIZE. The first element on the list [0] links + together free entries of size POOL_BLOCK_SIZE, the second element + [1] links together entries of POOL_BLOCK_SIZE * 2, the third + POOL_BLOCK_SIZE * 3, etc, up to the number of blocks which fit + into a page. + +Arguments: + + PoolType - Supplies the type of pool being initialized (e.g. + nonpaged pool, paged pool...). + + Threshold - Supplies the threshold value for the specified pool. + +Return Value: + + None. + +--*/ + +{ + + PPOOL_DESCRIPTOR Descriptor; + ULONG Index; + PFAST_MUTEX FastMutex; + ULONG Size; + + ASSERT((PoolType & MUST_SUCCEED_POOL_TYPE_MASK) == 0); + + + if (PoolType == NonPagedPool) { + + // + // Initialize nonpaged pools. + // + +#if !DBG + if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) { +#endif //!DBG + PoolTrackTable = MiAllocatePoolPages(NonPagedPool, + MAX_TRACKER_TABLE * + sizeof(POOL_TRACKER_TABLE)); + + RtlZeroMemory(PoolTrackTable, MAX_TRACKER_TABLE * sizeof(POOL_TRACKER_TABLE)); + PoolBigPageTable = MiAllocatePoolPages(NonPagedPool, + MAX_BIGPAGE_TABLE * + sizeof(POOL_TRACKER_BIG_PAGES)); + + RtlZeroMemory(PoolBigPageTable, MAX_BIGPAGE_TABLE * sizeof(POOL_TRACKER_BIG_PAGES)); +#if !DBG + } +#endif //!DBG + + // + // Initialize the spinlocks for nonpaged pool. + // + + KeInitializeSpinLock (&ExpTaggedPoolLock); + KeInitializeSpinLock(&NonPagedPoolLock); + KeInitializeSpinLock(&PoolTraceLock); + + // + // Initialize the nonpaged pool descriptor. + // + + PoolVector[NonPagedPool] = &NonPagedPoolDescriptor; + ExpInitializePoolDescriptor(&NonPagedPoolDescriptor, + NonPagedPool, + 0, + Threshold, + (PVOID)&NonPagedPoolLock); + + // + // Initialize the nonpaged must succeed pool descriptor. + // + + PoolVector[NonPagedPoolMustSucceed] = &NonPagedPoolDescriptorMS; + ExpInitializePoolDescriptor(&NonPagedPoolDescriptorMS, + NonPagedPoolMustSucceed, + 0, + 0, + (PVOID)&NonPagedPoolLock); + +#if DBG + if (MmSpecialPoolTag != 0) { + MmInitializeSpecialPool(); + } +#endif //DBG + + } else { + + // + // Allocate memory for the paged pool descriptors and fast mutexes. + // + + Size = (ExpNumberOfPagedPools + 1) * (sizeof(FAST_MUTEX) + sizeof(POOL_DESCRIPTOR)); + Descriptor = (PPOOL_DESCRIPTOR)ExAllocatePoolWithTag (NonPagedPoolMustSucceed, + Size, + 'looP'); + if (PoolTrackTable) { + ExpInsertPoolTracker('looP', + MAX_TRACKER_TABLE * sizeof(POOL_TRACKER_TABLE), + NonPagedPool); + + ExpInsertPoolTracker('looP', + MAX_BIGPAGE_TABLE * sizeof(POOL_TRACKER_BIG_PAGES), + NonPagedPool); + } + + FastMutex = (PFAST_MUTEX)(Descriptor + ExpNumberOfPagedPools + 1); + PoolVector[PagedPool] = Descriptor; + ExpPagedPoolDescriptor = Descriptor; + for (Index = 0; Index < (ExpNumberOfPagedPools + 1); Index += 1) { + ExInitializeFastMutex(FastMutex); + ExpInitializePoolDescriptor(Descriptor, + PagedPool, + Index, + Threshold, + (PVOID)FastMutex); + + Descriptor += 1; + FastMutex += 1; + } + } + + // + // The maximum cache alignment must be less than the size of the + // smallest pool block because the lower bits are being cleared + // in ExFreePool to find the entry's address. + // + +#if POOL_CACHE_SUPPORTED + + // + // Compute pool cache information. + // + + PoolCacheSize = HalGetDmaAlignmentRequirement(); + + ASSERT(PoolCacheSize >= POOL_OVERHEAD); + + PoolCacheOverhead = PoolCacheSize + PoolCacheSize - (sizeof(POOL_HEADER) + 1); + +#ifndef CHECK_POOL_TAIL + + PoolBuddyMax = + (POOL_PAGE_SIZE - (POOL_OVERHEAD + (3*POOL_SMALLEST_BLOCK) + 2*PoolCacheSize)); + +#else + + PoolBuddyMax = + (POOL_PAGE_SIZE - (POOL_OVERHEAD + 2*PoolCacheSize + (4*POOL_SMALLEST_BLOCK))); + +#endif // CHECK_POOL_TAIL + +#endif //POOL_CACHE_SUPPORTED + + return; +} + +#if DBG + + +VOID +ExpVerifyPool( + PPOOL_DESCRIPTOR PoolDescriptor + ) + +/*++ + +Routine Description: + + This function verifies the specified pool + +Arguments: + + PoolDesc - Supplies a pointer to a pool descriptor. + +Return Value: + + None. + +--*/ + +{ + + PLIST_ENTRY Entry; + ULONG Index; + PLIST_ENTRY ListHead; + ULONG Number; + PPOOL_HEADER PoolHeader; + + // + // Scan each of the allocation lists and perform the following checks: + // + // 1. Make sure each free block is in the correct list. + // + // 2. Make sure each free block is really free. + // + // 3. Make sure all the blocks in each page add up to a page. + // + // + + for (Index = 0; Index < POOL_LIST_HEADS; Index += 1) { + ListHead = &PoolDescriptor->ListHeads[Index]; + Entry = ListHead->Flink; + while (Entry != ListHead) { + PoolHeader = (PPOOL_HEADER)((PCHAR)Entry - POOL_OVERHEAD); + + // + // Assert that the pool block is not allocated. + // + + ASSERT(PoolHeader->PoolType == 0); + + ASSERT(PoolHeader->PoolIndex == PACK_POOL_INDEX(PoolDescriptor->PoolIndex)); + + // + // Assert that the pool block is in the correct list. + // + + Number = PoolHeader->BlockSize; + if (Number > POOL_SMALL_LISTS) { + Number = (Number >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1; + } + + ASSERT(Index == (Number - 1)); + + // + // Check to make sure the pool block is properly filled. + // + +#if DEADBEEF + + if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { + + // + // Make sure the pool block has not been modified after it was freed. + // + + ExCheckFreedPool( __LINE__, PoolHeader ); + + CHECK_POOL_PAGE(PoolHeader); + } + +#endif //DEADBEEF + + Entry = Entry->Flink; + } + } + + return; +} + +#else + +#define ExpVerifyPool(PoolDesc) + +#endif + +PVOID +ExAllocatePool( + IN POOL_TYPE PoolType, + IN ULONG NumberOfBytes + ) + +/*++ + +Routine Description: + + This function allocates a block of pool of the specified type and + returns a pointer to the allocated block. This function is used to + access both the page-aligned pools, and the list head entries (less than + a page) pools. + + If the number of bytes specifies a size that is too large to be + satisfied by the appropriate list, then the page-aligned + pool allocator is used. The allocated block will be page-aligned + and a page-sized multiple. + + Otherwise, the appropriate pool list entry is used. The allocated + block will be 64-bit aligned, but will not be page aligned. The + pool allocator calculates the smallest number of POOL_BLOCK_SIZE + that can be used to satisfy the request. If there are no blocks + available of this size, then a block of the next larger block size + is allocated and split. One piece is placed back into the pool, and + the other piece is used to satisfy the request. If the allocator + reaches the paged-sized block list, and nothing is there, the + page-aligned pool allocator is called. The page is split and added + to the pool... + +Arguments: + + PoolType - Supplies the type of pool to allocate. If the pool type + is one of the "MustSucceed" pool types, then this call will + always succeed and return a pointer to allocated pool. + Otherwise, if the system can not allocate the requested amount + of memory a NULL is returned. + + Valid pool types: + + NonPagedPool + PagedPool + NonPagedPoolMustSucceed, + NonPagedPoolCacheAligned + PagedPoolCacheAligned + NonPagedPoolCacheAlignedMustS + + NumberOfBytes - Supplies the number of bytes to allocate. + +Return Value: + + NULL - The PoolType is not one of the "MustSucceed" pool types, and + not enough pool exists to satisfy the request. + + NON-NULL - Returns a pointer to the allocated pool. + +--*/ + +{ + return ExAllocatePoolWithTag(PoolType, NumberOfBytes, 'enoN'); +} + + +PVOID +ExAllocatePoolWithTag( + IN POOL_TYPE PoolType, + IN ULONG NumberOfBytes, + IN ULONG Tag + ) + +/*++ + +Routine Description: + + This function allocates a block of pool of the specified type and + returns a pointer to the allocated block. This function is used to + access both the page-aligned pools and the list head entries (less + than a page) pools. + + If the number of bytes specifies a size that is too large to be + satisfied by the appropriate list, then the page-aligned pool + allocator is used. The allocated block will be page-aligned and a + page-sized multiple. + + Otherwise, the appropriate pool list entry is used. The allocated + block will be 64-bit aligned, but will not be page aligned. The + pool allocator calculates the smallest number of POOL_BLOCK_SIZE + that can be used to satisfy the request. If there are no blocks + available of this size, then a block of the next larger block size + is allocated and split. One piece is placed back into the pool, and + the other piece is used to satisfy the request. If the allocator + reaches the paged-sized block list, and nothing is there, the + page-aligned pool allocator is called. The page is split and added + to the pool. + +Arguments: + + PoolType - Supplies the type of pool to allocate. If the pool type + is one of the "MustSucceed" pool types, then this call will + always succeed and return a pointer to allocated pool. Otherwise, + if the system can not allocate the requested amount of memory a + NULL is returned. + + Valid pool types: + + NonPagedPool + PagedPool + NonPagedPoolMustSucceed, + NonPagedPoolCacheAligned + PagedPoolCacheAligned + NonPagedPoolCacheAlignedMustS + + NumberOfBytes - Supplies the number of bytes to allocate. + +Return Value: + + NULL - The PoolType is not one of the "MustSucceed" pool types, and + not enough pool exists to satisfy the request. + + NON-NULL - Returns a pointer to the allocated pool. + +--*/ + +{ + + PVOID Block; + PPOOL_HEADER Entry; + PSMALL_POOL_LOOKASIDE LookasideList; + PPOOL_HEADER NextEntry; + PPOOL_HEADER SplitEntry; + KIRQL LockHandle; + PPOOL_DESCRIPTOR PoolDesc; + PVOID Lock; + ULONG Index; + ULONG ListNumber; + ULONG NeededSize; + ULONG PoolIndex; + POOL_TYPE CheckType; + PLIST_ENTRY ListHead; + USHORT PoolTagHash; + PKPRCB Prcb; + POOL_TYPE NewPoolType; + +#if POOL_CACHE_SUPPORTED + ULONG CacheOverhead; +#else +#define CacheOverhead POOL_OVERHEAD +#endif + +#if DBG + VOID + CalculatePoolUtilization( + IN PPOOL_DESCRIPTOR PoolDesc, + IN ULONG BytesWanted + ); +#endif + + + ASSERT(NumberOfBytes != 0); + + // + // Isolate the base pool type and select a pool from which to allocate + // the specified block size. + // + + CheckType = PoolType & BASE_POOL_TYPE_MASK; + PoolDesc = PoolVector[CheckType]; + + // + // Check to determine if the requested block can be allocated from one + // of the pool lists or must be directed allocated from virtual memory. + // + + if (NumberOfBytes > POOL_BUDDY_MAX) { + + // + // The requested size is greater than the largest block maintained + // by allocation lists. + // + + ASSERT((PoolType & MUST_SUCCEED_POOL_TYPE_MASK) == 0); + + LOCK_POOL(PoolDesc, LockHandle); + + PoolDesc->RunningAllocs++; + Entry = (PPOOL_HEADER)MiAllocatePoolPages(CheckType, NumberOfBytes); + UNLOCK_POOL(PoolDesc, LockHandle); + if (Entry != NULL) { + PPOOL_TRACKER_BIG_PAGES p; + + PoolDesc->TotalBigPages += BYTES_TO_PAGES(NumberOfBytes); + if (PoolBigPageTable != FALSE) { + if (!(p = ExpAddTagForBigPages((PVOID)Entry, Tag))) { + Tag = ' GIB'; + } +#if DBG || (i386 && !FPO) + else + if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) && Entry) { + p->NumberOfPages = (USHORT)BYTES_TO_PAGES(NumberOfBytes); +#if i386 && !FPO + p->AllocatorBackTraceIndex = RtlLogStackBackTrace(); +#endif // i386 && !FPO + } +#endif // DBG || (i386 && !FPO) + + ExpInsertPoolTracker(Tag, ROUND_TO_PAGES(NumberOfBytes), PoolType); + } + + } else { + KdPrint(("EX: ExAllocatePool( %d ) returning NULL\n",NumberOfBytes)); + + if ((PoolType & POOL_RAISE_IF_ALLOCATION_FAILURE) != 0) { + ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); + } + } + +#if DBG + + if (ExpEchoPoolCalls){ + PVOID CallingAddress; + PVOID CallersCaller; + + DbgPrint("0x%lx EXALLOC: from %s size %d", + Entry, + PoolTypeNames[PoolType & MaxPoolType], + NumberOfBytes + ); + + RtlGetCallersAddress(&CallingAddress, &CallersCaller); + DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller); + } + +#endif //DBG + + return Entry; + + } else { + + // + // The requested size is less than of equal to the size of the + // maximum block maintained by the allocation lists. + // + +#if DBG + if ((MmSpecialPoolTag != 0) && (Tag == MmSpecialPoolTag)) { + return MmAllocateSpecialPool(NumberOfBytes, Tag); + } +#endif //DBG + + // + // If the request is for cache aligned memory adjust the number of + // bytes. + // + +#if POOL_CACHE_SUPPORTED //only compile for machines which have a nonzero value. + + CacheOverhead = POOL_OVERHEAD; + if (PoolType & CACHE_ALIGNED_POOL_TYPE_MASK) { + NumberOfBytes += PoolCacheOverhead; + CacheOverhead = PoolCacheSize; + } + +#endif //POOL_CACHE_SUPPORTED + + // + // Compute the Index of the listhead for blocks of the requested + // size. + // + + ListNumber = ((NumberOfBytes + POOL_OVERHEAD + (POOL_SMALLEST_BLOCK - 1)) >> POOL_BLOCK_SHIFT); + +#ifdef CHECK_POOL_TAIL + + ListNumber += 1; + +#endif // CHECK_POOL_TAIL + + + NeededSize = ListNumber; + if (ListNumber > POOL_SMALL_LISTS) { + ListNumber = (ListNumber >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 2; + } + + // + // If the pool type is paged, then pick a starting pool number and + // attempt to lock each paged pool in circular succession. Otherwise, + // lock the nonpaged pool as the same lock is used for both nonpaged + // and nonpage must succeed. + // + // N.B. The paged pool is selected in a round robin fashion using a + // simple counter. Note that the counter is incremented using a + // a noninterlocked sequence, but the pool index is never allowed + // to get out of range. + // + + Prcb = KeGetCurrentPrcb(); + if (CheckType == PagedPool) { + + // + // If the requested pool block is a small block, then attempt to + // allocate the requested pool from the per processor single entry + // lookaside list. If the allocation attempt fails, then select a + // pool to alocate from and allocate the block normally. + // + +#if !defined(CHECK_POOL_TAIL) && (DEADBEEF == 0) + + if (NeededSize <= POOL_SMALL_LISTS) { + +#if defined(_PPC_) + + if ((Entry = (PPOOL_HEADER)InterlockedExchange((PLONG)&Prcb->PagedFreeEntry[NeededSize - 1], + (LONG)NULL)) != NULL) { + + PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex); + Prcb->PagedPoolLookasideHits += 1; + +#else + + LookasideList = &ExpSmallPagedPoolLookasideLists[NeededSize - 1]; + LookasideList->TotalAllocates += 1; + if ((Isx86FeaturePresent(KF_CMPXCHG8B)) && + (Entry = (PPOOL_HEADER)ExInterlockedPopEntrySList(&LookasideList->SListHead, + &LookasideList->Lock)) != NULL) { + + + Entry -= 1; + LookasideList->AllocateHits += 1; + +#endif + NewPoolType = (PoolType & (BASE_POOL_TYPE_MASK | POOL_QUOTA_MASK)) + 1; +#if defined (_ALPHA_) + // + // On Alpha, Entry->PoolType cannot be updated without synchronizing with + // updates to Entry->PreviousSize. Otherwise, the lack of byte granularity + // can cause one update to get lost. In order to avoid an expensive interlocked + // operation, check PoolType to see if it really needs to be updated. + // + if (Entry->PoolType != NewPoolType) { + ULONG NewHeader; + ULONG OldHeader; + + NewHeader = Entry->Ulong1; + do { + OldHeader = NewHeader; + ((PPOOL_HEADER)(&NewHeader))->PoolType = NewPoolType; + NewHeader = (ULONG)InterlockedCompareExchange((PVOID *)&Entry->Ulong1, + (PVOID)NewHeader, + (PVOID)OldHeader); + + } while ( NewHeader != OldHeader ); + } +#else + Entry->PoolType = NewPoolType; +#endif + + if (PoolTrackTable != NULL) { + ExpInsertPoolTracker(Tag, + Entry->BlockSize << POOL_BLOCK_SHIFT, + PoolType); + + } + + Entry->PoolTag = Tag; + return (PUCHAR)Entry + CacheOverhead; + } + } + +#endif + + // + // If there is more than one paged pool, then attempt to find + // one that can be immediately locked. + // + + PoolIndex = 1; + if (ExpNumberOfPagedPools != PoolIndex) { + ExpPoolIndex += 1; + PoolIndex = ExpPoolIndex; + if (PoolIndex > ExpNumberOfPagedPools) { + PoolIndex = 1; + ExpPoolIndex = 1; + } + + Index = PoolIndex; + do { + Lock = PoolDesc[PoolIndex].LockAddress; + if (ExTryToAcquireFastMutex((PFAST_MUTEX)Lock) != FALSE) { + goto PoolLocked; + } + + PoolIndex += 1; + if (PoolIndex > ExpNumberOfPagedPools) { + PoolIndex = 1; + } + + } while (PoolIndex != Index); + } + + // + // None of the paged pools could be conditionally locked or there + // is only one paged pool. The first pool considered is picked as + // the victim to wait on. + // + + Lock = PoolDesc[PoolIndex].LockAddress; + ExAcquireFastMutex((PFAST_MUTEX)Lock); + +PoolLocked: + PoolDesc = &PoolDesc[PoolIndex]; + + } else { + + // + // If the requested pool block is a small block, then attempt to + // allocate the requested pool from the per processor single entry + // lookaside list. If the allocation attempt fails, then allocate + // the pool normally. + // + +#if !defined(CHECK_POOL_TAIL) && (DEADBEEF == 0) + + if (NeededSize <= POOL_SMALL_LISTS) { + LookasideList = &ExpSmallNPagedPoolLookasideLists[NeededSize - 1]; + LookasideList->TotalAllocates += 1; + if ((Entry = (PPOOL_HEADER)ExInterlockedPopEntrySList(&LookasideList->SListHead, + &LookasideList->Lock)) != NULL) { + + + Entry -= 1; + LookasideList->AllocateHits += 1; + NewPoolType = (PoolType & (BASE_POOL_TYPE_MASK | POOL_QUOTA_MASK)) + 1; +#if defined (_ALPHA_) + // + // On Alpha, Entry->PoolType cannot be updated without synchronizing with + // updates to Entry->PreviousSize. Otherwise, the lack of byte granularity + // can cause one update to get lost. In order to avoid an expensive interlocked + // operation, check PoolType to see if it really needs to be updated. + // + if (Entry->PoolType != NewPoolType) { + ULONG NewHeader; + ULONG OldHeader; + + NewHeader = Entry->Ulong1; + do { + OldHeader = NewHeader; + ((PPOOL_HEADER)(&NewHeader))->PoolType = NewPoolType; + NewHeader = (ULONG)InterlockedCompareExchange((PVOID *)&Entry->Ulong1, + (PVOID)NewHeader, + (PVOID)OldHeader); + + } while ( NewHeader != OldHeader ); + } +#else + Entry->PoolType = NewPoolType; +#endif + if (PoolTrackTable != NULL) { + ExpInsertPoolTracker(Tag, + Entry->BlockSize << POOL_BLOCK_SHIFT, + PoolType); + + } + + Entry->PoolTag = Tag; + return (PUCHAR)Entry + CacheOverhead; + } + } + +#endif + + PoolIndex = 0; + ExAcquireSpinLock(&NonPagedPoolLock, &LockHandle); + } + + ASSERT(PoolIndex == PoolDesc->PoolIndex); + + // + // The following code has an outer loop and an inner loop. + // + // The outer loop is utilized to repeat a nonpaged must succeed + // allocation if necessary. + // + // The inner loop is used to repeat an allocation attempt if there + // are no entries in any of the pool lists. + // + + PoolDesc->RunningAllocs += 1; + ListHead = &PoolDesc->ListHeads[ListNumber - 1]; + do { + + // + // Attempt to allocate the requested block from the current free + // blocks. + // + + do { + + // + // If the list is not empty, then allocate a block from the + // selected list. + // + + if (IsListEmpty(ListHead) == FALSE) { + + CHECK_LIST( __LINE__, ListHead, 0 ); + Block = RemoveHeadList(ListHead); + CHECK_LIST( __LINE__, ListHead, 0 ); + + Entry = (PPOOL_HEADER)((PCHAR)Block - POOL_OVERHEAD); + + ASSERT(Entry->BlockSize >= NeededSize); + + ASSERT(Entry->PoolIndex == PACK_POOL_INDEX(PoolIndex)); + + ASSERT(Entry->PoolType == 0); + + if (Entry->BlockSize != NeededSize) { + + // + // The selected block is larger than the allocation + // request. Split the block and insert the remaining + // fragment in the appropriate list. + // + // If the entry is at the start of a page, then take + // the allocation from the front of the block so as + // to minimize fragmentation. Otherwise, take the + // allocation from the end of the block which may + // also reduce fragmentation is the block is at the + // end of a page. + // + + if (Entry->PreviousSize == 0) { + + // + // The entry is at the start of a page. + // + + SplitEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + NeededSize); + SplitEntry->BlockSize = Entry->BlockSize - (UCHAR)NeededSize; + SplitEntry->PreviousSize = (UCHAR)NeededSize; + + // + // If the allocated block is not at the end of a + // page, then adjust the size of the next block. + // + + NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)SplitEntry + SplitEntry->BlockSize); + if (PAGE_END(NextEntry) == FALSE) { + NextEntry->PreviousSize = SplitEntry->BlockSize; + } + + } else { + + // + // The entry is not at the start of a page. + // + + SplitEntry = Entry; + Entry->BlockSize -= (UCHAR)NeededSize; + Entry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + Entry->BlockSize); + Entry->PreviousSize = SplitEntry->BlockSize; + + // + // If the allocated block is not at the end of a + // page, then adjust the size of the next block. + // + + NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + NeededSize); + if (PAGE_END(NextEntry) == FALSE) { + NextEntry->PreviousSize = (UCHAR)NeededSize; + } + } + + // + // Set the size of the allocated entry, clear the pool + // type of the split entry, set the index of the split + // entry, and insert the split entry in the appropriate + // free list. + // + + Entry->BlockSize = (UCHAR)NeededSize; + Entry->PoolIndex = PACK_POOL_INDEX(PoolIndex); + SplitEntry->PoolType = 0; + SplitEntry->PoolIndex = PACK_POOL_INDEX(PoolIndex); + Index = SplitEntry->BlockSize; + if (Index > POOL_SMALL_LISTS) { + Index = (Index >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1; + } + + InsertTailList(&PoolDesc->ListHeads[Index - 1], + ((PLIST_ENTRY)((PCHAR)SplitEntry + POOL_OVERHEAD))); + } + + Entry->PoolType = (PoolType & (BASE_POOL_TYPE_MASK | POOL_QUOTA_MASK)) + 1; + +#if DEADBEEF + + if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { + + // + // Make sure the pool block has not been modified after it was freed + // and then fill it in with allocated tags. + // + + ExCheckFreedPool( __LINE__, Entry ); + + RtlFillMemoryUlong( (PCHAR)Entry + POOL_OVERHEAD, + (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD, + ALLOCATED_POOL ); + + CHECK_POOL_PAGE(Entry); + } + +#endif //DEADBEEF + + if (PoolTrackTable != NULL) { + PoolTagHash = ExpInsertPoolTracker(Tag, + Entry->BlockSize << POOL_BLOCK_SHIFT, + PoolType); + + } + + UNLOCK_POOL(PoolDesc, LockHandle); + + Entry->PoolTag = Tag; + +#if i386 && !FPO + if (NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) { + USHORT AllocatorBackTraceIndex; + + if (AllocatorBackTraceIndex = RtlLogStackBackTrace()) { + Entry->AllocatorBackTraceIndex = AllocatorBackTraceIndex | + POOL_BACKTRACEINDEX_PRESENT; + Entry->PoolTagHash = PoolTagHash; + } + } +#endif // i386 && !FPO + +#if DBG + if (ExpEchoPoolCalls){ + PVOID CallingAddress; + PVOID CallersCaller; + + DbgPrint("0x%lx EXALLOC: from %s size %d", + ((PCH)Entry + POOL_OVERHEAD), + PoolTypeNames[PoolType & MaxPoolType], + NumberOfBytes + ); + + RtlGetCallersAddress(&CallingAddress, &CallersCaller); + DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller); + } +#endif + + return (PCHAR)Entry + CacheOverhead; + + } else { + ListHead += 1; + } + + } while (ListHead != &PoolDesc->ListHeads[POOL_LIST_HEADS]); + + // + // A block of the desired size does not exist and there are + // no large blocks that can be split to satify the allocation. + // Attempt to expand the pool by allocating another page to be + // added to the pool. + // + // If the pool type is paged pool, then the paged pool page lock + // must be held during the allocation of the pool pages. + // + + LOCK_IF_PAGED_POOL(CheckType); + + Entry = (PPOOL_HEADER)MiAllocatePoolPages(CheckType, PAGE_SIZE); + + UNLOCK_IF_PAGED_POOL(CheckType); + + if (Entry == NULL) { + if ((PoolType & MUST_SUCCEED_POOL_TYPE_MASK) != 0) { + + // + // Must succeed pool was requested. Reset the the type, + // the pool descriptor address, and continue the search. + // + + CheckType = NonPagedPoolMustSucceed; + PoolDesc = PoolVector[NonPagedPoolMustSucceed]; + ListHead = &PoolDesc->ListHeads[ListNumber - 1]; + continue; + + } else { + + // + // No more pool of the specified type is available. + // + + KdPrint(("EX: ExAllocatePool( %d ) returning NULL\n", + NumberOfBytes)); + + UNLOCK_POOL(PoolDesc, LockHandle); + + if ((PoolType & POOL_RAISE_IF_ALLOCATION_FAILURE) != 0) { + ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); + } + + return NULL; + } + } + +#if DEADBEEF + + if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { + + // + // Fill in the entire page with freed pool tags. + // + + ExFillFreedPool( (PCHAR)Entry + POOL_OVERHEAD, + PAGE_SIZE - POOL_OVERHEAD, + FREED_POOL ); + + Entry->PoolTag = FREED_POOL; + } + +#endif + + // + // Insert the allocate page in the last allocation list. + // + + PoolDesc->TotalPages += 1; + Entry->PoolType = 0; + Entry->PoolIndex = PACK_POOL_INDEX(PoolIndex); + + // + // N.B. A byte is used to store the block size in units of the + // smallest block size. Therefore, if the number of small + // blocks in the page is greater than 255, the block size + // is set to 255. + // + + if ((PAGE_SIZE / POOL_SMALLEST_BLOCK) > 255) { + Entry->BlockSize = 255; + + } else { + Entry->BlockSize = PAGE_SIZE / POOL_SMALLEST_BLOCK; + } + + Entry->PreviousSize = 0; + ListHead = &PoolDesc->ListHeads[POOL_LIST_HEADS - 1]; + InsertHeadList(ListHead, ((PLIST_ENTRY)((PCHAR)Entry + POOL_OVERHEAD))); + } while (TRUE); + } +} + +USHORT +ExpInsertPoolTracker ( + ULONG Key, + ULONG Size, + POOL_TYPE PoolType + ) + +/*++ + +Routine Description: + + This function insert a pool tag in the tag table and increments the + number of allocates and updates the total allocation size. + +Arguments: + + Key - Supplies the key value used to locate a matching entry in the + tag table. + + Size - Supplies the allocation size. + + PoolType - Supplies the pool type. + +Return Value: + + The tag index is returned as the function value. + +--*/ + +{ + + USHORT Result; + ULONG Hash; + ULONG Index; + KIRQL OldIrql; + + // + // Ignore protected pool bit except for returned hash index + // + + if (Key & PROTECTED_POOL) { + Key &= ~PROTECTED_POOL; + Result = (USHORT)(PROTECTED_POOL >> 16); + } else { + Result = 0; + } + + if (Key == PoolHitTag) { + DbgBreakPoint(); + } + + // + // Compute hash index and search for pool tag. + // + + Hash = ((40543*((((((((PUCHAR)&Key)[0]<<2)^((PUCHAR)&Key)[1])<<2)^((PUCHAR)&Key)[2])<<2)^((PUCHAR)&Key)[3]))>>2) & TRACKER_TABLE_MASK; + Index = Hash; + ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql); + do { + if ((PoolTrackTable[Hash].Key == Key) || + (PoolTrackTable[Hash].Key == 0)) { + goto EntryFound; + } + + Hash = (Hash + 1) & TRACKER_TABLE_MASK; + } while (Hash != Index); + + // + // No matching entry and no free entry was found. + // + + Hash = MAX_TRACKER_TABLE - 1; + + // + // Update pool tracker table entry. + // + +EntryFound: + PoolTrackTable[Hash].Key = Key; + if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) { + PoolTrackTable[Hash].PagedAllocs += 1; + PoolTrackTable[Hash].PagedBytes += Size; + + } else { + PoolTrackTable[Hash].NonPagedAllocs += 1; + PoolTrackTable[Hash].NonPagedBytes += Size; + } + + ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); + return (USHORT)Hash | Result; +} + + +VOID +ExpRemovePoolTracker ( + ULONG Key, + ULONG Size, + POOL_TYPE PoolType + ) + +/*++ + +Routine Description: + + This function increments the number of frees and updates the total + allocation size. + +Arguments: + + Key - Supplies the key value used to locate a matching entry in the + tag table. + + Size - Supplies the allocation size. + + PoolType - Supplies the pool type. + +Return Value: + + None. + +--*/ + +{ + + ULONG Hash; + ULONG Index; + KIRQL OldIrql; + + // + // Ignore protected pool bit + // + + Key &= ~PROTECTED_POOL; + if (Key == PoolHitTag) { + DbgBreakPoint(); + } + + // + // Compute hash index and search for pool tag. + // + + Hash = ((40543*((((((((PUCHAR)&Key)[0]<<2)^((PUCHAR)&Key)[1])<<2)^((PUCHAR)&Key)[2])<<2)^((PUCHAR)&Key)[3]))>>2) & TRACKER_TABLE_MASK; + Index = Hash; + ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql); + do { + if (PoolTrackTable[Hash].Key == Key) { + goto EntryFound; + } + + if (PoolTrackTable[Hash].Key == 0) { + KdPrint(("POOL: Unable to find tracker %lx, table corrupted\n", Key)); + goto ExitRoutine; + } + + + Hash = (Hash + 1) & TRACKER_TABLE_MASK; + } while (Hash != Index); + + // + // No matching entry and no free entry was found. + // + + Hash = MAX_TRACKER_TABLE - 1; + + // + // Update pool tracker table entry. + // + +EntryFound: + if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) { + PoolTrackTable[Hash].PagedBytes -= Size; + PoolTrackTable[Hash].PagedFrees += 1; + + } else { + PoolTrackTable[Hash].NonPagedBytes -= Size; + PoolTrackTable[Hash].NonPagedFrees += 1; + } + +ExitRoutine: + ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); + return; +} + + +PPOOL_TRACKER_BIG_PAGES +ExpAddTagForBigPages ( + IN PVOID Va, + IN ULONG Key + ) +{ + ULONG Hash; + BOOLEAN Inserted = TRUE; + KIRQL OldIrql; + + Hash = ((ULONG)Va >> PAGE_SHIFT) & BIGPAGE_TABLE_MASK; + ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql); + while (PoolBigPageTable[Hash].Va != NULL) { + Hash += 1; + if (Hash > MAX_BIGPAGE_TABLE) { + if (!Inserted) { + if (!FirstPrint) { + KdPrint(("POOL:unable to insert big page slot %lx\n",Key)); + FirstPrint = TRUE; + } + + ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); + return NULL; + } + + Hash = 0; + Inserted = FALSE; + } + } + + PoolBigPageTable[Hash].Va = Va; + PoolBigPageTable[Hash].Key = Key; + ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); + return &PoolBigPageTable[Hash]; +} + + +ULONG +ExpFindAndRemoveTagBigPages ( + IN PVOID Va + ) + +{ + + ULONG Hash; + BOOLEAN Inserted = TRUE; + KIRQL OldIrql; + ULONG ReturnKey; + + Hash = ((ULONG)Va >> PAGE_SHIFT) & BIGPAGE_TABLE_MASK; + ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql); + while (PoolBigPageTable[Hash].Va != Va) { + Hash += 1; + if (Hash > MAX_BIGPAGE_TABLE) { + if (!Inserted) { + if (!FirstPrint) { + KdPrint(("POOL:unable to find big page slot %lx\n",Va)); + FirstPrint = TRUE; + } + + ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); + return ' GIB'; + } + + Hash = 0; + Inserted = FALSE; + } + } + + PoolBigPageTable[Hash].Va = NULL; + ReturnKey = PoolBigPageTable[Hash].Key; + ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); + return ReturnKey; +} + + +ULONG +ExpAllocatePoolWithQuotaHandler( + IN NTSTATUS ExceptionCode, + IN PVOID PoolAddress, + IN BOOLEAN ContinueSearch + ) + +/*++ + +Routine Description: + + This function is called when an exception occurs in ExFreePool + while quota is being charged to a process. + + Its function is to deallocate the pool block and continue the search + for an exception handler. + +Arguments: + + ExceptionCode - Supplies the exception code that caused this + function to be entered. + + PoolAddress - Supplies the address of a pool block that needs to be + deallocated. + + ContinueSearch - Supplies a value that if TRUE causes the exception + search to continue. This is used in allocate pool with quota + calls that do not contain the pool quota mask bit set. + +Return Value: + + EXCEPTION_CONTINUE_SEARCH - The exception should be propagated to the + caller of ExAllocatePoolWithQuota. + +--*/ + +{ + if ( PoolAddress ) { + ASSERT(ExceptionCode == STATUS_QUOTA_EXCEEDED); + ExFreePool(PoolAddress); + + } else { + ASSERT(ExceptionCode == STATUS_INSUFFICIENT_RESOURCES); + } + + return ContinueSearch ? EXCEPTION_CONTINUE_SEARCH : EXCEPTION_EXECUTE_HANDLER; +} + +PVOID +ExAllocatePoolWithQuota( + IN POOL_TYPE PoolType, + IN ULONG NumberOfBytes + ) + +/*++ + +Routine Description: + + This function allocates a block of pool of the specified type, + returns a pointer to the allocated block, and if the binary buddy + allocator was used to satisfy the request, charges pool quota to the + current process. This function is used to access both the + page-aligned pools, and the binary buddy. + + If the number of bytes specifies a size that is too large to be + satisfied by the appropriate binary buddy pool, then the + page-aligned pool allocator is used. The allocated block will be + page-aligned and a page-sized multiple. No quota is charged to the + current process if this is the case. + + Otherwise, the appropriate binary buddy pool is used. The allocated + block will be 64-bit aligned, but will not be page aligned. After + the allocation completes, an attempt will be made to charge pool + quota (of the appropriate type) to the current process object. If + the quota charge succeeds, then the pool block's header is adjusted + to point to the current process. The process object is not + dereferenced until the pool is deallocated and the appropriate + amount of quota is returned to the process. Otherwise, the pool is + deallocated, a "quota exceeded" condition is raised. + +Arguments: + + PoolType - Supplies the type of pool to allocate. If the pool type + is one of the "MustSucceed" pool types and sufficient quota + exists, then this call will always succeed and return a pointer + to allocated pool. Otherwise, if the system can not allocate + the requested amount of memory a STATUS_INSUFFICIENT_RESOURCES + status is raised. + + NumberOfBytes - Supplies the number of bytes to allocate. + +Return Value: + + NON-NULL - Returns a pointer to the allocated pool. + + Unspecified - If insuffient quota exists to complete the pool + allocation, the return value is unspecified. + +--*/ + +{ + return (ExAllocatePoolWithQuotaTag (PoolType, NumberOfBytes, 'enoN')); +} + + +PVOID +ExAllocatePoolWithQuotaTag( + IN POOL_TYPE PoolType, + IN ULONG NumberOfBytes, + IN ULONG Tag + ) + +/*++ + +Routine Description: + + This function allocates a block of pool of the specified type, + returns a pointer to the allocated block, and if the binary buddy + allocator was used to satisfy the request, charges pool quota to the + current process. This function is used to access both the + page-aligned pools, and the binary buddy. + + If the number of bytes specifies a size that is too large to be + satisfied by the appropriate binary buddy pool, then the + page-aligned pool allocator is used. The allocated block will be + page-aligned and a page-sized multiple. No quota is charged to the + current process if this is the case. + + Otherwise, the appropriate binary buddy pool is used. The allocated + block will be 64-bit aligned, but will not be page aligned. After + the allocation completes, an attempt will be made to charge pool + quota (of the appropriate type) to the current process object. If + the quota charge succeeds, then the pool block's header is adjusted + to point to the current process. The process object is not + dereferenced until the pool is deallocated and the appropriate + amount of quota is returned to the process. Otherwise, the pool is + deallocated, a "quota exceeded" condition is raised. + +Arguments: + + PoolType - Supplies the type of pool to allocate. If the pool type + is one of the "MustSucceed" pool types and sufficient quota + exists, then this call will always succeed and return a pointer + to allocated pool. Otherwise, if the system can not allocate + the requested amount of memory a STATUS_INSUFFICIENT_RESOURCES + status is raised. + + NumberOfBytes - Supplies the number of bytes to allocate. + +Return Value: + + NON-NULL - Returns a pointer to the allocated pool. + + Unspecified - If insuffient quota exists to complete the pool + allocation, the return value is unspecified. + +--*/ + +{ + PVOID p; + PEPROCESS Process; + PPOOL_HEADER Entry; + BOOLEAN IgnoreQuota = FALSE; + BOOLEAN RaiseOnQuotaFailure = TRUE; + + if ( PoolType & POOL_QUOTA_FAIL_INSTEAD_OF_RAISE ) { + RaiseOnQuotaFailure = FALSE; + PoolType &= ~POOL_QUOTA_FAIL_INSTEAD_OF_RAISE; + } + + if (PoolTrackTable +#if i386 && !FPO + || (NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) +#endif // i386 && !FPO + ) { + IgnoreQuota = TRUE; + } else { + PoolType = (POOL_TYPE)((UCHAR)PoolType + POOL_QUOTA_MASK); + } + + p = ExAllocatePoolWithTag(PoolType, NumberOfBytes, Tag); + + // + // Note - NULL is page aligned. + // + + if (!PAGE_ALIGNED(p) && !IgnoreQuota ) { + +#if POOL_CACHE_SUPPORTED + + // + // Align entry on pool allocation boundary. + // + + if (((ULONG)p & POOL_CACHE_CHECK) == 0) { + Entry = (PPOOL_HEADER)((ULONG)p - PoolCacheSize); + } else { + Entry = (PPOOL_HEADER)((PCH)p - POOL_OVERHEAD); + } + +#else + Entry = (PPOOL_HEADER)((PCH)p - POOL_OVERHEAD); +#endif //POOL_CACHE_SUPPORTED + + Process = PsGetCurrentProcess(); + + // + // Catch exception and back out allocation if necessary + // + + try { + + Entry->ProcessBilled = NULL; + + if ( Process != PsInitialSystemProcess ) { + PsChargePoolQuota(Process, + PoolType & BASE_POOL_TYPE_MASK, + (ULONG)(Entry->BlockSize << POOL_BLOCK_SHIFT)); + + ObReferenceObject(Process); + Entry->ProcessBilled = Process; + } + + } except ( ExpAllocatePoolWithQuotaHandler(GetExceptionCode(),p,RaiseOnQuotaFailure)) { + if ( RaiseOnQuotaFailure ) { + KeBugCheck(GetExceptionCode()); + } + else { + p = NULL; + } + } + + } else { + if ( !p && RaiseOnQuotaFailure ) { + ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); + } + } + + return p; +} + +VOID +ExFreePool( + IN PVOID P + ) +{ +#ifdef POOL_TAGGING + ExFreePoolWithTag(P, 0); + return; +} + +VOID +ExFreePoolWithTag( + IN PVOID P, + IN ULONG TagToFree + ) +{ +#else + ULONG TagToFree = 0; +#endif + +/*++ + +Routine Description: + + This function deallocates a block of pool. This function is used to + deallocate to both the page aligned pools, and the buddy (less than + a page) pools. + + If the address of the block being deallocated is page-aligned, then + the page-aliged pool deallocator is used. + + Otherwise, the binary buddy pool deallocator is used. Deallocation + looks at the allocated block's pool header to determine the pool + type and block size being deallocated. If the pool was allocated + using ExAllocatePoolWithQuota, then after the deallocation is + complete, the appropriate process's pool quota is adjusted to reflect + the deallocation, and the process object is dereferenced. + +Arguments: + + P - Supplies the address of the block of pool being deallocated. + +Return Value: + + None. + +--*/ + + POOL_TYPE CheckType; + PPOOL_HEADER Entry; + ULONG Index; + KIRQL LockHandle; + PSMALL_POOL_LOOKASIDE LookasideList; + PPOOL_HEADER NextEntry; + ULONG PoolIndex; + POOL_TYPE PoolType; + PPOOL_DESCRIPTOR PoolDesc; + PEPROCESS ProcessBilled = NULL; + PKPRCB Prcb; + BOOLEAN Combined; + ULONG BigPages; + ULONG Tag; + +#if DBG + BOOLEAN + ExpCheckForResource( + IN PVOID p, + IN ULONG Size + ); +#endif //DBG + +#if DBG + if ((P > MmSpecialPoolStart) && (P < MmSpecialPoolEnd)) { + MmFreeSpecialPool (P); + return; + } +#endif //DBG + + // + // If entry is page aligned, then call free block to the page aligned + // pool. Otherwise, free the block to the allocation lists. + // + + if (PAGE_ALIGNED(P)) { + + PoolType = MmDeterminePoolType(P); + CheckType = PoolType & BASE_POOL_TYPE_MASK; + PoolDesc = PoolVector[PoolType]; + + LOCK_POOL(PoolDesc, LockHandle); + + PoolDesc->RunningDeAllocs++; + BigPages = MiFreePoolPages(P); + PoolDesc->TotalBigPages -= BigPages; + +#if DBG + + // + // Check is an ERESOURCE is current active in this memory block. + // + + ExpCheckForResource(P, BigPages * PAGE_SIZE); + +#endif // DBG + + UNLOCK_POOL(PoolDesc, LockHandle); + + if (PoolTrackTable != NULL) { + Tag = ExpFindAndRemoveTagBigPages(P); + if (Tag & PROTECTED_POOL) { + Tag &= ~PROTECTED_POOL; + TagToFree &= ~PROTECTED_POOL; + if (Tag != TagToFree) { + DbgPrint( "EX: Invalid attempt to free protected pool block %x (%c%c%c%c)\n", + P, + Tag, + Tag >> 8, + Tag >> 16, + Tag >> 24 + ); + DbgBreakPoint(); + } + } + + ExpRemovePoolTracker(Tag, + BigPages * PAGE_SIZE, + PoolType); + } + +#if DBG + + if (ExpEchoPoolCalls){ + PVOID CallingAddress; + PVOID CallersCaller; + + DbgPrint("0x%lx EXDEALLOC: from %s", P, PoolTypeNames[PoolType]); + RtlGetCallersAddress(&CallingAddress, &CallersCaller); + DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller); + } + +#endif //DBG + + } else { + + // + // Align the entry address to a pool allocation boundary. + // + +#if POOL_CACHE_SUPPORTED + + if (((ULONG)P & POOL_CACHE_CHECK) == 0) { + Entry = (PPOOL_HEADER)((ULONG)P - PoolCacheSize); + + } else { + Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD); + } + +#else + + Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD); + +#endif //POOL_CACHE_SUPPORTED + + PoolType = (Entry->PoolType & POOL_TYPE_MASK) - 1; + CheckType = PoolType & BASE_POOL_TYPE_MASK; + +#if DBG + + // + // Check if an ERESOURCE is currently active in this memory block. + // + + ExpCheckForResource(Entry, (ULONG)(Entry->BlockSize << POOL_BLOCK_SHIFT)); + + // + // Check if the pool type field is defined correctly. + // + + if (Entry->PoolType == 0) { + DbgPrint("EX: Invalid pool header 0x%lx 0x%lx\n",P,*(PULONG)P); + KeBugCheckEx(BAD_POOL_HEADER, 1, (ULONG)Entry, *(PULONG)Entry, 0); + } + + // + // Check if the pool index field is defined correctly. + // + + if ((CheckType == NonPagedPool) && (Entry->PoolIndex != 0)) { + DbgPrint("EX: Invalid pool header 0x%lx 0x%lx\n",Entry,*(PULONG)Entry); + KeBugCheckEx(BAD_POOL_HEADER, 2, (ULONG)Entry, *(PULONG)Entry, 0); + + } else if (((CheckType == PagedPool) && (Entry->PoolIndex == 0)) || + (((Entry->PoolIndex >> 4) & 0xf) != (Entry->PoolIndex & 0xf))) { + DbgPrint("EX: Invalid pool header 0x%lx 0x%lx\n",Entry,*(PULONG)Entry); + KeBugCheckEx(BAD_POOL_HEADER, 4, (ULONG)Entry, *(PULONG)Entry, 0); + } + +#endif // DBG + + if (Entry->PoolType & POOL_QUOTA_MASK) { + if (PoolTrackTable == NULL) { + ProcessBilled = Entry->ProcessBilled; + Entry->PoolTag = 'atoQ'; + } + } + +#if DBG + + if (ExpEchoPoolCalls){ + PVOID CallingAddress; + PVOID CallersCaller; + + DbgPrint("0x%lx EXDEALLOC: from %s", P, PoolTypeNames[PoolType]); + RtlGetCallersAddress(&CallingAddress, &CallersCaller); + DbgPrint(" Callers:%lx, %lx\n", CallingAddress, CallersCaller); + } + +#endif + +#ifdef CHECK_POOL_TAIL + + if (NtGlobalFlag & FLG_POOL_ENABLE_TAIL_CHECK) { + PCHAR PoolBlock; + ULONG CountBytes; + ULONG CountBytesEqual; + + PoolBlock = (PCHAR)(((PPOOL_BLOCK)Entry + Entry->BlockSize)) - POOL_SMALLEST_BLOCK; + CountBytes = POOL_SMALLEST_BLOCK; + CountBytesEqual = RtlCompareMemoryUlong(PoolBlock, + CountBytes, + ALLOCATED_POOL); + + if (CountBytesEqual != CountBytes) { + DbgPrint("EX: Pool block at %lx modified at %lx past requested size of %lx\n", + PoolBlock, + PoolBlock + CountBytesEqual, + (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_SMALLEST_BLOCK - POOL_OVERHEAD); + + DbgBreakPoint(); + } + } + +#endif //CHECK_POOL_TAIL + +#if i386 && !FPO + if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) && + PoolTrackTable != NULL && + Entry->AllocatorBackTraceIndex != 0 && + Entry->AllocatorBackTraceIndex & POOL_BACKTRACEINDEX_PRESENT) { + if (Entry->PoolTagHash & (PROTECTED_POOL >> 16)) { + Entry->PoolTagHash &= ~(PROTECTED_POOL >> 16); + Tag = PROTECTED_POOL; + } else { + Tag = 0; + } + Entry->PoolTag = Tag | PoolTrackTable[Entry->PoolTagHash].Key; + } +#endif // i386 && !FPO + + // + // If pool tagging is enabled, then update the pool tracking database. + // Otherwise, check to determine if quota was charged when the pool + // block was allocated. + // + + if (PoolTrackTable != NULL) { + Tag = Entry->PoolTag; + if (Tag & PROTECTED_POOL) { + Tag &= ~PROTECTED_POOL; + TagToFree &= ~PROTECTED_POOL; + if (Tag != TagToFree) { + DbgPrint( "EX: Invalid attempt to free protected pool block %x (%c%c%c%c)\n", + P, + Tag, + Tag >> 8, + Tag >> 16, + Tag >> 24 + ); + DbgBreakPoint(); + } + } + ExpRemovePoolTracker(Tag, + Entry->BlockSize << POOL_BLOCK_SHIFT , + PoolType); + + } else if (ProcessBilled != NULL) { + PsReturnPoolQuota(ProcessBilled, + PoolType & BASE_POOL_TYPE_MASK, + (ULONG)Entry->BlockSize << POOL_BLOCK_SHIFT); + ObDereferenceObject(ProcessBilled); + } + + // + // If the pool block is a small block, then attempt to free the block + // to the single entry lookaside list. If the free atempts fails, then + // free the block by merging it back into the pool data structures. + // + + PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex); + PoolDesc = PoolVector[PoolType]; + Index = Entry->BlockSize; + if (Index > POOL_SMALL_LISTS) { + Index = (Index >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1; + + } else { + + // + // Attempt to free the small block to the per rpocessor single + // entry lookaside list. + // + +#if !defined(CHECK_POOL_TAIL) && (DEADBEEF == 0) + + Prcb = KeGetCurrentPrcb(); + if (CheckType == PagedPool) { + +#if defined(_PPC_) + + if ((Entry = (PPOOL_HEADER)InterlockedExchange((PLONG)&Prcb->PagedFreeEntry[Index - 1], + (LONG)Entry)) == NULL) { + +#else + + LookasideList = &ExpSmallPagedPoolLookasideLists[Index - 1]; + LookasideList->TotalFrees += 1; + if ((Isx86FeaturePresent(KF_CMPXCHG8B)) && + (ExQueryDepthSList(&LookasideList->SListHead) < LookasideList->Depth)) { + LookasideList->FreeHits += 1; + Entry += 1; + ExInterlockedPushEntrySList(&LookasideList->SListHead, + (PSINGLE_LIST_ENTRY)Entry, + &LookasideList->Lock); + +#endif + + return; + } + + PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex); + + } else { + LookasideList = &ExpSmallNPagedPoolLookasideLists[Index - 1]; + LookasideList->TotalFrees += 1; + if (ExQueryDepthSList(&LookasideList->SListHead) < LookasideList->Depth) { + LookasideList->FreeHits += 1; + Entry += 1; + ExInterlockedPushEntrySList(&LookasideList->SListHead, + (PSINGLE_LIST_ENTRY)Entry, + &LookasideList->Lock); + + return; + } + + PoolIndex = UNPACK_POOL_INDEX(Entry->PoolIndex); + } +#endif + + } + + // + // If the pool type is paged pool, then get the appropriate pool + // descriptor address. + // + + if (CheckType == PagedPool) { + PoolDesc = &PoolDesc[PoolIndex]; + } + + ASSERT(PoolIndex == PoolDesc->PoolIndex); + + LOCK_POOL(PoolDesc, LockHandle); + +#if DEADBEEF + if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { + CHECK_POOL_PAGE(Entry); + } +#endif + + PoolDesc->RunningDeAllocs += 1; + +#if DEADBEEF + if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { + + // + // Fill in the block with its previous pool tags. + // + + ExFillFreedPool( (PCHAR)Entry + POOL_OVERHEAD, + (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD, + Entry->PoolTag ); + } +#endif + + // + // Free the specified pool block. + // + // Check to see if the next entry is free. + // + + Combined = FALSE; + NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + Entry->BlockSize); + if (PAGE_END(NextEntry) == FALSE) { + if (NextEntry->PoolType == 0) { + + // + // This block is free, combine with the released block. + // + + CHECK_LIST(__LINE__, ((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD)), P); + + Combined = TRUE; + RemoveEntryList(((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD))); + Entry->BlockSize += NextEntry->BlockSize; + +#if DEADBEEF + if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { + + // + // Overwrite the poolheader with its tag because we're being combined + // an the previous block. + // + + ExFillFreedPool( (PCHAR)NextEntry, + POOL_OVERHEAD + sizeof(LIST_ENTRY), + NextEntry->PoolTag ); + } +#endif + } + } + + // + // Check to see if the previous entry is free. + // + + if (Entry->PreviousSize != 0) { + NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry - Entry->PreviousSize); + if (NextEntry->PoolType == 0) { + + // + // This block is free, combine with the released block. + // + + CHECK_LIST(__LINE__, ((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD)), P); + + Combined = TRUE; + RemoveEntryList(((PLIST_ENTRY)((PCHAR)NextEntry + POOL_OVERHEAD))); + NextEntry->BlockSize += Entry->BlockSize; + +#if DEADBEEF + if (NtGlobalFlag & FLG_POOL_ENABLE_FREE_CHECK) { + + // + // Overwrite the poolheader with its tag because we're being combined + // an the previous block. + // + + ExFillFreedPool( (PCHAR)Entry, + POOL_OVERHEAD + sizeof(LIST_ENTRY), + Entry->PoolTag ); + } +#endif + + Entry = NextEntry; + } + } + + // + // If the block being freed has been combined into a full page, + // then return the free the page to memory management. + // + + if (PAGE_ALIGNED(Entry) && + (PAGE_END((PPOOL_BLOCK)Entry + Entry->BlockSize) != FALSE)) { + + // + // If the pool type is paged pool, then the paged pool page lock + // must be held during the free of the pool pages. + // + + LOCK_IF_PAGED_POOL(CheckType); + + MiFreePoolPages(Entry); + + UNLOCK_IF_PAGED_POOL(CheckType); + + PoolDesc->TotalPages -= 1; + + } else { + + // + // Insert this element into the list. + // + + Entry->PoolType = 0; + Entry->PoolIndex = PACK_POOL_INDEX(PoolIndex); + Index = Entry->BlockSize; + if (Index > POOL_SMALL_LISTS) { + Index = (Index >> SHIFT_OFFSET) + POOL_SMALL_LISTS + 1; + } + + // + // If the freed block was combined with any other block, then + // adjust the size of the next block if necessary. + // + + if (Combined != FALSE) { + + // + // The size of this entry has changed, if this entry is + // not the last one in the page, update the pool block + // after this block to have a new previous allocation size. + // + + NextEntry = (PPOOL_HEADER)((PPOOL_BLOCK)Entry + Entry->BlockSize); + if (PAGE_END(NextEntry) == FALSE) { + NextEntry->PreviousSize = Entry->BlockSize; + } + + // + // Reduce fragmentation and insert at the tail in hopes + // neighbors for this will be freed before this is reallocated. + // + + CHECK_LIST(__LINE__, &PoolDesc->ListHeads[Index - 1], P); + + InsertTailList(&PoolDesc->ListHeads[Index - 1], + ((PLIST_ENTRY)((PCHAR)Entry + POOL_OVERHEAD))); + + } else { + + CHECK_LIST(__LINE__, &PoolDesc->ListHeads[Index - 1], P); + + InsertHeadList(&PoolDesc->ListHeads[Index - 1], + ((PLIST_ENTRY)((PCHAR)Entry + POOL_OVERHEAD))); + } + } + + UNLOCK_POOL(PoolDesc, LockHandle); + + } + + return; +} + + +ULONG +ExQueryPoolBlockSize ( + IN PVOID PoolBlock, + OUT PBOOLEAN QuotaCharged + ) + +/*++ + +Routine Description: + + This function returns the size of the pool block. + +Arguments: + + PoolBlock - Supplies the address of the block of pool. + + QuotaCharged - Supplies a BOOLEAN variable to receive whether or not the + pool block had quota charged. + + NOTE: If the entry is bigger than a page, the value PAGE_SIZE is returned + rather than the correct number of bytes. + +Return Value: + + Size of pool block. + +--*/ + +{ + PPOOL_HEADER Entry; + ULONG size; + + if (PAGE_ALIGNED(PoolBlock)) { + *QuotaCharged = FALSE; + return PAGE_SIZE; + } + +#if POOL_CACHE_SUPPORTED + + // + // Align entry on pool allocation boundary. + // + + if (((ULONG)PoolBlock & POOL_CACHE_CHECK) == 0) { + Entry = (PPOOL_HEADER)((ULONG)PoolBlock - PoolCacheSize); + size = (Entry->BlockSize << POOL_BLOCK_SHIFT) - PoolCacheSize; + + } else { + Entry = (PPOOL_HEADER)((PCHAR)PoolBlock - POOL_OVERHEAD); + size = (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD; + } + +#else + + Entry = (PPOOL_HEADER)((PCHAR)PoolBlock - POOL_OVERHEAD); + size = (Entry->BlockSize << POOL_BLOCK_SHIFT) - POOL_OVERHEAD; + +#endif //POOL_CACHE_SUPPORTED + +#ifdef CHECK_POOL_TAIL + + size = size - POOL_SMALLEST_BLOCK; + +#endif + + if ( PoolTrackTable ) { + *QuotaCharged = FALSE; + } + else { + *QuotaCharged = (BOOLEAN) (Entry->ProcessBilled != NULL); + } + return size; +} + +VOID +ExQueryPoolUsage( + OUT PULONG PagedPoolPages, + OUT PULONG NonPagedPoolPages, + OUT PULONG PagedPoolAllocs, + OUT PULONG PagedPoolFrees, + OUT PULONG PagedPoolLookasideHits, + OUT PULONG NonPagedPoolAllocs, + OUT PULONG NonPagedPoolFrees, + OUT PULONG NonPagedPoolLookasideHits + ) + +{ + + ULONG Count; + ULONG Index; + PPOOL_DESCRIPTOR pd; + PKPRCB Prcb; + + // + // Sum all the paged pool usage. + // + + pd = PoolVector[PagedPool]; + *PagedPoolPages = 0; + *PagedPoolAllocs = 0; + *PagedPoolFrees = 0; + + for (Index = 0; Index < ExpNumberOfPagedPools + 1; Index += 1) { + *PagedPoolPages += pd[Index].TotalPages + pd[Index].TotalBigPages; + *PagedPoolAllocs += pd[Index].RunningAllocs; + *PagedPoolFrees += pd[Index].RunningDeAllocs; + } + + // + // Sum all the nonpaged pool usage. + // + + pd = PoolVector[NonPagedPool]; + *NonPagedPoolPages = pd->TotalPages + pd->TotalBigPages; + *NonPagedPoolAllocs = pd->RunningAllocs; + *NonPagedPoolFrees = pd->RunningDeAllocs; + + // + // Sum all the nonpaged must succeed usage. + // + + pd = PoolVector[NonPagedPoolMustSucceed]; + *NonPagedPoolPages += pd->TotalPages + pd->TotalBigPages; + *NonPagedPoolAllocs += pd->RunningAllocs; + *NonPagedPoolFrees += pd->RunningDeAllocs; + + // + // Sum all the lookaside hits for paged and nonpaged pool. + // + + for (Index = 0; Index < (ULONG)KeNumberProcessors; Index += 1) { + Prcb = KiProcessorBlock[Index]; + if (Prcb != NULL) { + +#if defined(_PPC_) + + *PagedPoolLookasideHits += Prcb->PagedPoolLookasideHits; + for (Count = 0; Count < POOL_SMALL_LISTS; Count +=1) { + *NonPagedPoolLookasideHits += ExpSmallNPagedPoolLookasideLists[Count].AllocateHits; + } + +#else + + for (Count = 0; Count < POOL_SMALL_LISTS; Count +=1) { + *PagedPoolLookasideHits += ExpSmallPagedPoolLookasideLists[Count].AllocateHits; + *NonPagedPoolLookasideHits += ExpSmallNPagedPoolLookasideLists[Count].AllocateHits; + } + +#endif + + } + } + + return; +} + +VOID +ExReturnPoolQuota( + IN PVOID P + ) + +/*++ + +Routine Description: + + This function returns quota charged to a subject process when the + specified pool block was allocated. + +Arguments: + + P - Supplies the address of the block of pool being deallocated. + +Return Value: + + None. + +--*/ + +{ + + PPOOL_HEADER Entry; + POOL_TYPE PoolType; + PEPROCESS Process; + + // + // Align the entry address to a pool allocation boundary. + // + +#if POOL_CACHE_SUPPORTED + + if (((ULONG)P & POOL_CACHE_CHECK) == 0) { + Entry = (PPOOL_HEADER)((ULONG)P - PoolCacheSize); + + } else { + Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD); + } + +#else + + Entry = (PPOOL_HEADER)((PCHAR)P - POOL_OVERHEAD); + +#endif //POOL_CACHE_SUPPORTED + + // + // If quota was charged, then return the appropriate quota to the + // subject process. + // + + PoolType = (Entry->PoolType & POOL_TYPE_MASK) - 1; + if ((Entry->PoolType & POOL_QUOTA_MASK) && + (PoolTrackTable == NULL)) { + Process = Entry->ProcessBilled; + Entry->PoolTag = 'atoQ'; + Entry->PoolType &= ~POOL_QUOTA_MASK; + if (Process != NULL) { + PsReturnPoolQuota(Process, + PoolType & BASE_POOL_TYPE_MASK, + (ULONG)Entry->BlockSize << POOL_BLOCK_SHIFT); + + ObDereferenceObject(Process); + } + } + + return; +} + +#if DBG || (i386 && !FPO) + +// +// Only works on checked builds or free x86 builds with FPO turned off +// See comment in mm\allocpag.c +// + +NTSTATUS +ExpSnapShotPoolPages( + IN PVOID Address, + IN ULONG Size, + IN OUT PSYSTEM_POOL_INFORMATION PoolInformation, + IN OUT PSYSTEM_POOL_ENTRY *PoolEntryInfo, + IN ULONG Length, + IN OUT PULONG RequiredLength + ) +{ + NTSTATUS Status; + CLONG i; + PPOOL_HEADER p; + + if (PAGE_ALIGNED(Address)) { + for (i = 0; i < MAX_BIGPAGE_TABLE; i++) { + if (PoolBigPageTable[i].NumberOfPages != 0 && + PoolBigPageTable[i].Va == Address + ) { + PoolInformation->NumberOfEntries += 1; + *RequiredLength += sizeof(SYSTEM_POOL_ENTRY); + if (Length < *RequiredLength) { + Status = STATUS_INFO_LENGTH_MISMATCH; + + } else { + (*PoolEntryInfo)->Allocated = TRUE; + (*PoolEntryInfo)->Size = (PoolBigPageTable[i].NumberOfPages * PAGE_SIZE); + (*PoolEntryInfo)->AllocatorBackTraceIndex = 0; +#if i386 && !FPO + if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB)) { + (*PoolEntryInfo)->AllocatorBackTraceIndex = + PoolBigPageTable[i].AllocatorBackTraceIndex; + } +#endif // i386 && !FPO + (*PoolEntryInfo)->ProcessChargedQuota = 0; +#if !DBG + if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) +#endif //!DBG + (*PoolEntryInfo)->TagUlong = PoolBigPageTable[i].Key; + (*PoolEntryInfo)++; + Status = STATUS_SUCCESS; + } + + return Status; + } + } + } + + p = (PPOOL_HEADER)Address; + if ((Size == PAGE_SIZE) && (p->PreviousSize == 0)) { + ULONG EntrySize; + + do { + EntrySize = p->BlockSize << POOL_BLOCK_SHIFT; + if (EntrySize == 0) { + return STATUS_COMMITMENT_LIMIT; + } + PoolInformation->NumberOfEntries += 1; + *RequiredLength += sizeof(SYSTEM_POOL_ENTRY); + if (Length < *RequiredLength) { + Status = STATUS_INFO_LENGTH_MISMATCH; + + } else { + (*PoolEntryInfo)->Size = EntrySize; + if (p->PoolType != 0) { + (*PoolEntryInfo)->Allocated = TRUE; + (*PoolEntryInfo)->AllocatorBackTraceIndex = 0; + (*PoolEntryInfo)->ProcessChargedQuota = 0; +#if !DBG + if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) +#endif //!DBG + (*PoolEntryInfo)->TagUlong = p->PoolTag; +#if i386 && !FPO + if ((NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) && + p->AllocatorBackTraceIndex != 0 && + p->AllocatorBackTraceIndex & POOL_BACKTRACEINDEX_PRESENT + ) { + (*PoolEntryInfo)->AllocatorBackTraceIndex = p->AllocatorBackTraceIndex ^ POOL_BACKTRACEINDEX_PRESENT; +#if !DBG + if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) { +#else + if (TRUE) { +#endif //!DBG + if (p->PoolTagHash & (PROTECTED_POOL >> 16)) { + (*PoolEntryInfo)->TagUlong = PROTECTED_POOL; + } else { + (*PoolEntryInfo)->TagUlong = 0; + } + (*PoolEntryInfo)->TagUlong |= PoolTrackTable[p->PoolTagHash&~(PROTECTED_POOL >> 16)].Key; + } + } +#endif // i386 && !FPO + + } else { + (*PoolEntryInfo)->Allocated = FALSE; + (*PoolEntryInfo)->AllocatorBackTraceIndex = 0; + (*PoolEntryInfo)->ProcessChargedQuota = 0; +#if !DBG + if (NtGlobalFlag & FLG_POOL_ENABLE_TAGGING) +#endif //!DBG + (*PoolEntryInfo)->TagUlong = p->PoolTag; + } + + (*PoolEntryInfo)++; + Status = STATUS_SUCCESS; + } + + p = (PPOOL_HEADER)((PCHAR)p + EntrySize); + } + while (PAGE_END(p) == FALSE); + + } else { + PoolInformation->NumberOfEntries += 1; + *RequiredLength += sizeof(SYSTEM_POOL_ENTRY); + if (Length < *RequiredLength) { + Status = STATUS_INFO_LENGTH_MISMATCH; + + } else { + (*PoolEntryInfo)->Allocated = TRUE; + (*PoolEntryInfo)->Size = Size; + (*PoolEntryInfo)->AllocatorBackTraceIndex = 0; + (*PoolEntryInfo)->ProcessChargedQuota = 0; + (*PoolEntryInfo)++; + Status = STATUS_SUCCESS; + } + } + + return Status; +} + +NTSTATUS +ExSnapShotPool( + IN POOL_TYPE PoolType, + IN PSYSTEM_POOL_INFORMATION PoolInformation, + IN ULONG Length, + OUT PULONG ReturnLength OPTIONAL + ) + +{ + + ULONG Index; + PVOID Lock; + KIRQL LockHandle; + PPOOL_DESCRIPTOR PoolDesc; + ULONG RequiredLength; + NTSTATUS Status; + + RequiredLength = FIELD_OFFSET(SYSTEM_POOL_INFORMATION, Entries); + if (Length < RequiredLength) { + return STATUS_INFO_LENGTH_MISMATCH; + } + + try { + + // + // If the pool type is paged, then lock all of the paged pools. + // Otherwise, lock the nonpaged pool. + // + + PoolDesc = PoolVector[PoolType]; + if (PoolType == PagedPool) { + Index = 0; + KeRaiseIrql(APC_LEVEL, &LockHandle); \ + do { + Lock = PoolDesc[Index].LockAddress; + ExAcquireFastMutex((PFAST_MUTEX)Lock); + Index += 1; + } while (Index < ExpNumberOfPagedPools); + + } else { + ExAcquireSpinLock(&NonPagedPoolLock, &LockHandle); + } + + PoolInformation->EntryOverhead = POOL_OVERHEAD; + PoolInformation->NumberOfEntries = 0; + +#if POOL_CACHE_SUPPORTED //only compile for machines which have a nonzero value. + + if (PoolType & CACHE_ALIGNED_POOL_TYPE_MASK) { + PoolInformation->EntryOverhead = (USHORT)PoolCacheSize; + } + +#endif //POOL_CACHE_SUPPORTED + + Status = MmSnapShotPool(PoolType, + ExpSnapShotPoolPages, + PoolInformation, + Length, + &RequiredLength); + + } finally { + + // + // If the pool type is paged, then unlock all of the paged pools. + // Otherwise, unlock the nonpaged pool. + // + + if (PoolType == PagedPool) { + Index = 0; + do { + Lock = PoolDesc[Index].LockAddress; + ExReleaseFastMutex((PFAST_MUTEX)Lock); + Index += 1; + } while (Index < ExpNumberOfPagedPools); + + KeLowerIrql(LockHandle); + + } else { + ExReleaseSpinLock(&NonPagedPoolLock, LockHandle); + } + } + + if (ARGUMENT_PRESENT(ReturnLength)) { + *ReturnLength = RequiredLength; + } + + return Status; +} +#endif // DBG || (i386 && !FPO) + +#if i386 && !FPO +USHORT +ExGetPoolBackTraceIndex( + IN PVOID P + ) +{ + CLONG i; + PPOOL_HEADER Entry; + + if (NtGlobalFlag & FLG_KERNEL_STACK_TRACE_DB) { + if ( PAGE_ALIGNED(P) ) { + for (i = 0; i < MAX_BIGPAGE_TABLE; i++) { + if (PoolBigPageTable[i].NumberOfPages != 0 && + PoolBigPageTable[i].Va == P + ) { + return PoolBigPageTable[i].AllocatorBackTraceIndex; + } + } + + } else { +#if POOL_CACHE_SUPPORTED + + // + // Align entry on pool allocation boundary. + // + + if (((ULONG)P & POOL_CACHE_CHECK) == 0) { + Entry = (PPOOL_HEADER)((ULONG)P - PoolCacheSize); + } else { + Entry = (PPOOL_HEADER)((PCH)P - POOL_OVERHEAD); + } + +#else + Entry = (PPOOL_HEADER)((PCH)P - POOL_OVERHEAD); +#endif //POOL_CACHE_SUPPORTED + + if (Entry->AllocatorBackTraceIndex != 0 && + Entry->AllocatorBackTraceIndex & POOL_BACKTRACEINDEX_PRESENT + ) { + return (Entry->AllocatorBackTraceIndex ^ POOL_BACKTRACEINDEX_PRESENT); + } + } + } + + return 0; +} +#endif // i386 && !FPO |