1 files changed, 1376 insertions, 0 deletions

diff --git a/private/ntos/mm/sysptes.c b/private/ntos/mm/sysptes.c
new file mode 100644
index 000000000..d117890ae
--- /dev/null
+++ b/private/ntos/mm/sysptes.c
@@ -0,0 +1,1376 @@
+/*++
+
+Copyright (c) 1989  Microsoft Corporation
+
+Module Name:
+
+   sysptes.c
+
+Abstract:
+
+    This module contains the routines which reserve and release
+    system wide PTEs reserved within the non paged portion of the
+    system space.  These PTEs are used for mapping I/O devices
+    and mapping kernel stacks for threads.
+
+Author:
+
+    Lou Perazzoli (loup) 6-Apr-1989
+
+Revision History:
+
+--*/
+
+#include "mi.h"
+
+#ifdef ALLOC_PRAGMA
+#pragma alloc_text(INIT,MiInitializeSystemPtes)
+#endif
+
+
+ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes];
+ULONG MmSystemPtesStart[MaximumPtePoolTypes];
+ULONG MmSystemPtesEnd[MaximumPtePoolTypes];
+
+#define MM_MIN_SYSPTE_FREE 500
+#define MM_MAX_SYSPTE_FREE 3000
+
+PMMPTE MmFlushPte1;
+
+MMPTE MmFlushCounter;
+
+//
+// PTEs are binned at sizes 1, 2, 4, 8, and 16.
+//
+
+#ifdef _ALPHA_
+
+//
+// alpha has 8k pages size and stacks consume 9 pages (including guard page).
+//
+
+ULONG MmSysPteIndex[MM_SYS_PTE_TABLES_MAX] = {1,2,4,9,16};
+
+UCHAR MmSysPteTables[17] = {0,0,1,2,2,3,3,3,3,3,4,4,4,4,4,4,4};
+
+#else
+
+ULONG MmSysPteIndex[MM_SYS_PTE_TABLES_MAX] = {1,2,4,8,16};
+
+UCHAR MmSysPteTables[17] = {0,0,1,2,2,3,3,3,3,4,4,4,4,4,4,4,4};
+#endif
+
+MMPTE MmFreeSysPteListBySize [MM_SYS_PTE_TABLES_MAX];
+PMMPTE MmLastSysPteListBySize [MM_SYS_PTE_TABLES_MAX];
+ULONG MmSysPteListBySizeCount [MM_SYS_PTE_TABLES_MAX];
+ULONG MmSysPteMinimumFree [MM_SYS_PTE_TABLES_MAX] = {100,50,30,20,20};
+
+//
+// Initial sizes for PTE lists.
+//
+
+#define MM_PTE_LIST_1  400
+#define MM_PTE_LIST_2  100
+#define MM_PTE_LIST_4   60
+#define MM_PTE_LIST_8   50
+#define MM_PTE_LIST_16  40
+
+#define MM_PTE_TABLE_LIMIT 16
+
+PMMPTE
+MiReserveSystemPtes2 (
+    IN ULONG NumberOfPtes,
+    IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType,
+    IN ULONG Alignment,
+    IN ULONG Offset,
+    IN ULONG BugCheckOnFailure
+    );
+
+VOID
+MiFeedSysPtePool (
+    IN ULONG Index
+    );
+
+VOID
+MiDumpSystemPtes (
+    IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
+    );
+
+ULONG
+MiCountFreeSystemPtes (
+    IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
+    );
+
+
+PMMPTE
+MiReserveSystemPtes (
+    IN ULONG NumberOfPtes,
+    IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType,
+    IN ULONG Alignment,
+    IN ULONG Offset,
+    IN ULONG BugCheckOnFailure
+    )
+
+/*++
+
+Routine Description:
+
+    This function locates the specified number of unused PTEs to locate
+    within the non paged portion of system space.
+
+Arguments:
+
+    NumberOfPtes - Supplies the number of PTEs to locate.
+
+    SystemPtePoolType - Supplies the PTE type of the pool to expand, one of
+                        SystemPteSpace or NonPagedPoolExpansion.
+
+    Alignment - Supplies the virtual address alignment for the address
+                the returned PTE maps. For example, if the value is 64K,
+                the returned PTE will map an address on a 64K boundary.
+                An alignment of zero means to align on a page boundary.
+
+    Offset - Supplies the offset into the alignment for the virtual address.
+             For example, if the Alignment is 64k and the Offset is 4k,
+             the returned address will be 4k above a 64k boundary.
+
+    BugCheckOnFailure - Supplies FALSE if NULL should be returned if
+                        the request cannot be satisfied, TRUE if
+                        a bugcheck should be issued.
+
+Return Value:
+
+    Returns the address of the first PTE located.
+    NULL if no system PTEs can be located and BugCheckOnFailure is FALSE.
+
+Environment:
+
+    Kernel mode, DISPATCH_LEVEL or below.
+
+--*/
+
+{
+    PMMPTE PointerPte;
+    PMMPTE Previous;
+    KIRQL OldIrql;
+    ULONG PteMask;
+    ULONG MaskSize;
+    ULONG Index;
+
+    if (SystemPtePoolType == SystemPteSpace) {
+
+        MaskSize = (Alignment - 1) >> (PAGE_SHIFT - PTE_SHIFT);
+        PteMask = MaskSize & (Offset >> (PAGE_SHIFT - PTE_SHIFT));
+
+        //
+        // Acquire the system space lock to synchronize access to this
+        // routine.
+        //
+
+        ExAcquireSpinLock ( &MmSystemSpaceLock, &OldIrql );
+
+        if (NumberOfPtes <= MM_PTE_TABLE_LIMIT) {
+            Index = MmSysPteTables [NumberOfPtes];
+            ASSERT (NumberOfPtes <= MmSysPteIndex[Index]);
+            PointerPte = &MmFreeSysPteListBySize[Index];
+#if DBG
+            if (MmDebug & MM_DBG_SYS_PTES) {
+                PMMPTE PointerPte1;
+                PointerPte1 = &MmFreeSysPteListBySize[Index];
+                while (PointerPte1->u.List.NextEntry != MM_EMPTY_PTE_LIST) {
+                    PMMPTE PointerFreedPte;
+                    ULONG j;
+
+                    PointerPte1 = MmSystemPteBase + PointerPte1->u.List.NextEntry;
+                    PointerFreedPte = PointerPte1;
+                    for (j = 0; j < MmSysPteIndex[Index]; j++) {
+                        ASSERT (PointerFreedPte->u.Hard.Valid == 0);
+                        PointerFreedPte++;
+                    }
+                }
+            }
+#endif //DBG
+
+            Previous = PointerPte;
+
+            while (PointerPte->u.List.NextEntry != MM_EMPTY_PTE_LIST) {
+
+                //
+                //  Try to find suitable PTEs with the proper alignment.
+                //
+
+                Previous = PointerPte;
+                PointerPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
+                if (PointerPte == MmFlushPte1) {
+                    KeFlushEntireTb (TRUE, TRUE);
+                    MmFlushCounter.u.List.NextEntry += 1;
+                    MmFlushPte1 = NULL;
+                }
+                if ((Alignment == 0) ||
+                    (((ULONG)PointerPte & MaskSize) == PteMask)) {
+
+                    //
+                    // Proper alignment and offset, update list index.
+                    //
+
+                    ASSERT ((ULONG)(PointerPte->u.List.NextEntry + MmSystemPteBase) >=
+                             MmSystemPtesStart[SystemPtePoolType] ||
+                             PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST);
+                    ASSERT ((ULONG)(PointerPte->u.List.NextEntry + MmSystemPteBase) <=
+                             MmSystemPtesEnd[SystemPtePoolType] ||
+                             PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST);
+
+                    Previous->u.List.NextEntry = PointerPte->u.List.NextEntry;
+                    MmSysPteListBySizeCount [Index] -= 1;
+
+                    if (NumberOfPtes != 1) {
+
+                        //
+                        // Check to see if the TB should be flushed.
+                        //
+
+                        if ((PointerPte + 1)->u.List.NextEntry == MmFlushCounter.u.List.NextEntry) {
+                            KeFlushEntireTb (TRUE, TRUE);
+                            MmFlushCounter.u.List.NextEntry += 1;
+                            MmFlushPte1 = NULL;
+                        }
+                    }
+                    if (PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) {
+                        MmLastSysPteListBySize[Index] = Previous;
+                    }
+#if DBG
+
+                    if (MmDebug & MM_DBG_SYS_PTES) {
+                        PMMPTE PointerPte1;
+                        PointerPte1 = &MmFreeSysPteListBySize[Index];
+                        while (PointerPte1->u.List.NextEntry != MM_EMPTY_PTE_LIST) {
+                            PMMPTE PointerFreedPte;
+                            ULONG j;
+
+                            PointerPte1 = MmSystemPteBase + PointerPte1->u.List.NextEntry;
+                            PointerFreedPte = PointerPte1;
+                            for (j = 0; j < MmSysPteIndex[Index]; j++) {
+                                ASSERT (PointerFreedPte->u.Hard.Valid == 0);
+                                PointerFreedPte++;
+                            }
+                        }
+                    }
+#endif //DBG
+                    ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql);
+
+#if DBG
+                    PointerPte->u.List.NextEntry = 0xABCDE;
+                    if (MmDebug & MM_DBG_SYS_PTES) {
+
+                        PMMPTE PointerFreedPte;
+                        ULONG j;
+
+                        PointerFreedPte = PointerPte;
+                        for (j = 0; j < MmSysPteIndex[Index]; j++) {
+                            ASSERT (PointerFreedPte->u.Hard.Valid == 0);
+                            PointerFreedPte++;
+                        }
+                    }
+                    if (!((ULONG)PointerPte >= MmSystemPtesStart[SystemPtePoolType])) {
+                        KeBugCheckEx (MEMORY_MANAGEMENT,
+                                      0x652,(ULONG)PointerPte,
+                                      NumberOfPtes,
+                                      SystemPtePoolType);
+                    }
+                    if (!((ULONG)PointerPte <= MmSystemPtesEnd[SystemPtePoolType])) {
+                        KeBugCheckEx (MEMORY_MANAGEMENT,
+                                      0x653,(ULONG)PointerPte,
+                                      NumberOfPtes,
+                                      SystemPtePoolType); //fixfix make assert
+                    }
+#endif //DBG
+
+                    if (MmSysPteListBySizeCount[Index] <
+                                            MmSysPteMinimumFree[Index]) {
+                        MiFeedSysPtePool (Index);
+                    }
+                    return PointerPte;
+                }
+            }
+            NumberOfPtes = MmSysPteIndex [Index];
+        }
+        ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql);
+    }
+    PointerPte = MiReserveSystemPtes2 (NumberOfPtes,
+                                       SystemPtePoolType,
+                                       Alignment,
+                                       Offset,
+                                       BugCheckOnFailure);
+#if DBG
+    if (MmDebug & MM_DBG_SYS_PTES) {
+
+        PMMPTE PointerFreedPte;
+        ULONG j;
+
+        PointerFreedPte = PointerPte;
+        for (j = 0; j < NumberOfPtes; j++) {
+            ASSERT (PointerFreedPte->u.Hard.Valid == 0);
+            PointerFreedPte++;
+        }
+    }
+#endif //DBG
+    return PointerPte;
+}
+
+VOID
+MiFeedSysPtePool (
+    IN ULONG Index
+    )
+
+/*++
+
+Routine Description:
+
+    This routine adds PTEs to the look aside lists.
+
+Arguments:
+
+    Index - Supplies the index for the look aside list to fill.
+
+Return Value:
+
+    None.
+
+
+Environment:
+
+    Kernel mode, internal to SysPtes.
+
+--*/
+
+{
+    ULONG i;
+    PMMPTE PointerPte;
+
+    if (MmTotalFreeSystemPtes[SystemPteSpace] < MM_MIN_SYSPTE_FREE) {
+        return;
+    }
+
+    for (i = 0; i < 10 ; i++ ) {
+        PointerPte = MiReserveSystemPtes2 (MmSysPteIndex [Index],
+                                           SystemPteSpace,
+                                           0,
+                                           0,
+                                           FALSE);
+        if (PointerPte == NULL) {
+            return;
+        }
+        MiReleaseSystemPtes (PointerPte,
+                             MmSysPteIndex [Index],
+                             SystemPteSpace);
+    }
+    return;
+}
+
+
+PMMPTE
+MiReserveSystemPtes2 (
+    IN ULONG NumberOfPtes,
+    IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType,
+    IN ULONG Alignment,
+    IN ULONG Offset,
+    IN ULONG BugCheckOnFailure
+    )
+
+/*++
+
+Routine Description:
+
+    This function locates the specified number of unused PTEs to locate
+    within the non paged portion of system space.
+
+Arguments:
+
+    NumberOfPtes - Supplies the number of PTEs to locate.
+
+    SystemPtePoolType - Supplies the PTE type of the pool to expand, one of
+                        SystemPteSpace or NonPagedPoolExpansion.
+
+    Alignment - Supplies the virtual address alignment for the address
+                the returned PTE maps. For example, if the value is 64K,
+                the returned PTE will map an address on a 64K boundary.
+                An alignment of zero means to align on a page boundary.
+
+    Offset - Supplies the offset into the alignment for the virtual address.
+             For example, if the Alignment is 64k and the Offset is 4k,
+             the returned address will be 4k above a 64k boundary.
+
+    BugCheckOnFailure - Supplies FALSE if NULL should be returned if
+                        the request cannot be satisfied, TRUE if
+                        a bugcheck should be issued.
+
+Return Value:
+
+    Returns the address of the first PTE located.
+    NULL if no system PTEs can be located and BugCheckOnFailure is FALSE.
+
+Environment:
+
+    Kernel mode, DISPATCH_LEVEL or below.
+
+--*/
+
+{
+    PMMPTE PointerPte;
+    PMMPTE PointerFollowingPte;
+    PMMPTE Previous;
+    ULONG SizeInSet;
+    KIRQL OldIrql;
+    ULONG MaskSize;
+    ULONG NumberOfRequiredPtes;
+    ULONG OffsetSum;
+    ULONG PtesToObtainAlignment;
+    PMMPTE NextSetPointer;
+    ULONG LeftInSet;
+    ULONG PteOffset;
+    MMPTE_FLUSH_LIST PteFlushList;
+
+    MaskSize = (Alignment - 1) >> (PAGE_SHIFT - PTE_SHIFT);
+
+    OffsetSum = (Offset >> (PAGE_SHIFT - PTE_SHIFT)) |
+                            (Alignment >> (PAGE_SHIFT - PTE_SHIFT));
+
+    ExAcquireSpinLock ( &MmSystemSpaceLock, &OldIrql );
+
+    //
+    // The nonpaged PTE pool use the invalid PTEs to define the pool
+    // structure.   A global pointer points to the first free set
+    // in the list, each free set contains the number free and a pointer
+    // to the next free set.  The free sets are kept in an ordered list
+    // such that the pointer to the next free set is always greater
+    // than the address of the current free set.
+    //
+    // As to not limit the size of this pool, a two PTEs are used
+    // to define a free region.  If the region is a single PTE, the
+    // Prototype field within the PTE is set indicating the set
+    // consists of a single PTE.
+    //
+    // The page frame number field is used to define the next set
+    // and the number free.  The two flavors are:
+    //
+    //                           o          V
+    //                           n          l
+    //                           e          d
+    //  +-----------------------+-+----------+
+    //  |  next set             |0|0        0|
+    //  +-----------------------+-+----------+
+    //  |  number in this set   |0|0        0|
+    //  +-----------------------+-+----------+
+    //
+    //
+    //  +-----------------------+-+----------+
+    //  |  next set             |1|0        0|
+    //  +-----------------------+-+----------+
+    //  ...
+    //
+
+    //
+    // Acquire the system space lock to synchronize access to this
+    // routine.
+    //
+
+    PointerPte = &MmFirstFreeSystemPte[SystemPtePoolType];
+    Previous = PointerPte;
+
+    if (PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) {
+
+        //
+        // End of list and none found, return NULL or bugcheck.
+        //
+
+        if (BugCheckOnFailure) {
+            KeBugCheckEx (NO_MORE_SYSTEM_PTES,
+                          (ULONG)SystemPtePoolType,
+                          NumberOfPtes,
+                          MmTotalFreeSystemPtes[SystemPtePoolType],
+                          MmNumberOfSystemPtes);
+        }
+
+        ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );
+        return NULL;
+    }
+
+    PointerPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
+
+    if (Alignment <= PAGE_SIZE) {
+
+        //
+        // Don't deal with aligment issues.
+        //
+
+        while (TRUE) {
+
+            if (PointerPte->u.List.OneEntry) {
+                SizeInSet = 1;
+
+            } else {
+
+                PointerFollowingPte = PointerPte + 1;
+                SizeInSet = PointerFollowingPte->u.List.NextEntry;
+            }
+
+            if (NumberOfPtes < SizeInSet) {
+
+                //
+                // Get the PTEs from this set and reduce the size of the
+                // set.  Note that the size of the current set cannot be 1.
+                //
+
+                if ((SizeInSet - NumberOfPtes) == 1) {
+
+                    //
+                    // Collapse to the single PTE format.
+                    //
+
+                    PointerPte->u.List.OneEntry = 1;
+
+                } else {
+
+                    PointerFollowingPte->u.List.NextEntry = SizeInSet - NumberOfPtes;
+
+                    //
+                    // Get the required PTEs from the end of the set.
+                    //
+
+#if DBG
+                    if (MmDebug & MM_DBG_SYS_PTES) {
+                        MiDumpSystemPtes(SystemPtePoolType);
+                        PointerFollowingPte = PointerPte + (SizeInSet - NumberOfPtes);
+                        DbgPrint("allocated 0x%lx Ptes at %lx\n",NumberOfPtes,PointerFollowingPte);
+                    }
+#endif //DBG
+                }
+
+                MmTotalFreeSystemPtes[SystemPtePoolType] -= NumberOfPtes;
+#if DBG
+                if (MmDebug & MM_DBG_SYS_PTES) {
+                    ASSERT (MmTotalFreeSystemPtes[SystemPtePoolType] ==
+                             MiCountFreeSystemPtes (SystemPtePoolType));
+                }
+#endif //DBG
+
+                ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );
+
+                PointerPte =  PointerPte + (SizeInSet - NumberOfPtes);
+                goto Flush;
+            }
+
+            if (NumberOfPtes == SizeInSet) {
+
+                //
+                // Satisfy the request with this complete set and change
+                // the list to reflect the fact that this set is gone.
+                //
+
+                Previous->u.List.NextEntry = PointerPte->u.List.NextEntry;
+
+                //
+                // Release the system PTE lock.
+                //
+
+#if DBG
+                if (MmDebug & MM_DBG_SYS_PTES) {
+                        MiDumpSystemPtes(SystemPtePoolType);
+                        PointerFollowingPte = PointerPte + (SizeInSet - NumberOfPtes);
+                        DbgPrint("allocated 0x%lx Ptes at %lx\n",NumberOfPtes,PointerFollowingPte);
+                }
+#endif
+
+                MmTotalFreeSystemPtes[SystemPtePoolType] -= NumberOfPtes;
+#if DBG
+                if (MmDebug & MM_DBG_SYS_PTES) {
+                    ASSERT (MmTotalFreeSystemPtes[SystemPtePoolType] ==
+                             MiCountFreeSystemPtes (SystemPtePoolType));
+                }
+#endif //DBG
+                ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );
+                goto Flush;
+            }
+
+            //
+            // Point to the next set and try again
+            //
+
+            if (PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) {
+
+                //
+                // End of list and none found, return NULL or bugcheck.
+                //
+
+                if (BugCheckOnFailure) {
+                    KeBugCheckEx (NO_MORE_SYSTEM_PTES,
+                                  (ULONG)SystemPtePoolType,
+                                  NumberOfPtes,
+                                  MmTotalFreeSystemPtes[SystemPtePoolType],
+                                  MmNumberOfSystemPtes);
+                }
+
+                ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );
+                return NULL;
+            }
+            Previous = PointerPte;
+            PointerPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
+            ASSERT (PointerPte > Previous);
+        }
+
+    } else {
+
+        //
+        // Deal with the alignment issues.
+        //
+
+        while (TRUE) {
+
+            if (PointerPte->u.List.OneEntry) {
+                SizeInSet = 1;
+
+            } else {
+
+                PointerFollowingPte = PointerPte + 1;
+                SizeInSet = PointerFollowingPte->u.List.NextEntry;
+            }
+
+            PtesToObtainAlignment =
+                (((OffsetSum - ((ULONG)PointerPte & MaskSize)) & MaskSize) >>
+                    PTE_SHIFT);
+
+            NumberOfRequiredPtes = NumberOfPtes + PtesToObtainAlignment;
+
+            if (NumberOfRequiredPtes < SizeInSet) {
+
+                //
+                // Get the PTEs from this set and reduce the size of the
+                // set.  Note that the size of the current set cannot be 1.
+                //
+                // This current block will be slit into 2 blocks if
+                // the PointerPte does not match the aligment.
+                //
+
+                //
+                // Check to see if the first PTE is on the proper
+                // alignment, if so, eliminate this block.
+                //
+
+                LeftInSet = SizeInSet - NumberOfRequiredPtes;
+
+                //
+                // Set up the new set at the end of this block.
+                //
+
+                NextSetPointer = PointerPte + NumberOfRequiredPtes;
+                NextSetPointer->u.List.NextEntry =
+                                       PointerPte->u.List.NextEntry;
+
+                PteOffset = NextSetPointer - MmSystemPteBase;
+
+                if (PtesToObtainAlignment == 0) {
+
+                    Previous->u.List.NextEntry += NumberOfRequiredPtes;
+
+                } else {
+
+                    //
+                    // Point to the new set at the end of the block
+                    // we are giving away.
+                    //
+
+                    PointerPte->u.List.NextEntry = PteOffset;
+
+                    //
+                    // Update the size of the current set.
+                    //
+
+                    if (PtesToObtainAlignment == 1) {
+
+                        //
+                        // Collapse to the single PTE format.
+                        //
+
+                        PointerPte->u.List.OneEntry = 1;
+
+                    } else {
+
+                        //
+                        // Set the set size in the next PTE.
+                        //
+
+                        PointerFollowingPte->u.List.NextEntry =
+                                                        PtesToObtainAlignment;
+                    }
+                }
+
+                //
+                // Set up the new set at the end of the block.
+                //
+
+                if (LeftInSet == 1) {
+                    NextSetPointer->u.List.OneEntry = 1;
+                } else {
+                    NextSetPointer->u.List.OneEntry = 0;
+                    NextSetPointer += 1;
+                    NextSetPointer->u.List.NextEntry = LeftInSet;
+                }
+                MmTotalFreeSystemPtes[SystemPtePoolType] -= NumberOfPtes;
+#if DBG
+                if (MmDebug & MM_DBG_SYS_PTES) {
+                    ASSERT (MmTotalFreeSystemPtes[SystemPtePoolType] ==
+                             MiCountFreeSystemPtes (SystemPtePoolType));
+                }
+#endif //DBG
+                ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );
+
+                PointerPte = PointerPte + PtesToObtainAlignment;
+                goto Flush;
+            }
+
+            if (NumberOfRequiredPtes == SizeInSet) {
+
+                //
+                // Satisfy the request with this complete set and change
+                // the list to reflect the fact that this set is gone.
+                //
+
+                if (PtesToObtainAlignment == 0) {
+
+                    //
+                    // This block exactly satifies the request.
+                    //
+
+                    Previous->u.List.NextEntry =
+                                            PointerPte->u.List.NextEntry;
+
+                } else {
+
+                    //
+                    // A portion at the start of this block remains.
+                    //
+
+                    if (PtesToObtainAlignment == 1) {
+
+                        //
+                        // Collapse to the single PTE format.
+                        //
+
+                        PointerPte->u.List.OneEntry = 1;
+
+                    } else {
+                      PointerFollowingPte->u.List.NextEntry =
+                                                        PtesToObtainAlignment;
+
+                    }
+                }
+
+                MmTotalFreeSystemPtes[SystemPtePoolType] -= NumberOfPtes;
+#if DBG
+                if (MmDebug & MM_DBG_SYS_PTES) {
+                    ASSERT (MmTotalFreeSystemPtes[SystemPtePoolType] ==
+                             MiCountFreeSystemPtes (SystemPtePoolType));
+                }
+#endif //DBG
+                ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );
+
+                PointerPte = PointerPte + PtesToObtainAlignment;
+                goto Flush;
+            }
+
+            //
+            // Point to the next set and try again
+            //
+
+            if (PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) {
+
+                //
+                // End of list and none found, return NULL or bugcheck.
+                //
+
+                if (BugCheckOnFailure) {
+                    KeBugCheckEx (NO_MORE_SYSTEM_PTES,
+                                  (ULONG)SystemPtePoolType,
+                                  NumberOfPtes,
+                                  MmTotalFreeSystemPtes[SystemPtePoolType],
+                                  MmNumberOfSystemPtes);
+                }
+
+                ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );
+                return NULL;
+            }
+            Previous = PointerPte;
+            PointerPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
+            ASSERT (PointerPte > Previous);
+        }
+    }
+Flush:
+
+    if (SystemPtePoolType == SystemPteSpace) {
+        PVOID BaseAddress;
+        ULONG j;
+
+        PteFlushList.Count = 0;
+        Previous = PointerPte;
+        BaseAddress = MiGetVirtualAddressMappedByPte (Previous);
+
+        for (j = 0; j < NumberOfPtes ; j++) {
+            if (PteFlushList.Count != MM_MAXIMUM_FLUSH_COUNT) {
+                PteFlushList.FlushPte[PteFlushList.Count] = Previous;
+                PteFlushList.FlushVa[PteFlushList.Count] = BaseAddress;
+                PteFlushList.Count += 1;
+            }
+            *Previous = ZeroKernelPte;
+            BaseAddress = (PVOID)((PCHAR)BaseAddress + PAGE_SIZE);
+            Previous++;
+        }
+
+        KeRaiseIrql (DISPATCH_LEVEL, &OldIrql);
+        MiFlushPteList (&PteFlushList, TRUE, ZeroKernelPte);
+        KeLowerIrql (OldIrql);
+    }
+    return PointerPte;
+}
+
+VOID
+MiReleaseSystemPtes (
+    IN PMMPTE StartingPte,
+    IN ULONG NumberOfPtes,
+    IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
+    )
+
+/*++
+
+Routine Description:
+
+    This function releases the specified number of PTEs
+    within the non paged portion of system space.
+
+    Note that the PTEs must be invalid and the page frame number
+    must have been set to zero.
+
+Arguments:
+
+    StartingPte - Supplies the address of the first PTE to release.
+
+    NumberOfPtes - Supplies the number of PTEs to release.
+
+Return Value:
+
+    none.
+
+Environment:
+
+    Kernel mode.
+
+--*/
+
+{
+
+    ULONG Size;
+    ULONG i;
+    ULONG PteOffset;
+    PMMPTE PointerPte;
+    PMMPTE PointerFollowingPte;
+    PMMPTE NextPte;
+    KIRQL OldIrql;
+    ULONG Index;
+    MMPTE TempPte;
+
+    //
+    // Check to make sure the PTEs don't map anything.
+    //
+
+    ASSERT (NumberOfPtes != 0);
+#if DBG
+    if (!((ULONG)StartingPte >= MmSystemPtesStart[SystemPtePoolType])) {
+        KeBugCheckEx (MEMORY_MANAGEMENT,
+                      0x656,(ULONG)StartingPte,
+                      NumberOfPtes,
+                      SystemPtePoolType);
+    }
+
+    if (!((ULONG)StartingPte <= MmSystemPtesEnd[SystemPtePoolType])) {
+        KeBugCheckEx (MEMORY_MANAGEMENT,
+                      0x657,(ULONG)StartingPte,
+                      NumberOfPtes,
+                      SystemPtePoolType);
+    }
+#endif //DBG
+
+#if DBG
+    if (MmDebug & MM_DBG_SYS_PTES) {
+        DbgPrint("releasing 0x%lx system PTEs at location %lx\n",NumberOfPtes,StartingPte);
+    }
+#endif
+
+    //
+    // Zero PTEs.
+    //
+
+    RtlFillMemoryUlong (StartingPte,
+                        NumberOfPtes * sizeof (MMPTE),
+                        ZeroKernelPte.u.Long);
+
+    //
+    // Acquire system space spin lock to synchronize access.
+    //
+
+    PteOffset = StartingPte - MmSystemPteBase;
+
+    ExAcquireSpinLock ( &MmSystemSpaceLock, &OldIrql );
+
+    if ((SystemPtePoolType == SystemPteSpace) &&
+        (NumberOfPtes <= MM_PTE_TABLE_LIMIT)) {
+
+        Index = MmSysPteTables [NumberOfPtes];
+        NumberOfPtes = MmSysPteIndex [Index];
+
+        if (MmTotalFreeSystemPtes[SystemPteSpace] >= MM_MIN_SYSPTE_FREE) {
+
+            //
+            // Don't add to the pool if the size is greater than 15 + the minimum.
+            //
+
+            i = MmSysPteMinimumFree[Index];
+            if (MmTotalFreeSystemPtes[SystemPteSpace] >= MM_MAX_SYSPTE_FREE) {
+
+                //
+                // Lots of free PTEs, quadrouple the limit.
+                //
+
+                i = i * 4;
+            }
+            i += 15;
+            if (MmSysPteListBySizeCount[Index] <= i) {
+
+#if DBG
+                if (MmDebug & MM_DBG_SYS_PTES) {
+                    PMMPTE PointerPte1;
+
+                    PointerPte1 = &MmFreeSysPteListBySize[Index];
+                    while (PointerPte1->u.List.NextEntry != MM_EMPTY_PTE_LIST) {
+                        PMMPTE PointerFreedPte;
+                        ULONG j;
+
+                        PointerPte1 = MmSystemPteBase + PointerPte1->u.List.NextEntry;
+                        PointerFreedPte = PointerPte1;
+                        for (j = 0; j < MmSysPteIndex[Index]; j++) {
+                            ASSERT (PointerFreedPte->u.Hard.Valid == 0);
+                            PointerFreedPte++;
+                        }
+                    }
+                }
+#endif //DBG
+                MmSysPteListBySizeCount [Index] += 1;
+                PointerPte = MmLastSysPteListBySize[Index];
+                ASSERT (PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST);
+                PointerPte->u.List.NextEntry = PteOffset;
+                MmLastSysPteListBySize[Index] = StartingPte;
+                StartingPte->u.List.NextEntry = MM_EMPTY_PTE_LIST;
+
+#if DBG
+                if (MmDebug & MM_DBG_SYS_PTES) {
+                    PMMPTE PointerPte1;
+                    PointerPte1 = &MmFreeSysPteListBySize[Index];
+                    while (PointerPte1->u.List.NextEntry != MM_EMPTY_PTE_LIST) {
+                        PMMPTE PointerFreedPte;
+                        ULONG j;
+
+                        PointerPte1 = MmSystemPteBase + PointerPte1->u.List.NextEntry;
+                        PointerFreedPte = PointerPte1;
+                        for (j = 0; j < MmSysPteIndex[Index]; j++) {
+                            ASSERT (PointerFreedPte->u.Hard.Valid == 0);
+                            PointerFreedPte++;
+                        }
+                    }
+                }
+#endif //DBG
+                if (NumberOfPtes == 1) {
+                    if (MmFlushPte1 == NULL) {
+                        MmFlushPte1 = StartingPte;
+                    }
+                } else {
+                    (StartingPte + 1)->u.List.NextEntry = MmFlushCounter.u.List.NextEntry;
+                }
+
+                ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql);
+                return;
+            }
+        }
+    }
+
+    MmTotalFreeSystemPtes[SystemPtePoolType] += NumberOfPtes;
+
+    PteOffset = StartingPte - MmSystemPteBase;
+    PointerPte = &MmFirstFreeSystemPte[SystemPtePoolType];
+
+    while (TRUE) {
+        NextPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
+        if (PteOffset < PointerPte->u.List.NextEntry) {
+
+            //
+            // Insert in the list at this point.  The
+            // previous one should point to the new freed set and
+            // the new freed set should point to the place
+            // the previous set points to.
+            //
+            // Attempt to combine the clusters before we
+            // insert.
+            //
+            // Locate the end of the current structure.
+            //
+
+            ASSERT ((StartingPte + NumberOfPtes) <= NextPte);
+
+            PointerFollowingPte = PointerPte + 1;
+            if (PointerPte->u.List.OneEntry) {
+                Size = 1;
+            } else {
+                Size = PointerFollowingPte->u.List.NextEntry;
+            }
+            if ((PointerPte + Size) == StartingPte) {
+
+                //
+                // We can combine the clusters.
+                //
+
+                NumberOfPtes = Size + NumberOfPtes;
+                PointerFollowingPte->u.List.NextEntry = NumberOfPtes;
+                PointerPte->u.List.OneEntry = 0;
+
+                //
+                // Point the starting PTE to the beginning of
+                // the new free set and try to combine with the
+                // following free cluster.
+                //
+
+                StartingPte = PointerPte;
+
+            } else {
+
+                //
+                // Can't combine with previous. Make this Pte the
+                // start of a cluster.
+                //
+
+                //
+                // Point this cluster to the next cluster.
+                //
+
+                StartingPte->u.List.NextEntry = PointerPte->u.List.NextEntry;
+
+                //
+                // Point the current cluster to this cluster.
+                //
+
+                PointerPte->u.List.NextEntry = PteOffset;
+
+                //
+                // Set the size of this cluster.
+                //
+
+                if (NumberOfPtes == 1) {
+                    StartingPte->u.List.OneEntry = 1;
+
+                } else {
+                    StartingPte->u.List.OneEntry = 0;
+                    PointerFollowingPte = StartingPte + 1;
+                    PointerFollowingPte->u.List.NextEntry = NumberOfPtes;
+                }
+            }
+
+            //
+            // Attempt to combine the newly created cluster with
+            // the following cluster.
+            //
+
+            if ((StartingPte + NumberOfPtes) == NextPte) {
+
+                //
+                // Combine with following cluster.
+                //
+
+                //
+                // Set the next cluster to the value contained in the
+                // cluster we are merging into this one.
+                //
+
+                StartingPte->u.List.NextEntry = NextPte->u.List.NextEntry;
+                StartingPte->u.List.OneEntry = 0;
+                PointerFollowingPte = StartingPte + 1;
+
+                if (NextPte->u.List.OneEntry) {
+                    Size = 1;
+
+                } else {
+                    NextPte++;
+                    Size = NextPte->u.List.NextEntry;
+                }
+                PointerFollowingPte->u.List.NextEntry = NumberOfPtes + Size;
+            }
+#if DBG
+            if (MmDebug & MM_DBG_SYS_PTES) {
+                MiDumpSystemPtes(SystemPtePoolType);
+            }
+#endif
+
+#if DBG
+            if (MmDebug & MM_DBG_SYS_PTES) {
+                ASSERT (MmTotalFreeSystemPtes[SystemPtePoolType] ==
+                         MiCountFreeSystemPtes (SystemPtePoolType));
+            }
+#endif //DBG
+            ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );
+            return;
+        }
+
+        //
+        // Point to next freed cluster.
+        //
+
+        PointerPte = NextPte;
+    }
+}
+
+VOID
+MiInitializeSystemPtes (
+    IN PMMPTE StartingPte,
+    IN ULONG NumberOfPtes,
+    IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
+    )
+
+/*++
+
+Routine Description:
+
+    This routine initializes the system PTE pool.
+
+Arguments:
+
+    StartingPte - Supplies the address of the first PTE to put in the pool.
+
+    NumberOfPtes - Supplies the number of PTEs to put in the pool.
+
+Return Value:
+
+    none.
+
+Environment:
+
+    Kernel mode.
+
+--*/
+
+{
+    LONG i;
+    LONG j;
+
+    //
+    // Set the base of the system PTE pool to this PTE.
+    //
+
+    MmSystemPteBase = MiGetPteAddress (0xC0000000);
+    MmSystemPtesStart[SystemPtePoolType] = (ULONG)StartingPte;
+    MmSystemPtesEnd[SystemPtePoolType] = (ULONG)((StartingPte + NumberOfPtes -1));
+
+    if (NumberOfPtes <= 1) {
+
+        //
+        // Not enough PTEs to make a valid chain, just indicate
+        // not PTEs are free.
+        //
+
+        MmFirstFreeSystemPte[SystemPtePoolType] = ZeroKernelPte;
+        MmFirstFreeSystemPte[SystemPtePoolType].u.List.NextEntry =
+                                                                MM_EMPTY_LIST;
+        return;
+
+    }
+
+    //
+    // Zero the system pte pool.
+    //
+
+    RtlFillMemoryUlong (StartingPte,
+                        NumberOfPtes * sizeof (MMPTE),
+                        ZeroKernelPte.u.Long);
+
+    //
+    // The page frame field points to the next cluster.  As we only
+    // have one cluster at initialization time, mark it as the last
+    // cluster.
+    //
+
+    StartingPte->u.List.NextEntry = MM_EMPTY_LIST;
+
+    MmFirstFreeSystemPte[SystemPtePoolType] = ZeroKernelPte;
+    MmFirstFreeSystemPte[SystemPtePoolType].u.List.NextEntry =
+                                                StartingPte - MmSystemPteBase;
+
+    //
+    // Point to the next PTE to fill in the size of this cluster.
+    //
+
+    StartingPte++;
+    *StartingPte = ZeroKernelPte;
+    StartingPte->u.List.NextEntry = NumberOfPtes;
+
+    MmTotalFreeSystemPtes[SystemPtePoolType] = NumberOfPtes;
+    ASSERT (MmTotalFreeSystemPtes[SystemPtePoolType] ==
+                         MiCountFreeSystemPtes (SystemPtePoolType));
+
+    if (SystemPtePoolType == SystemPteSpace) {
+
+        ULONG Lists[MM_SYS_PTE_TABLES_MAX] = {MM_PTE_LIST_1, MM_PTE_LIST_2, MM_PTE_LIST_4, MM_PTE_LIST_8, MM_PTE_LIST_16};
+        PMMPTE PointerPte;
+        ULONG total;
+
+        for (j = 0; j < MM_SYS_PTE_TABLES_MAX ; j++) {
+            MmFreeSysPteListBySize [j].u.List.NextEntry = MM_EMPTY_PTE_LIST;
+            MmLastSysPteListBySize [j] = &MmFreeSysPteListBySize [j];
+        }
+        MmFlushCounter.u.List.NextEntry += 1;
+
+        //
+        // Initialize the by size lists.
+        //
+
+        total = MM_PTE_LIST_1 * MmSysPteIndex[0] +
+                MM_PTE_LIST_2 * MmSysPteIndex[1] +
+                MM_PTE_LIST_4 * MmSysPteIndex[2] +
+                MM_PTE_LIST_8 * MmSysPteIndex[3] +
+                MM_PTE_LIST_16 * MmSysPteIndex[4];
+
+        PointerPte = MiReserveSystemPtes (total,
+                                          SystemPteSpace,
+                                          64*1024,
+                                          0,
+                                          TRUE);
+
+#ifdef MIPS
+        {
+            ULONG inserted;
+
+            //
+            // For MIPS make sure buffers exist at all alignemnts.
+            //
+
+            do {
+                inserted = FALSE;
+                for (i = 0; i < MM_SYS_PTE_TABLES_MAX; i++) {
+                    if (Lists[i]) {
+                        Lists[i] -= 1;
+                        MiReleaseSystemPtes (PointerPte,
+                                             MmSysPteIndex[i],
+                                             SystemPteSpace);
+                        inserted = TRUE;
+                        PointerPte += MmSysPteIndex[i];
+                    }
+                }
+            } while (inserted);
+        }
+
+#else
+        for (i = (MM_SYS_PTE_TABLES_MAX - 1); i >= 0; i--) {
+            do {
+                Lists[i] -= 1;
+                MiReleaseSystemPtes (PointerPte,
+                                     MmSysPteIndex[i],
+                                     SystemPteSpace);
+                PointerPte += MmSysPteIndex[i];
+            } while (Lists[i] != 0  );
+        }
+#endif //MIPS
+        MmFlushCounter.u.List.NextEntry += 1;
+        MmFlushPte1 = NULL;
+    }
+
+    return;
+}
+
+
+#if DBG
+
+VOID
+MiDumpSystemPtes (
+    IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
+    )
+
+
+{
+    PMMPTE PointerPte;
+    PMMPTE PointerNextPte;
+    ULONG ClusterSize;
+    PMMPTE EndOfCluster;
+
+    PointerPte = &MmFirstFreeSystemPte[SystemPtePoolType];
+    if (PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) {
+        return;
+    }
+
+    PointerPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
+
+    for (;;) {
+        if (PointerPte->u.List.OneEntry) {
+            ClusterSize = 1;
+        } else {
+            PointerNextPte = PointerPte + 1;
+            ClusterSize = PointerNextPte->u.List.NextEntry;
+        }
+
+        EndOfCluster = PointerPte + (ClusterSize - 1);
+
+        DbgPrint("System Pte at %lx for %lx entries (%lx)\n",PointerPte,
+                ClusterSize, EndOfCluster);
+
+        if (PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) {
+            break;
+        }
+
+        PointerPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
+    }
+    return;
+}
+
+ULONG
+MiCountFreeSystemPtes (
+    IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType
+    )
+
+{
+    PMMPTE PointerPte;
+    PMMPTE PointerNextPte;
+    ULONG ClusterSize;
+    PMMPTE EndOfCluster;
+    ULONG FreeCount = 0;
+
+    PointerPte = &MmFirstFreeSystemPte[SystemPtePoolType];
+    if (PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) {
+        return 0;
+    }
+
+    PointerPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
+
+    for (;;) {
+        if (PointerPte->u.List.OneEntry) {
+            ClusterSize = 1;
+        } else {
+            PointerNextPte = PointerPte + 1;
+            ClusterSize = PointerNextPte->u.List.NextEntry;
+        }
+
+        FreeCount += ClusterSize;
+
+        EndOfCluster = PointerPte + (ClusterSize - 1);
+
+        if (PointerPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) {
+            break;
+        }
+
+        PointerPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
+    }
+    return FreeCount;
+}
+
+#endif //DBG