path: root/private/ntos/mm/iosup.c

/*++

Copyright (c) 1989  Microsoft Corporation

Module Name:

   iosup.c

Abstract:

    This module contains routines which provide support for the I/O system.

Author:

    Lou Perazzoli (loup) 25-Apr-1989

Revision History:

--*/

#include "mi.h"

#undef MmIsRecursiveIoFault

BOOLEAN
MmIsRecursiveIoFault(
    VOID
    );

BOOLEAN
MiCheckForContiguousMemory (
    IN PVOID BaseAddress,
    IN ULONG SizeInPages,
    IN ULONG HighestPfn
    );

PVOID
MiFindContiguousMemory (
    IN ULONG HighestPfn,
    IN ULONG SizeInPages
    );

PVOID
MiMapLockedPagesInUserSpace (
     IN PMDL MemoryDescriptorList,
     IN PVOID StartingVa
     );

VOID
MiUnmapLockedPagesInUserSpace (
     IN PVOID BaseAddress,
     IN PMDL MemoryDescriptorList
     );

VOID
MiFlushTb (
    VOID
    );


#ifdef ALLOC_PRAGMA
#pragma alloc_text(PAGE, MmLockPagableDataSection)
#pragma alloc_text(PAGE, MiLookupDataTableEntry)
#pragma alloc_text(PAGE, MiMapLockedPagesInUserSpace)
#pragma alloc_text(PAGE, MmSetBankedSection)
#pragma alloc_text(PAGE, MmUnmapIoSpace)
#pragma alloc_text(PAGE, MmMapVideoDisplay)
#pragma alloc_text(PAGE, MmUnmapVideoDisplay)

#pragma alloc_text(PAGELK, MiUnmapLockedPagesInUserSpace)
#pragma alloc_text(PAGELK, MmAllocateNonCachedMemory)
#pragma alloc_text(PAGELK, MmFreeNonCachedMemory)
#pragma alloc_text(PAGELK, MiFindContiguousMemory)
#pragma alloc_text(PAGELK, MmLockPagedPool)
#pragma alloc_text(PAGELK, MmUnlockPagedPool)
#endif

extern POOL_DESCRIPTOR NonPagedPoolDescriptor;

extern ULONG MmAllocatedNonPagedPool;

extern ULONG MmDelayPageFaults;

KEVENT MmCollidedLockEvent;
ULONG MmCollidedLockWait;

ULONG MmLockPagesCount;

ULONG MmLockedCode;

#ifdef LARGE_PAGES
ULONG MmLargeVideoMapped;
#endif

#if DBG
ULONG MmReferenceCountCheck = 75;
#endif //DBG



VOID
MmProbeAndLockPages (
     IN OUT PMDL MemoryDescriptorList,
     IN KPROCESSOR_MODE AccessMode,
     IN LOCK_OPERATION Operation
     )

/*++

Routine Description:

    This routine probes the specified pages, makes the pages resident and
    locks the physical pages mapped by the virtual pages in memory.  The
    Memory descriptor list is updated to describe the physical pages.

Arguments:

    MemoryDescriptorList - Supplies a pointer to a Memory Descriptor List
                            (MDL). The supplied MDL must supply a virtual
                            address, byte offset and length field.  The
                            physical page portion of the MDL is updated when
                            the pages are locked in memory.

    AccessMode - Supplies the access mode in which to probe the arguments.
                 One of KernelMode or UserMode.

    Operation - Supplies the operation type.  One of IoReadAccess, IoWriteAccess
                or IoModifyAccess.

Return Value:

    None - exceptions are raised.

Environment:

    Kernel mode.  APC_LEVEL and below for pageable addresses,
                  DISPATCH_LEVEL and below for non-pageable addresses.

--*/

{
    PULONG Page;
    PMMPTE PointerPte;
    PMMPTE PointerPte1;
    PMMPTE PointerPde;
    PVOID Va;
    PVOID EndVa;
    PMMPFN Pfn1 ;
    ULONG PageFrameIndex;
    PEPROCESS CurrentProcess;
    KIRQL OldIrql;
    ULONG NumberOfPagesToLock;
    NTSTATUS status;

    ASSERT (MemoryDescriptorList->ByteCount != 0);
    ASSERT (((ULONG)MemoryDescriptorList->StartVa & (PAGE_SIZE - 1)) == 0);
    ASSERT (((ULONG)MemoryDescriptorList->ByteOffset & ~(PAGE_SIZE - 1)) == 0);

    ASSERT ((MemoryDescriptorList->MdlFlags & (
                    MDL_PAGES_LOCKED |
                    MDL_MAPPED_TO_SYSTEM_VA |
                    MDL_SOURCE_IS_NONPAGED_POOL |
                    MDL_PARTIAL |
                    MDL_SCATTER_GATHER_VA |
                    MDL_IO_SPACE)) == 0);

    Page = (PULONG)(MemoryDescriptorList + 1);

    Va = (PCHAR) MemoryDescriptorList->StartVa + MemoryDescriptorList->ByteOffset;

    PointerPte = MiGetPteAddress (Va);
    PointerPte1 = PointerPte;

    //
    // Endva is one byte past the end of the buffer, if ACCESS_MODE is not
    // kernel, make sure the EndVa is in user space AND the byte count
    // does not cause it to wrap.
    //

    EndVa = (PVOID)(((PCHAR)MemoryDescriptorList->StartVa +
                            MemoryDescriptorList->ByteOffset) +
                            MemoryDescriptorList->ByteCount);

    if ((AccessMode != KernelMode) &&
        ((EndVa > (PVOID)MM_USER_PROBE_ADDRESS) || (Va >= EndVa))) {
        *Page = MM_EMPTY_LIST;
        ExRaiseStatus (STATUS_ACCESS_VIOLATION);
        return;
    }

    //
    // There is an optimization which could be performed here.  If
    // the operation is for WriteAccess and the complete page is
    // being modified, we can remove the current page, if it is not
    // resident, and substitute a demand zero page.
    // Note, that after analysis by marking the thread and then
    // noting if a page read was done, this rarely occurs.
    //
    //

    MemoryDescriptorList->Process = (PEPROCESS)NULL;

    if (!MI_IS_PHYSICAL_ADDRESS(Va)) {
        do {

            *Page = MM_EMPTY_LIST;
            PointerPde = MiGetPdeAddress (Va);

            //
            // Make sure the page is resident.
            //

            if ((PointerPde->u.Hard.Valid == 0) ||
                (PointerPte1->u.Hard.Valid == 0)) {

                status = MmAccessFault (FALSE, Va, KernelMode);
            }

            //
            // Touch the page in case the previous fault caused
            // an access violation.  This is quicker than checking
            // the status code.
            //

            *(volatile CHAR *)Va;

            if ((Operation != IoReadAccess) &&
                (Va <= MM_HIGHEST_USER_ADDRESS)) {

                //
                // Probe for write access as well.
                //

                ProbeForWriteChar ((PCHAR)Va);
            }

            Va = (PVOID)(((ULONG)(PCHAR)Va + PAGE_SIZE) & ~(PAGE_SIZE - 1));
            Page += 1;
            PointerPte1 += 1;
        } while (Va < EndVa);
    }

    Va = (PVOID)(MemoryDescriptorList->StartVa);
    Page = (PULONG)(MemoryDescriptorList + 1);

    //
    // Indicate that this is a write operation.
    //

    if (Operation != IoReadAccess) {
        MemoryDescriptorList->MdlFlags |= MDL_WRITE_OPERATION;
    } else {
        MemoryDescriptorList->MdlFlags &= ~(MDL_WRITE_OPERATION);
    }

    //
    // Acquire the PFN database lock.
    //

    LOCK_PFN2 (OldIrql);

    if (Va <= MM_HIGHEST_USER_ADDRESS) {

        //
        // These are addresses with user space, check to see if the
        // working set size will allow these pages to be locked.
        //

        CurrentProcess = PsGetCurrentProcess ();
        NumberOfPagesToLock =
                        (((ULONG)EndVa - ((ULONG)Va + 1)) >> PAGE_SHIFT) + 1;

        PageFrameIndex = NumberOfPagesToLock + CurrentProcess->NumberOfLockedPages;

        if ((PageFrameIndex >
           (CurrentProcess->Vm.MaximumWorkingSetSize - MM_FLUID_WORKING_SET))
               &&
            ((MmLockPagesCount + NumberOfPagesToLock) > MmLockPagesLimit)) {

            UNLOCK_PFN (OldIrql);
            ExRaiseStatus (STATUS_WORKING_SET_QUOTA);
            return;
        }

        CurrentProcess->NumberOfLockedPages = PageFrameIndex;
        MmLockPagesCount += NumberOfPagesToLock;
        MemoryDescriptorList->Process = CurrentProcess;
    }

    MemoryDescriptorList->MdlFlags |= MDL_PAGES_LOCKED;

    do {

        PointerPde = MiGetPdeAddress (Va);

        if (MI_IS_PHYSICAL_ADDRESS(Va)) {

            //
            // On certains architectures (e.g., MIPS) virtual addresses
            // may be physical and hence have no corresponding PTE.
            //

            PageFrameIndex = MI_CONVERT_PHYSICAL_TO_PFN (Va);

        } else {

            while ((PointerPde->u.Hard.Valid == 0) ||
                   (PointerPte->u.Hard.Valid == 0)) {

                //
                // PDE is not resident, release PFN lock touch the page and make
                // it appear.
                //

                UNLOCK_PFN (OldIrql);

                status = MmAccessFault (FALSE, Va, KernelMode);

                if (!NT_SUCCESS(status)) {

                    //
                    // An exception occurred.  Unlock the pages locked
                    // so far.
                    //

                    MmUnlockPages (MemoryDescriptorList);

                    //
                    // Raise an exception of access violation to the caller.
                    //

                    ExRaiseStatus (status);
                    return;
                }

                LOCK_PFN (OldIrql);
            }

            PageFrameIndex = PointerPte->u.Hard.PageFrameNumber;
        }

        if (PageFrameIndex > MmHighestPhysicalPage) {

            //
            // This is an I/O space address don't allow operations
            // on addresses not in the PFN database.
            //

            MemoryDescriptorList->MdlFlags |= MDL_IO_SPACE;

        } else {
            ASSERT ((MemoryDescriptorList->MdlFlags & MDL_IO_SPACE) == 0);

            Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
            Pfn1->u3.e2.ReferenceCount += 1;

            ASSERT (Pfn1->u3.e2.ReferenceCount < MmReferenceCountCheck);
        }

        *Page = PageFrameIndex;

        Page += 1;
        PointerPte += 1;
        Va = (PVOID)((PCHAR)Va + PAGE_SIZE);
    } while (Va < EndVa);

    UNLOCK_PFN2 (OldIrql);

    return;
}

NTKERNELAPI
VOID
MmProbeAndLockSelectedPages (
    IN OUT PMDL MemoryDescriptorList,
    IN PFILE_SEGMENT_ELEMENT SegmentArray,
    IN KPROCESSOR_MODE AccessMode,
    IN LOCK_OPERATION Operation
    )

/*++

Routine Description:

    This routine probes the specified pages, makes the pages resident and
    locks the physical pages mapped by the virtual pages in memory.  The
    Memory descriptor list is updated to describe the physical pages.

Arguments:

    MemoryDescriptorList - Supplies a pointer to a Memory Descriptor List
                            (MDL). The MDL must supply the length. The
                            physical page portion of the MDL is updated when
                            the pages are locked in memory.

    SegmentArray - Supplies a pointer to a list of buffer segments to be 
                   probed and locked.

    AccessMode - Supplies the access mode in which to probe the arguments.
                 One of KernelMode or UserMode.

    Operation - Supplies the operation type.  One of IoReadAccess, IoWriteAccess
                or IoModifyAccess.

Return Value:

    None - exceptions are raised.

Environment:

    Kernel mode.  APC_LEVEL and below.

--*/

{
    PMDL TempMdl;
    ULONG MdlHack[(sizeof(MDL)/4) + 1];
    PULONG Page;
    PFILE_SEGMENT_ELEMENT LastSegment;

    PAGED_CODE();
    ASSERT (MemoryDescriptorList->ByteCount != 0);
    ASSERT (((ULONG)MemoryDescriptorList->ByteOffset & ~(PAGE_SIZE - 1)) == 0);

    ASSERT ((MemoryDescriptorList->MdlFlags & (
                    MDL_PAGES_LOCKED |
                    MDL_MAPPED_TO_SYSTEM_VA |
                    MDL_SOURCE_IS_NONPAGED_POOL |
                    MDL_PARTIAL |
                    MDL_SCATTER_GATHER_VA |
                    MDL_IO_SPACE)) == 0);

    //
    // Initialize TempMdl.
    //

    TempMdl = (PMDL) &MdlHack;
    MmInitializeMdl( TempMdl, NULL, PAGE_SIZE );

    Page = (PULONG) (MemoryDescriptorList + 1);

    //
    // Calculate the end of the segment list.
    //

    LastSegment = SegmentArray + 
                  BYTES_TO_PAGES(MemoryDescriptorList->ByteCount);

    //
    // Build a small Mdl for each segement and call probe and lock pages.
    // Then copy the PFNs to the real mdl.
    //

    while (SegmentArray < LastSegment) {

        TempMdl->MdlFlags = 0;
        TempMdl->StartVa = (PVOID) SegmentArray->Buffer;

        SegmentArray++;
        MmProbeAndLockPages( TempMdl, AccessMode, Operation );

        *Page++ = *((PULONG) (TempMdl + 1));
    }

    //
    // Copy the flags and process fields. 
    //

    MemoryDescriptorList->MdlFlags = TempMdl->MdlFlags;
    MemoryDescriptorList->Process = TempMdl->Process;

#ifdef _MIPS_

    //
    // Becuase the caches are virtual on mips they need to be completely
    // flushed.  Only the first level dcache needs to be swept; however,
    // the only kernel interface to do this with is sweep I-cache range
    // since it sweeps the both the first level I and D caches.
    //

    KeSweepIcacheRange( TRUE, NULL, KeGetPcr()->FirstLevelDcacheSize );

    //
    // Set a flag the MDL to indicate this MDL is a scatter/gather MDL.
    //

    MemoryDescriptorList->MdlFlags |= MDL_SCATTER_GATHER_VA;

#endif
}

VOID
MmUnlockPages (
     IN OUT PMDL MemoryDescriptorList
     )

/*++

Routine Description:

    This routine unlocks physical pages which are described by a Memory
    Descriptor List.

Arguments:

    MemoryDescriptorList - Supplies a pointer to a memory description list
                            (MDL). The supplied MDL must have been supplied
                            to MmLockPages to lock the pages down.  As the
                            pages are unlocked, the MDL is updated.

Return Value:

    None.

Environment:

    Kernel mode, IRQL of DISPATCH_LEVEL or below.

--*/

{
    ULONG NumberOfPages;
    PULONG Page;
    PVOID StartingVa;
    KIRQL OldIrql;
    PMMPFN Pfn1;

    ASSERT ((MemoryDescriptorList->MdlFlags & MDL_PAGES_LOCKED) != 0);
    ASSERT ((MemoryDescriptorList->MdlFlags & MDL_SOURCE_IS_NONPAGED_POOL) == 0);
    ASSERT ((MemoryDescriptorList->MdlFlags & MDL_PARTIAL) == 0);
    ASSERT (MemoryDescriptorList->ByteCount != 0);

    if (MemoryDescriptorList->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) {

        //
        // This MDL has been mapped into system space, unmap now.
        //

        MmUnmapLockedPages (MemoryDescriptorList->MappedSystemVa,
                            MemoryDescriptorList);
    }

    StartingVa = (PVOID)((PCHAR)MemoryDescriptorList->StartVa +
                        MemoryDescriptorList->ByteOffset);

    Page = (PULONG)(MemoryDescriptorList + 1);
    NumberOfPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES(StartingVa,
                                              MemoryDescriptorList->ByteCount);
    ASSERT (NumberOfPages != 0);

    LOCK_PFN2 (OldIrql);

    if (MemoryDescriptorList->Process != NULL) {
        MemoryDescriptorList->Process->NumberOfLockedPages -= NumberOfPages;
        MmLockPagesCount -= NumberOfPages;
        ASSERT ((LONG)MemoryDescriptorList->Process->NumberOfLockedPages >= 0);
    }

    if ((MemoryDescriptorList->MdlFlags & MDL_IO_SPACE) == 0) {

        //
        // Only unlock if not I/O space.
        //

        do {

            if (*Page == MM_EMPTY_LIST) {

                //
                // There are no more locked pages.
                //

                UNLOCK_PFN2 (OldIrql);
                return;
            }
            ASSERT ((*Page <= MmHighestPhysicalPage) &&
                    (*Page >= MmLowestPhysicalPage));

            //
            // If this was a write operation set the modified bit in the
            // pfn database.
            //

            if (MemoryDescriptorList->MdlFlags & MDL_WRITE_OPERATION) {
                Pfn1 = MI_PFN_ELEMENT (*Page);
                Pfn1->u3.e1.Modified = 1;
                if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
                             (Pfn1->u3.e1.WriteInProgress == 0)) {
                    MiReleasePageFileSpace (Pfn1->OriginalPte);
                    Pfn1->OriginalPte.u.Soft.PageFileHigh = 0;
                }
            }

            MiDecrementReferenceCount (*Page);
            *Page = MM_EMPTY_LIST;
            Page += 1;
            NumberOfPages -= 1;
        } while (NumberOfPages != 0);
    }
    UNLOCK_PFN2 (OldIrql);

    MemoryDescriptorList->MdlFlags &= ~MDL_PAGES_LOCKED;

    return;
}

VOID
MmBuildMdlForNonPagedPool (
    IN OUT PMDL MemoryDescriptorList
    )

/*++

Routine Description:

    This routine fills in the "pages" portion of the MDL using the PFN
    numbers corresponding the the buffers which resides in non-paged pool.

    Unlike MmProbeAndLockPages, there is no corresponding unlock as no
    reference counts are incremented as the buffers being in nonpaged
    pool are always resident.

Arguments:

    MemoryDescriptorList - Supplies a pointer to a Memory Descriptor List
                            (MDL). The supplied MDL must supply a virtual
                            address, byte offset and length field.  The
                            physical page portion of the MDL is updated when
                            the pages are locked in memory.  The virtual
                            address must be within the non-paged portion
                            of the system space.

Return Value:

    None.

Environment:

    Kernel mode, IRQL of DISPATCH_LEVEL or below.

--*/

{
    PULONG Page;
    PMMPTE PointerPte;
    PMMPTE LastPte;
    PVOID EndVa;
    ULONG PageFrameIndex;

    Page = (PULONG)(MemoryDescriptorList + 1);

    ASSERT (MemoryDescriptorList->ByteCount != 0);
    ASSERT ((MemoryDescriptorList->MdlFlags & (
                    MDL_PAGES_LOCKED |
                    MDL_MAPPED_TO_SYSTEM_VA |
                    MDL_SOURCE_IS_NONPAGED_POOL |
                    MDL_PARTIAL)) == 0);

    MemoryDescriptorList->Process = (PEPROCESS)NULL;

    //
    // Endva is last byte of the buffer.
    //

    MemoryDescriptorList->MdlFlags |= MDL_SOURCE_IS_NONPAGED_POOL;

    MemoryDescriptorList->MappedSystemVa =
            (PVOID)((PCHAR)MemoryDescriptorList->StartVa +
                                           MemoryDescriptorList->ByteOffset);

    EndVa = (PVOID)(((PCHAR)MemoryDescriptorList->MappedSystemVa +
                            MemoryDescriptorList->ByteCount - 1));

    LastPte = MiGetPteAddress (EndVa);

    ASSERT (MmIsNonPagedSystemAddressValid (MemoryDescriptorList->StartVa));

    PointerPte = MiGetPteAddress (MemoryDescriptorList->StartVa);

    if (MI_IS_PHYSICAL_ADDRESS(EndVa)) {
        PageFrameIndex = MI_CONVERT_PHYSICAL_TO_PFN (
                                MemoryDescriptorList->StartVa);

        do {
            *Page = PageFrameIndex;
            Page += 1;
            PageFrameIndex += 1;
            PointerPte += 1;
        } while (PointerPte <= LastPte);
    } else {
        do {
            PageFrameIndex = PointerPte->u.Hard.PageFrameNumber;
            *Page = PageFrameIndex;
            Page += 1;
            PointerPte += 1;
        } while (PointerPte <= LastPte);
    }

    return;
}

PVOID
MmMapLockedPages (
     IN PMDL MemoryDescriptorList,
     IN KPROCESSOR_MODE AccessMode
     )

/*++

Routine Description:

    This function maps physical pages described by a memory description
    list into the system virtual address space or the user portion of
    the virtual address space.

Arguments:

    MemoryDescriptorList - Supplies a valid Memory Descriptor List which has
                            been updated by MmProbeAndLockPages.


    AccessMode - Supplies an indicator of where to map the pages;
                 KernelMode indicates that the pages should be mapped in the
                 system part of the address space, UserMode indicates the
                 pages should be mapped in the user part of the address space.

Return Value:

    Returns the base address where the pages are mapped.  The base address
    has the same offset as the virtual address in the MDL.

    This routine will raise an exception if the processor mode is USER_MODE
    and quota limits or VM limits are exceeded.

Environment:

    Kernel mode.  DISPATCH_LEVEL or below if access mode is KernelMode,
                APC_LEVEL or below if access mode is UserMode.

--*/

{
    ULONG NumberOfPages;
    ULONG SavedPageCount;
    PULONG Page;
    PMMPTE PointerPte;
    PVOID BaseVa;
    MMPTE TempPte;
    PVOID StartingVa;
    PMMPFN Pfn2;
    KIRQL OldIrql;

    StartingVa = (PVOID)((PCHAR)MemoryDescriptorList->StartVa +
                        MemoryDescriptorList->ByteOffset);

    ASSERT (MemoryDescriptorList->ByteCount != 0);

    if (AccessMode == KernelMode) {

        Page = (PULONG)(MemoryDescriptorList + 1);
        NumberOfPages = COMPUTE_PAGES_SPANNED (StartingVa,
                                               MemoryDescriptorList->ByteCount);
        SavedPageCount = NumberOfPages;

        //
        // Map the pages into the system part of the address space as
        // kernel read/write.
        //

        ASSERT ((MemoryDescriptorList->MdlFlags & (
                        MDL_MAPPED_TO_SYSTEM_VA |
                        MDL_SOURCE_IS_NONPAGED_POOL |
                        MDL_PARTIAL_HAS_BEEN_MAPPED)) == 0);
        ASSERT ((MemoryDescriptorList->MdlFlags & (
                        MDL_PAGES_LOCKED |
                        MDL_PARTIAL)) != 0);

#if defined(_ALPHA_)

        //
        // See if KSEG0 can be used to map this.
        //

        if ((NumberOfPages == 1) &&
            (*Page < ((1*1024*1024*1024) >> PAGE_SHIFT))) {
            BaseVa = (PVOID)(KSEG0_BASE + (*Page << PAGE_SHIFT) +
                            MemoryDescriptorList->ByteOffset);
            MemoryDescriptorList->MappedSystemVa = BaseVa;
            MemoryDescriptorList->MdlFlags |= MDL_MAPPED_TO_SYSTEM_VA;

            goto Update;
        }
#endif //ALPHA

#if defined(_MIPS_)

        //
        // See if KSEG0 can be used to map this.
        //

        if ((NumberOfPages == 1) &&
            (MI_GET_PAGE_COLOR_FROM_VA (MemoryDescriptorList->StartVa) ==
                (MM_COLOR_MASK & *Page)) &&
            (*Page < ((512*1024*1024) >> PAGE_SHIFT))) {
            BaseVa = (PVOID)(KSEG0_BASE + (*Page << PAGE_SHIFT) +
                            MemoryDescriptorList->ByteOffset);
            MemoryDescriptorList->MappedSystemVa = BaseVa;
            MemoryDescriptorList->MdlFlags |= MDL_MAPPED_TO_SYSTEM_VA;

            goto Update;
        }
#endif //MIPS


#if defined(_X86_)

        //
        // See if KSEG0 can be used to map this.
        //

        if ((NumberOfPages == 1) &&
            (*Page < MmKseg2Frame)) {
            BaseVa = (PVOID)(MM_KSEG0_BASE + (*Page << PAGE_SHIFT) +
                            MemoryDescriptorList->ByteOffset);
            MemoryDescriptorList->MappedSystemVa = BaseVa;
            MemoryDescriptorList->MdlFlags |= MDL_MAPPED_TO_SYSTEM_VA;

            goto Update;
        }
#endif //X86

        PointerPte = MiReserveSystemPtes (
                                    NumberOfPages,
                                    SystemPteSpace,
                                    MM_COLOR_ALIGNMENT,
                                    ((ULONG)MemoryDescriptorList->StartVa &
                                                       MM_COLOR_MASK_VIRTUAL),
                        MemoryDescriptorList->MdlFlags & MDL_MAPPING_CAN_FAIL ? 0 : 1);
        if (PointerPte == NULL) {

            //
            // Not enough system PTES are available.
            //

            return NULL;
        }
        BaseVa = (PVOID)((PCHAR)MiGetVirtualAddressMappedByPte (PointerPte) +
                                MemoryDescriptorList->ByteOffset);

        TempPte = ValidKernelPte;

#if defined(_MIPS_)
        
        //
        // If this is a Scatter/Gather Mdl then disable caching since the
        // page colors will be wrong in the MDL.
        //

        if (MemoryDescriptorList->MdlFlags & MDL_SCATTER_GATHER_VA) {
            MI_DISABLE_CACHING( TempPte );
        }
#endif

#if DBG
        LOCK_PFN2 (OldIrql);
#endif //DBG

        do {

            if (*Page == MM_EMPTY_LIST) {
                break;
            }
            ASSERT (*Page != 0);
            TempPte.u.Hard.PageFrameNumber = *Page;
            ASSERT (PointerPte->u.Hard.Valid == 0);

#if DBG
            if ((MemoryDescriptorList->MdlFlags & MDL_IO_SPACE) == 0) {
                Pfn2 = MI_PFN_ELEMENT (*Page);
                ASSERT (Pfn2->u3.e2.ReferenceCount != 0);
                Pfn2->u3.e2.ReferenceCount += 1;
                ASSERT (Pfn2->u3.e2.ReferenceCount < MmReferenceCountCheck);
                ASSERT ((((ULONG)PointerPte >> PTE_SHIFT) & MM_COLOR_MASK) ==
                     (((ULONG)Pfn2->u3.e1.PageColor)));
            }
#endif //DBG

            *PointerPte = TempPte;
            Page++;
            PointerPte++;
            NumberOfPages -= 1;
        } while (NumberOfPages != 0);
#if DBG
        UNLOCK_PFN2 (OldIrql);
#endif //DBG

        ExAcquireSpinLock ( &MmSystemSpaceLock, &OldIrql );
        if (MemoryDescriptorList->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) {

            //
            //
            // Another thread must have already mapped this.
            // Clean up the system ptes and release them.
            //

            ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );

#if DBG
        if ((MemoryDescriptorList->MdlFlags & MDL_IO_SPACE ) == 0) {
            PMMPFN Pfn3;
            ULONG j;
            PULONG Page1;

            Page1 = (PULONG)(MemoryDescriptorList + 1);
            for (j = 0; j < SavedPageCount ; j++ ) {
                if (*Page == MM_EMPTY_LIST) {
                    break;
                }
                Pfn3 = MI_PFN_ELEMENT (*Page1);
                Page1 += 1;
                ASSERT (Pfn3->u3.e2.ReferenceCount > 1);
                Pfn3->u3.e2.ReferenceCount -= 1;
                ASSERT (Pfn3->u3.e2.ReferenceCount < 256);
            }
        }
#endif //DBG
            PointerPte = MiGetPteAddress (BaseVa);

            MiReleaseSystemPtes (PointerPte,
                                 SavedPageCount,
                                 SystemPteSpace);

            return MemoryDescriptorList->MappedSystemVa;
        }

        MemoryDescriptorList->MappedSystemVa = BaseVa;
        *(volatile ULONG *)&MmLockPagesCount;  //need to force order.
        MemoryDescriptorList->MdlFlags |= MDL_MAPPED_TO_SYSTEM_VA;
        ExReleaseSpinLock ( &MmSystemSpaceLock, OldIrql );


#if defined(_MIPS_) || defined(_ALPHA_) || defined (_X86_)
Update:
#endif

        if ((MemoryDescriptorList->MdlFlags & MDL_PARTIAL) != 0) {
            MemoryDescriptorList->MdlFlags |= MDL_PARTIAL_HAS_BEEN_MAPPED;
        }

        return BaseVa;

    } else {

        return MiMapLockedPagesInUserSpace (MemoryDescriptorList, StartingVa);
    }
}


PVOID
MiMapLockedPagesInUserSpace (
     IN PMDL MemoryDescriptorList,
     IN PVOID StartingVa
     )

/*++

Routine Description:

    This function maps physical pages described by a memory description
    list into the system virtual address space or the user portion of
    the virtual address space.

Arguments:

    MemoryDescriptorList - Supplies a valid Memory Descriptor List which has
                            been updated by MmProbeAndLockPages.


    StartingVa - Supplies the starting address.

Return Value:

    Returns the base address where the pages are mapped.  The base address
    has the same offset as the virtual address in the MDL.

    This routine will raise an exception if the processor mode is USER_MODE
    and quota limits or VM limits are exceeded.

Environment:

    Kernel mode.  APC_LEVEL or below.

--*/

{
    ULONG NumberOfPages;
    PULONG Page;
    PMMPTE PointerPte;
    PMMPTE PointerPde;
    PVOID BaseVa;
    MMPTE TempPte;
    PVOID EndingAddress;
    PMMVAD Vad;
    PEPROCESS Process;
    PMMPFN Pfn2;

    PAGED_CODE ();
    Page = (PULONG)(MemoryDescriptorList + 1);
    NumberOfPages = COMPUTE_PAGES_SPANNED (StartingVa,
                                           MemoryDescriptorList->ByteCount);

    if (MemoryDescriptorList->MdlFlags & MDL_IO_SPACE) {
        ExRaiseStatus (STATUS_INVALID_ADDRESS);
    }

    //
    // Map the pages into the user part of the address as user
    // read/write no-delete.
    //

    TempPte = ValidUserPte;

    Process = PsGetCurrentProcess ();

    //
    // Get the working set mutex and address creation mutex.
    //

    LOCK_WS_AND_ADDRESS_SPACE (Process);

    try {

        Vad = (PMMVAD)NULL;
        BaseVa = MiFindEmptyAddressRange ( (NumberOfPages * PAGE_SIZE),
                                            X64K,
                                            0 );

        EndingAddress = (PVOID)((PCHAR)BaseVa + (NumberOfPages * PAGE_SIZE) - 1);

        Vad = ExAllocatePoolWithTag (NonPagedPool, sizeof(MMVAD), ' daV');

        if (Vad == NULL) {
            BaseVa = NULL;
            goto Done;
        }

        Vad->StartingVa = BaseVa;
        Vad->EndingVa = EndingAddress;
        Vad->ControlArea = NULL;
        Vad->FirstPrototypePte = NULL;
        Vad->u.LongFlags = 0;
        Vad->u.VadFlags.Protection = MM_READWRITE;
        Vad->u.VadFlags.PhysicalMapping = 1;
        Vad->u.VadFlags.PrivateMemory = 1;
        Vad->Banked = NULL;
        MiInsertVad (Vad);

    } except (EXCEPTION_EXECUTE_HANDLER) {
        if (Vad != (PMMVAD)NULL) {
            ExFreePool (Vad);
        }
        BaseVa = NULL;
        goto Done;
    }

    //
    // Get the working set mutex and address creation mutex.
    //

    PointerPte = MiGetPteAddress (BaseVa);

    do {

        if (*Page == MM_EMPTY_LIST) {
            break;
        }

        ASSERT (*Page != 0);
        ASSERT ((*Page <= MmHighestPhysicalPage) &&
                (*Page >= MmLowestPhysicalPage));

        PointerPde = MiGetPteAddress (PointerPte);
        MiMakePdeExistAndMakeValid(PointerPde, Process, FALSE);

        ASSERT (PointerPte->u.Hard.Valid == 0);
        TempPte.u.Hard.PageFrameNumber = *Page;
        *PointerPte = TempPte;

        //
        // A PTE just went from not present, not transition to
        // present.  The share count and valid count must be
        // updated in the page table page which contains this
        // Pte.
        //

        Pfn2 = MI_PFN_ELEMENT(PointerPde->u.Hard.PageFrameNumber);
        Pfn2->u2.ShareCount += 1;

        //
        // Another zeroed PTE has become non-zero.
        //

        MmWorkingSetList->UsedPageTableEntries
                            [MiGetPteOffset(PointerPte)] += 1;

        ASSERT (MmWorkingSetList->UsedPageTableEntries
                            [MiGetPteOffset(PointerPte)] <= PTE_PER_PAGE);

        Page++;
        PointerPte++;
        NumberOfPages -= 1;
    } while (NumberOfPages != 0);

Done:
    UNLOCK_WS (Process);
    UNLOCK_ADDRESS_SPACE (Process);
    if (BaseVa == NULL) {
        ExRaiseStatus (STATUS_INSUFFICIENT_RESOURCES);
    }

    return BaseVa;
}


VOID
MmUnmapLockedPages (
     IN PVOID BaseAddress,
     IN PMDL MemoryDescriptorList
     )

/*++

Routine Description:

    This routine unmaps locked pages which were previously mapped via
    a MmMapLockedPages function.

Arguments:

    BaseAddress - Supplies the base address where the pages were previously
                  mapped.

    MemoryDescriptorList - Supplies a valid Memory Descriptor List which has
                            been updated by MmProbeAndLockPages.

Return Value:

    None.

Environment:

    Kernel mode.  DISPATCH_LEVEL or below if base address is within system space;
                APC_LEVEL or below if base address is user space.

--*/

{
    ULONG NumberOfPages;
    ULONG i;
    PULONG Page;
    PMMPTE PointerPte;
    PMMPTE PointerBase;
    PVOID StartingVa;
    KIRQL OldIrql;

    ASSERT (MemoryDescriptorList->ByteCount != 0);
    ASSERT ((MemoryDescriptorList->MdlFlags & MDL_PARENT_MAPPED_SYSTEM_VA) == 0);

    if (MI_IS_PHYSICAL_ADDRESS (BaseAddress)) {

        //
        // MDL is not mapped into virtual space, just clear the fields
        // and return.
        //

        MemoryDescriptorList->MdlFlags &= ~(MDL_MAPPED_TO_SYSTEM_VA |
                                            MDL_PARTIAL_HAS_BEEN_MAPPED);
        return;
    }

    if (BaseAddress > MM_HIGHEST_USER_ADDRESS) {

        StartingVa = (PVOID)((PCHAR)MemoryDescriptorList->StartVa +
                            MemoryDescriptorList->ByteOffset);

        NumberOfPages = COMPUTE_PAGES_SPANNED (StartingVa,
                                               MemoryDescriptorList->ByteCount);

        PointerBase = MiGetPteAddress (BaseAddress);


        ASSERT ((MemoryDescriptorList->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) != 0);


#if DBG
        PointerPte = PointerBase;
        i = NumberOfPages;
        Page = (PULONG)(MemoryDescriptorList + 1);
        if ((MemoryDescriptorList->MdlFlags & MDL_LOCK_HELD) == 0) {
            LOCK_PFN2 (OldIrql);
        }

        while (i != 0) {
            ASSERT (PointerPte->u.Hard.Valid == 1);
            ASSERT (*Page == PointerPte->u.Hard.PageFrameNumber);
            if ((MemoryDescriptorList->MdlFlags & MDL_IO_SPACE ) == 0) {
                PMMPFN Pfn3;
                Pfn3 = MI_PFN_ELEMENT (*Page);
                ASSERT (Pfn3->u3.e2.ReferenceCount > 1);
                Pfn3->u3.e2.ReferenceCount -= 1;
                ASSERT (Pfn3->u3.e2.ReferenceCount < 256);
            }

            Page += 1;
            PointerPte++;
            i -= 1;
        }

        if ((MemoryDescriptorList->MdlFlags & MDL_LOCK_HELD) == 0) {
            UNLOCK_PFN2 (OldIrql);
        }
#endif //DBG

        MemoryDescriptorList->MdlFlags &= ~(MDL_MAPPED_TO_SYSTEM_VA |
                                            MDL_PARTIAL_HAS_BEEN_MAPPED);

        MiReleaseSystemPtes (PointerBase, NumberOfPages, SystemPteSpace);
        return;

    } else {

        MiUnmapLockedPagesInUserSpace (BaseAddress,
                                   MemoryDescriptorList);
    }
}


VOID
MiUnmapLockedPagesInUserSpace (
     IN PVOID BaseAddress,
     IN PMDL MemoryDescriptorList
     )

/*++

Routine Description:

    This routine unmaps locked pages which were previously mapped via
    a MmMapLockedPages function.

Arguments:

    BaseAddress - Supplies the base address where the pages were previously
                  mapped.

    MemoryDescriptorList - Supplies a valid Memory Descriptor List which has
                            been updated by MmProbeAndLockPages.

Return Value:

    None.

Environment:

    Kernel mode.  DISPATCH_LEVEL or below if base address is within system space;
                APC_LEVEL or below if base address is user space.

--*/

{
    ULONG NumberOfPages;
    PULONG Page;
    PMMPTE PointerPte;
    PMMPTE PointerBase;
    PMMPTE PointerPde;
    PVOID StartingVa;
    KIRQL OldIrql;
    PMMVAD Vad;
    PVOID TempVa;
    PEPROCESS Process;
    CSHORT PageTableOffset;

    MmLockPagableSectionByHandle (ExPageLockHandle);

    StartingVa = (PVOID)((PCHAR)MemoryDescriptorList->StartVa +
                        MemoryDescriptorList->ByteOffset);

    Page = (PULONG)(MemoryDescriptorList + 1);
    NumberOfPages = COMPUTE_PAGES_SPANNED (StartingVa,
                                           MemoryDescriptorList->ByteCount);

    PointerPte = MiGetPteAddress (BaseAddress);
    PointerBase = PointerPte;

    //
    // This was mapped into the user portion of the address space and
    // the corresponding virtual address descriptor must be deleted.
    //

    //
    // Get the working set mutex and address creation mutex.
    //

    Process = PsGetCurrentProcess ();

    LOCK_WS_AND_ADDRESS_SPACE (Process);

    Vad = MiLocateAddress (BaseAddress);
    ASSERT (Vad != NULL);
    MiRemoveVad (Vad);

    //
    // Get the PFN mutex so we can safely decrement share and valid
    // counts on page table pages.
    //

    LOCK_PFN (OldIrql);

    do {

        if (*Page == MM_EMPTY_LIST) {
            break;
        }

        ASSERT (PointerPte->u.Hard.Valid == 1);

        (VOID)KeFlushSingleTb (BaseAddress,
                               TRUE,
                               FALSE,
                               (PHARDWARE_PTE)PointerPte,
                               ZeroPte.u.Flush);

        PointerPde = MiGetPteAddress(PointerPte);
        MiDecrementShareAndValidCount (PointerPde->u.Hard.PageFrameNumber);

        //
        // Another Pte has become zero.
        //

        PageTableOffset = (CSHORT)MiGetPteOffset( PointerPte );
        MmWorkingSetList->UsedPageTableEntries[PageTableOffset] -= 1;
        ASSERT (MmWorkingSetList->UsedPageTableEntries[PageTableOffset]
                            < PTE_PER_PAGE);

        //
        // If all the entries have been eliminated from the previous
        // page table page, delete the page table page itself.
        //

        if (MmWorkingSetList->UsedPageTableEntries[PageTableOffset] == 0) {

            TempVa = MiGetVirtualAddressMappedByPte (PointerPde);
            MiDeletePte (PointerPde,
                         TempVa,
                         FALSE,
                         Process,
                         (PMMPTE)NULL,
                         NULL);
        }

        Page++;
        PointerPte++;
        NumberOfPages -= 1;
        BaseAddress = (PVOID)((PCHAR)BaseAddress + PAGE_SIZE);
    } while (NumberOfPages != 0);

    UNLOCK_PFN (OldIrql);
    UNLOCK_WS (Process);
    UNLOCK_ADDRESS_SPACE (Process);
    ExFreePool (Vad);
    MmUnlockPagableImageSection(ExPageLockHandle);
    return;
}


PVOID
MmMapIoSpace (
     IN PHYSICAL_ADDRESS PhysicalAddress,
     IN ULONG NumberOfBytes,
     IN MEMORY_CACHING_TYPE CacheType
     )

/*++

Routine Description:

    This function maps the specified physical address into the non-pageable
    portion of the system address space.

Arguments:

    PhysicalAddress - Supplies the starting physical address to map.

    NumberOfBytes - Supplies the number of bytes to map.

    CacheType - Supplies MmNonCached if the phyiscal address is to be mapped
                as non-cached, MmCached if the address should be cached, and
                MmCacheFrameBuffer if the address should be cached as a frame
                buffer. For I/O device registers, this is usually specified
                as MmNonCached.

Return Value:

    Returns the virtual address which maps the specified physical addresses.
    The value NULL is returned if sufficient virtual address space for
    the mapping could not be found.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{
    ULONG NumberOfPages;
    ULONG PageFrameIndex;
    PMMPTE PointerPte;
    PVOID BaseVa;
    MMPTE TempPte;
    NTSTATUS Status;

    //
    // For compatibility for when CacheType used to be passed as a BOOLEAN
    // mask off the upper bits (TRUE == MmCached, FALSE == MmNonCached).
    //

    CacheType &= 0xFF;

    if (CacheType >= MmMaximumCacheType) {
        return (NULL);
    }

#ifdef i386
    ASSERT (PhysicalAddress.HighPart == 0);
#endif
#ifdef R4000
    ASSERT (PhysicalAddress.HighPart < 16);
#endif

    //PAGED_CODE();


    ASSERT (NumberOfBytes != 0);
    NumberOfPages = COMPUTE_PAGES_SPANNED (PhysicalAddress.LowPart,
                                           NumberOfBytes);

    PointerPte = MiReserveSystemPtes(NumberOfPages,
                                     SystemPteSpace,
                                     MM_COLOR_ALIGNMENT,
                                     (PhysicalAddress.LowPart &
                                                       MM_COLOR_MASK_VIRTUAL),
                                     FALSE);
    if (PointerPte == NULL) {
        return(NULL);
    }

    BaseVa = (PVOID)MiGetVirtualAddressMappedByPte (PointerPte);
    BaseVa = (PVOID)((PCHAR)BaseVa + BYTE_OFFSET(PhysicalAddress.LowPart));

    TempPte = ValidKernelPte;

#ifdef i386
    //
    // Set physical range to proper caching type.
    //

    Status = KeSetPhysicalCacheTypeRange(
                PhysicalAddress,
                NumberOfBytes,
                CacheType
                );

    //
    // If range could not be set, determine what to do
    //

    if (!NT_SUCCESS(Status)) {

        if ((Status == STATUS_NOT_SUPPORTED) &&
            ((CacheType == MmNonCached) || (CacheType == MmCached))) {

            //
            // The range may not have been set into the proper cache
            // type.  If the range is either MmNonCached or MmCached just
            // continue as the PTE will be marked properly.
            //

        } else if (Status == STATUS_UNSUCCESSFUL  &&  CacheType == MmCached) {

            //
            // If setting a range to Cached was unsuccessful things are not
            // optimal, but not fatal.  The range can be returned to the
            // caller and it will have whatever caching type it has - possibly
            // something below fully cached.
            //

#if DBG
            DbgPrint("MmMapIoSpace: Failed to set range to MmCached\n");
#endif

        } else {

            //
            // If there's still a problem, fail the request.
            //
#if DBG
            DbgPrint("MmMapIoSpace: KeSetPhysicalCacheTypeRange failed\n");
#endif

            MiReleaseSystemPtes(PointerPte, NumberOfPages, SystemPteSpace);

            return(NULL);
         }
    }
#endif

    if (CacheType == MmNonCached) {
        MI_DISABLE_CACHING (TempPte);
    }

    PageFrameIndex = (ULONG)(PhysicalAddress.QuadPart >> PAGE_SHIFT);

    do {
        ASSERT (PointerPte->u.Hard.Valid == 0);
        TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
        *PointerPte = TempPte;
        PointerPte++;
        PageFrameIndex += 1;
        NumberOfPages -= 1;
    } while (NumberOfPages != 0);

    return BaseVa;
}

VOID
MmUnmapIoSpace (
     IN PVOID BaseAddress,
     IN ULONG NumberOfBytes
     )

/*++

Routine Description:

    This function unmaps a range of physical address which were previously
    mapped via an MmMapIoSpace function call.

Arguments:

    BaseAddress - Supplies the base virtual address where the physical
                  address was previously mapped.

    NumberOfBytes - Supplies the number of bytes which were mapped.

Return Value:

    None.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{
    ULONG NumberOfPages;
    ULONG i;
    PMMPTE FirstPte;

    PAGED_CODE();
    ASSERT (NumberOfBytes != 0);
    NumberOfPages = COMPUTE_PAGES_SPANNED (BaseAddress, NumberOfBytes);
    FirstPte = MiGetPteAddress (BaseAddress);
    MiReleaseSystemPtes(FirstPte, NumberOfPages, SystemPteSpace);

    return;
}

PVOID
MmAllocateContiguousMemory (
    IN ULONG NumberOfBytes,
    IN PHYSICAL_ADDRESS HighestAcceptableAddress
    )

/*++

Routine Description:

    This function allocates a range of physically contiguous non-paged
    pool.  It relies on the fact that non-paged pool is built at
    system initialization time from a contiguous range of phyiscal
    memory.  It allocates the specified size of non-paged pool and
    then checks to ensure it is contiguous as pool expansion does
    not maintain the contiguous nature of non-paged pool.

    This routine is designed to be used by a driver's initialization
    routine to allocate a contiguous block of physical memory for
    issuing DMA requests from.

Arguments:

    NumberOfBytes - Supplies the number of bytes to allocate.

    HighestAcceptableAddress - Supplies the highest physical address
                               which is valid for the allocation.  For
                               example, if the device can only reference
                               phyiscal memory in the lower 16MB this
                               value would be set to 0xFFFFFF (16Mb - 1).

Return Value:

    NULL - a contiguous range could not be found to satisfy the request.

    NON-NULL - Returns a pointer (virtual address in the nonpaged portion
               of the system) to the allocated phyiscally contiguous
               memory.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{
    PMMPTE PointerPte;
    PMMPFN Pfn1;
    PVOID BaseAddress;
    ULONG SizeInPages;
    ULONG HighestPfn;
    ULONG i;

    PAGED_CODE();

    ASSERT (NumberOfBytes != 0);
    BaseAddress = ExAllocatePoolWithTag (NonPagedPoolCacheAligned,
                                         NumberOfBytes,
                                         'mCmM');

    SizeInPages = BYTES_TO_PAGES (NumberOfBytes);
    HighestPfn = (ULONG)(HighestAcceptableAddress.QuadPart >> PAGE_SHIFT);
    if (BaseAddress != NULL) {
        if (MiCheckForContiguousMemory( BaseAddress,
                                        SizeInPages,
                                        HighestPfn)) {

            return BaseAddress;
        } else {

            //
            // The allocation from pool does not meet the contingious
            // requirements. Free the page and see if any of the free
            // pool pages meet the requirement.
            //

            ExFreePool (BaseAddress);
        }
    } else {

        //
        // No pool was available, return NULL.
        //

        return NULL;
    }

    if (KeGetCurrentIrql() > APC_LEVEL) {
        return NULL;
    }

    BaseAddress = NULL;

    i = 3;
    for (; ; ) {
        BaseAddress = MiFindContiguousMemory (HighestPfn, SizeInPages);
        if ((BaseAddress != NULL) || (i == 0)) {
            break;
        }

        MmDelayPageFaults = TRUE;

        //
        // Attempt to move pages to the standby list.
        //

        MiEmptyAllWorkingSets ();
        MiFlushAllPages();

        KeDelayExecutionThread (KernelMode,
                                FALSE,
                                &Mm30Milliseconds);

        i -= 1;
    }
    MmDelayPageFaults = FALSE;
    return BaseAddress;
}

PVOID
MiFindContiguousMemory (
    IN ULONG HighestPfn,
    IN ULONG SizeInPages
    )

/*++

Routine Description:

    This function search nonpaged pool and the free, zeroed,
    and standby lists for contiguous pages that satisfy the
    request.

Arguments:

    HighestPfn - Supplies the highest acceptible physical page number.

    SizeInPages - Supplies the number of pages to allocate.


Return Value:

    NULL - a contiguous range could not be found to satisfy the request.

    NON-NULL - Returns a pointer (virtual address in the nonpaged portion
               of the system) to the allocated phyiscally contiguous
               memory.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/
{
    PMMPTE PointerPte;
    PMMPFN Pfn1;
    PVOID BaseAddress = NULL;
    KIRQL OldIrql;
    KIRQL OldIrql2;
    PMMFREE_POOL_ENTRY FreePageInfo;
    PLIST_ENTRY Entry;
    ULONG start;
    ULONG count;
    ULONG Page;
    ULONG found;
    MMPTE TempPte;
    ULONG PageColor;

    PAGED_CODE ();

    //
    // A suitable pool page was not allocated via the pool allocator.
    // Grab the pool lock and manually search of a page which meets
    // the requirements.
    //

    MmLockPagableSectionByHandle (ExPageLockHandle);
    OldIrql = ExLockPool (NonPagedPool);

    //
    // Trace through the page allocators pool headers for a page which
    // meets the requirements.
    //

    //
    // NonPaged pool is linked together through the pages themselves.
    //

    Entry = MmNonPagedPoolFreeListHead.Flink;

    while (Entry != &MmNonPagedPoolFreeListHead) {

        //
        // The list is not empty, see if this one meets the physical
        // requirements.
        //

        FreePageInfo = CONTAINING_RECORD(Entry,
                                         MMFREE_POOL_ENTRY,
                                         List);

        ASSERT (FreePageInfo->Signature == MM_FREE_POOL_SIGNATURE);
        if (FreePageInfo->Size >= SizeInPages) {

            //
            // This entry has sufficient space, check to see if the
            // pages meet the pysical requirements.
            //

            if (MiCheckForContiguousMemory( Entry,
                                            SizeInPages,
                                            HighestPfn)) {

                //
                // These page meet the requirements, note that
                // pages are being removed from the front of
                // the list entry and the whole list entry
                // will be removed and then the remainder inserted.
                //

                RemoveEntryList (&FreePageInfo->List);

                //
                // Adjust the number of free pages remaining in the pool.
                //

                MmNumberOfFreeNonPagedPool -= FreePageInfo->Size;
                ASSERT ((LONG)MmNumberOfFreeNonPagedPool >= 0);
                NonPagedPoolDescriptor.TotalBigPages += FreePageInfo->Size;

                //
                // Mark start and end for the block at the top of the
                // list.
                //

                Entry = PAGE_ALIGN(Entry);
                if (MI_IS_PHYSICAL_ADDRESS(Entry)) {

                    //
                    // On certains architectures (e.g., MIPS) virtual addresses
                    // may be physical and hence have no corresponding PTE.
                    //

                    Pfn1 = MI_PFN_ELEMENT (MI_CONVERT_PHYSICAL_TO_PFN (Entry));
                } else {
                    PointerPte = MiGetPteAddress(Entry);
                    ASSERT (PointerPte->u.Hard.Valid == 1);
                    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
                }

                ASSERT (Pfn1->u3.e1.StartOfAllocation == 0);
                Pfn1->u3.e1.StartOfAllocation = 1;

                //
                // Calculate the ending PFN address, note that since
                // these pages are contiguous, just add to the PFN.
                //

                Pfn1 += SizeInPages - 1;
                ASSERT (Pfn1->u3.e1.EndOfAllocation == 0);
                Pfn1->u3.e1.EndOfAllocation = 1;

                MmAllocatedNonPagedPool += FreePageInfo->Size;
                NonPagedPoolDescriptor.TotalBigPages += FreePageInfo->Size;

                if (SizeInPages == FreePageInfo->Size) {

                    //
                    // Unlock the pool and return.
                    //
                    BaseAddress = (PVOID)Entry;
                    goto Done;
                }

                BaseAddress = (PVOID)((PCHAR)Entry + (SizeInPages  << PAGE_SHIFT));

                //
                // Mark start and end of allocation in the PFN database.
                //

                if (MI_IS_PHYSICAL_ADDRESS(BaseAddress)) {

                    //
                    // On certains architectures (e.g., MIPS) virtual addresses
                    // may be physical and hence have no corresponding PTE.
                    //

                    Pfn1 = MI_PFN_ELEMENT (MI_CONVERT_PHYSICAL_TO_PFN (BaseAddress));
                } else {
                    PointerPte = MiGetPteAddress(BaseAddress);
                    ASSERT (PointerPte->u.Hard.Valid == 1);
                    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
                }

                ASSERT (Pfn1->u3.e1.StartOfAllocation == 0);
                Pfn1->u3.e1.StartOfAllocation = 1;

                //
                // Calculate the ending PTE's address, can't depend on
                // these pages being physically contiguous.
                //

                if (MI_IS_PHYSICAL_ADDRESS(BaseAddress)) {
                    Pfn1 += FreePageInfo->Size - (SizeInPages + 1);
                } else {
                    PointerPte += FreePageInfo->Size - (SizeInPages + 1);
                    ASSERT (PointerPte->u.Hard.Valid == 1);
                    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
                }
                ASSERT (Pfn1->u3.e1.EndOfAllocation == 0);
                Pfn1->u3.e1.EndOfAllocation = 1;

                ASSERT (((ULONG)BaseAddress & (PAGE_SIZE -1)) == 0);

                //
                // Unlock the pool.
                //

                ExUnlockPool (NonPagedPool, OldIrql);

                //
                // Free the entry at BaseAddress back into the pool.
                //

                ExFreePool (BaseAddress);
                BaseAddress = (PVOID)Entry;
                goto Done1;
            }
        }
        Entry = FreePageInfo->List.Flink;
    }

    //
    // No entry was found that meets the requirements.
    // Search the PFN database for pages that meet the
    // requirements.
    //

    start = 0;
    do {

        count = MmPhysicalMemoryBlock->Run[start].PageCount;
        Page = MmPhysicalMemoryBlock->Run[start].BasePage;

        if ((Page <= (1 + HighestPfn - SizeInPages)) &&
            (count >= SizeInPages)) {

            //
            // Check to see if these pages are on the right list.
            //

            found = 0;

            Pfn1 = MI_PFN_ELEMENT (Page);
            LOCK_PFN2 (OldIrql2);
            do {

                if ((Pfn1->u3.e1.PageLocation == ZeroedPageList) ||
                    (Pfn1->u3.e1.PageLocation == FreePageList) ||
                    (Pfn1->u3.e1.PageLocation == StandbyPageList)) {

                    if ((Pfn1->u1.Flink != 0) && (Pfn1->u2.Blink != 0)) {
                        found += 1;
                        if (found == SizeInPages) {

                            //
                            // A match has been found, remove these
                            // pages, add them to the free pool and
                            // return.
                            //

                            Page = 1 + Page - found;

                            //
                            // Try to find system ptes to expand the pool into.
                            //

                            PointerPte = MiReserveSystemPtes (SizeInPages,
                                                              NonPagedPoolExpansion,
                                                              0,
                                                              0,
                                                              FALSE);

                            if (PointerPte == NULL) {
                                UNLOCK_PFN2 (OldIrql2);
                                goto Done;
                            }

                            MmResidentAvailablePages -= SizeInPages;
                            MiChargeCommitmentCantExpand (SizeInPages, TRUE);
                            BaseAddress = MiGetVirtualAddressMappedByPte (PointerPte);
                            PageColor = MI_GET_PAGE_COLOR_FROM_VA(BaseAddress);
                            TempPte = ValidKernelPte;
                            MmAllocatedNonPagedPool += SizeInPages;
                            NonPagedPoolDescriptor.TotalBigPages += SizeInPages;
                            Pfn1 = MI_PFN_ELEMENT (Page - 1);

                            do {
                                Pfn1 += 1;
                                if (Pfn1->u3.e1.PageLocation == StandbyPageList) {
                                    MiUnlinkPageFromList (Pfn1);
                                    MiRestoreTransitionPte (Page);
                                } else {
                                    MiUnlinkFreeOrZeroedPage (Page);
                                }

                                MI_CHECK_PAGE_ALIGNMENT(Page,
                                                        PageColor & MM_COLOR_MASK);
                                Pfn1->u3.e1.PageColor = PageColor & MM_COLOR_MASK;
                                PageColor += 1;
                                TempPte.u.Hard.PageFrameNumber = Page;
                                *PointerPte = TempPte;

                                Pfn1->u3.e2.ReferenceCount = 1;
                                Pfn1->u2.ShareCount = 1;
                                Pfn1->PteAddress = PointerPte;
                                Pfn1->OriginalPte.u.Long = MM_DEMAND_ZERO_WRITE_PTE;
                                Pfn1->PteFrame = MiGetPteAddress(PointerPte)->u.Hard.PageFrameNumber;
                                Pfn1->u3.e1.PageLocation = ActiveAndValid;

                                if (found == SizeInPages) {
                                    Pfn1->u3.e1.StartOfAllocation = 1;
                                }
                                PointerPte += 1;
                                Page += 1;
                                found -= 1;
                            } while (found);

                            Pfn1->u3.e1.EndOfAllocation = 1;
                            UNLOCK_PFN2 (OldIrql2);
                            goto Done;
                        }
                    } else {
                        found = 0;
                    }
                } else {
                    found = 0;
                }
                Page += 1;
                Pfn1 += 1;
                count -= 1;

            } while (count && (Page <= HighestPfn));
            UNLOCK_PFN2 (OldIrql2);
        }
        start += 1;
    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

Done:

    ExUnlockPool (NonPagedPool, OldIrql);

Done1:

    MmUnlockPagableImageSection (ExPageLockHandle);
    return BaseAddress;
}

VOID
MmFreeContiguousMemory (
    IN PVOID BaseAddress
    )

/*++

Routine Description:

    This function deallocates a range of physically contiguous non-paged
    pool which was allocated with the MmAllocateContiguousMemory function.

Arguments:

    BaseAddress - Supplies the base virtual address where the physical
                  address was previously mapped.

Return Value:

    None.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{
    PAGED_CODE();
    ExFreePool (BaseAddress);
    return;
}

PHYSICAL_ADDRESS
MmGetPhysicalAddress (
     IN PVOID BaseAddress
     )

/*++

Routine Description:

    This function returns the corresponding physical address for a
    valid virtual address.

Arguments:

    BaseAddress - Supplies the virtual address for which to return the
                  physical address.

Return Value:

    Returns the corresponding physical address.

Environment:

    Kernel mode.  Any IRQL level.

--*/

{
    PMMPTE PointerPte;
    PHYSICAL_ADDRESS PhysicalAddress;

    if (MI_IS_PHYSICAL_ADDRESS(BaseAddress)) {
        PhysicalAddress.LowPart = MI_CONVERT_PHYSICAL_TO_PFN (BaseAddress);
    } else {

        PointerPte = MiGetPteAddress(BaseAddress);

        if (PointerPte->u.Hard.Valid == 0) {
            KdPrint(("MM:MmGetPhysicalAddressFailed base address was %lx",
                      BaseAddress));
            ZERO_LARGE (PhysicalAddress);
            return PhysicalAddress;
        }
        PhysicalAddress.LowPart = PointerPte->u.Hard.PageFrameNumber;
    }

    PhysicalAddress.HighPart = 0;
    PhysicalAddress.QuadPart = PhysicalAddress.QuadPart << PAGE_SHIFT;
    PhysicalAddress.LowPart += BYTE_OFFSET(BaseAddress);

    return PhysicalAddress;
}

PVOID
MmGetVirtualForPhysical (
    IN PHYSICAL_ADDRESS PhysicalAddress
     )

/*++

Routine Description:

    This function returns the corresponding virtual address for a physical
    address whose primary virtual address is in system space.

Arguments:

    PhysicalAddress - Supplies the physical address for which to return the
                  virtual address.

Return Value:

    Returns the corresponding virtual address.

Environment:

    Kernel mode.  Any IRQL level.

--*/

{
    ULONG PageFrameIndex;
    PMMPFN Pfn;

    PageFrameIndex = (ULONG)(PhysicalAddress.QuadPart >> PAGE_SHIFT);

    Pfn = MI_PFN_ELEMENT (PageFrameIndex);

    return (PVOID)((PCHAR)MiGetVirtualAddressMappedByPte (Pfn->PteAddress) +
                    BYTE_OFFSET (PhysicalAddress.LowPart));
}

PVOID
MmAllocateNonCachedMemory (
    IN ULONG NumberOfBytes
    )

/*++

Routine Description:

    This function allocates a range of noncached memory in
    the non-paged portion of the system address space.

    This routine is designed to be used by a driver's initialization
    routine to allocate a noncached block of virtual memory for
    various device specific buffers.

Arguments:

    NumberOfBytes - Supplies the number of bytes to allocate.

Return Value:

    NULL - the specified request could not be satisfied.

    NON-NULL - Returns a pointer (virtual address in the nonpaged portion
               of the system) to the allocated phyiscally contiguous
               memory.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{
    PMMPTE PointerPte;
    MMPTE TempPte;
    ULONG NumberOfPages;
    ULONG PageFrameIndex;
    PVOID BaseAddress;
    KIRQL OldIrql;

    MmLockPagableSectionByHandle (ExPageLockHandle);

    ASSERT (NumberOfBytes != 0);

    //
    // Acquire the PFN mutex to synchronize access to the pfn database.
    //

    LOCK_PFN (OldIrql);

    //
    // Obtain enough pages to contain the allocation.
    // the system PTE pool.  The system PTE pool contains non-paged PTEs
    // which are currently empty.
    //

    NumberOfPages = BYTES_TO_PAGES(NumberOfBytes);

    //
    // Check to make sure the phyiscal pages are available.
    //

    if (MmResidentAvailablePages <= (LONG)NumberOfPages) {
        BaseAddress = NULL;
        goto Done;
    }

    PointerPte = MiReserveSystemPtes (NumberOfPages,
                                      SystemPteSpace,
                                      0,
                                      0,
                                      FALSE);
    if (PointerPte == NULL) {
        BaseAddress = NULL;
        goto Done;
    }

    MmResidentAvailablePages -= (LONG)NumberOfPages;
    MiChargeCommitmentCantExpand (NumberOfPages, TRUE);

    BaseAddress = (PVOID)MiGetVirtualAddressMappedByPte (PointerPte);

    do {
        ASSERT (PointerPte->u.Hard.Valid == 0);
        MiEnsureAvailablePageOrWait (NULL, NULL);
        PageFrameIndex = MiRemoveAnyPage (MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));

        MI_MAKE_VALID_PTE (TempPte,
                           PageFrameIndex,
                           MM_READWRITE,
                           PointerPte);

        MI_SET_PTE_DIRTY (TempPte);
        MI_DISABLE_CACHING (TempPte);
        *PointerPte = TempPte;
        MiInitializePfn (PageFrameIndex, PointerPte, 1);
        PointerPte += 1;
        NumberOfPages -= 1;
    } while (NumberOfPages != 0);

    //
    // Flush any data for this page out of the dcaches.
    //

    KeSweepDcache (TRUE);

Done:
    UNLOCK_PFN (OldIrql);
    MmUnlockPagableImageSection (ExPageLockHandle);

    return BaseAddress;
}

VOID
MmFreeNonCachedMemory (
    IN PVOID BaseAddress,
    IN ULONG NumberOfBytes
    )

/*++

Routine Description:

    This function deallocates a range of noncached memory in
    the non-paged portion of the system address space.

Arguments:

    BaseAddress - Supplies the base virtual address where the noncached
                  memory resides.

    NumberOfBytes - Supplies the number of bytes allocated to the requst.
                    This must be the same number that was obtained with
                    the MmAllocateNonCachedMemory call.

Return Value:

    None.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{

    PMMPTE PointerPte;
    PMMPFN Pfn1;
    ULONG NumberOfPages;
    ULONG i;
    ULONG PageFrameIndex;
    KIRQL OldIrql;

    ASSERT (NumberOfBytes != 0);
    ASSERT (PAGE_ALIGN (BaseAddress) == BaseAddress);
    MI_MAKING_MULTIPLE_PTES_INVALID (TRUE);

    NumberOfPages = BYTES_TO_PAGES(NumberOfBytes);

    PointerPte = MiGetPteAddress (BaseAddress);

    MmLockPagableSectionByHandle (ExPageLockHandle);

    LOCK_PFN (OldIrql);

    i = NumberOfPages;

    do {


        PageFrameIndex = PointerPte->u.Hard.PageFrameNumber;

        //
        // Set the pointer to PTE as empty so the page
        // is deleted when the reference count goes to zero.
        //

        Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
        ASSERT (Pfn1->u2.ShareCount == 1);
        MiDecrementShareAndValidCount (Pfn1->PteFrame);
        MI_SET_PFN_DELETED (Pfn1);
        MiDecrementShareCountOnly (PageFrameIndex);
        PointerPte += 1;
        i -= 1;
    } while (i != 0);

    PointerPte -= NumberOfPages;

    MiReleaseSystemPtes (PointerPte, NumberOfPages, SystemPteSpace);

    //
    // Update the count of available resident pages.
    //

    MmResidentAvailablePages += NumberOfPages;
    MiReturnCommitment (NumberOfPages);

    UNLOCK_PFN (OldIrql);

    MmUnlockPagableImageSection (ExPageLockHandle);
    return;
}

ULONG
MmSizeOfMdl (
    IN PVOID Base,
    IN ULONG Length
    )

/*++

Routine Description:

    This function returns the number of bytes required for an MDL for a
    given buffer and size.

Arguments:

    Base - Supplies the base virtual address for the buffer.

    Length - Supplies the size of the buffer in bytes.

Return Value:

    Returns the number of bytes required to contain the MDL.

Environment:

    Kernel mode.  Any IRQL level.

--*/

{
    return( sizeof( MDL ) +
                (ADDRESS_AND_SIZE_TO_SPAN_PAGES( Base, Length ) *
                 sizeof( ULONG ))
          );
}



PMDL
MmCreateMdl (
    IN PMDL MemoryDescriptorList OPTIONAL,
    IN PVOID Base,
    IN ULONG Length
    )

/*++

Routine Description:

    This function optionally allocates and initializes an MDL.

Arguments:

    MemoryDescriptorList - Optionally supplies the address of the MDL
        to initialize.  If this address is supplied as NULL an MDL is
        allocated from non-paged pool and initialized.

    Base - Supplies the base virtual address for the buffer.

    Length - Supplies the size of the buffer in bytes.

Return Value:

    Returns the address of the initialized MDL.

Environment:

    Kernel mode, IRQL of DISPATCH_LEVEL or below.

--*/

{
    ULONG MdlSize;

    MdlSize = MmSizeOfMdl( Base, Length );

    if (!ARGUMENT_PRESENT( MemoryDescriptorList )) {
        MemoryDescriptorList = (PMDL)ExAllocatePoolWithTag (
                                                     NonPagedPoolMustSucceed,
                                                     MdlSize,
                                                     'ldmM');
    }

    MmInitializeMdl( MemoryDescriptorList, Base, Length );
    return ( MemoryDescriptorList );
}

BOOLEAN
MmSetAddressRangeModified (
    IN PVOID Address,
    IN ULONG Length
    )

/*++

Routine Description:

    This routine sets the modified bit in the PFN database for the
    pages that correspond to the specified address range.

    Note that the dirty bit in the PTE is cleared by this operation.

Arguments:

    Address - Supplies the address of the start of the range.  This
              range must reside within the system cache.

    Length - Supplies the length of the range.

Return Value:

    TRUE if at least one PTE was dirty in the range, FALSE otherwise.

Environment:

    Kernel mode.  APC_LEVEL and below for pageable addresses,
                  DISPATCH_LEVEL and below for non-pageable addresses.

--*/

{
    PMMPTE PointerPte;
    PMMPTE LastPte;
    PMMPFN Pfn1;
    PMMPTE FlushPte;
    MMPTE PteContents;
    MMPTE FlushContents;
    KIRQL OldIrql;
    PVOID VaFlushList[MM_MAXIMUM_FLUSH_COUNT];
    ULONG Count = 0;
    BOOLEAN Result = FALSE;

    //
    // Loop on the copy on write case until the page is only
    // writable.
    //

    PointerPte = MiGetPteAddress (Address);
    LastPte = MiGetPteAddress ((PVOID)((PCHAR)Address + Length - 1));

    LOCK_PFN2 (OldIrql);

    do {


        PteContents = *PointerPte;

        if (PteContents.u.Hard.Valid == 1) {

            Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);
            Pfn1->u3.e1.Modified = 1;

            if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
                         (Pfn1->u3.e1.WriteInProgress == 0)) {
                MiReleasePageFileSpace (Pfn1->OriginalPte);
                Pfn1->OriginalPte.u.Soft.PageFileHigh = 0;
            }

#ifdef NT_UP
            //
            // On uniprocessor systems no need to flush if this processor
            // doesn't think the PTE is dirty.
            //

            if (MI_IS_PTE_DIRTY (PteContents)) {
                Result = TRUE;
#else  //NT_UP
                Result |= (BOOLEAN)(MI_IS_PTE_DIRTY (PteContents));
#endif //NT_UP
                MI_SET_PTE_CLEAN (PteContents);
                *PointerPte = PteContents;
                FlushContents = PteContents;
                FlushPte = PointerPte;

                //
                // Clear the write bit in the PTE so new writes can be tracked.
                //

                if (Count != MM_MAXIMUM_FLUSH_COUNT) {
                    VaFlushList[Count] = Address;
                    Count += 1;
                }
#ifdef NT_UP
            }
#endif //NT_UP
        }
        PointerPte += 1;
        Address = (PVOID)((PCHAR)Address + PAGE_SIZE);
    } while (PointerPte <= LastPte);

    if (Count != 0) {
        if (Count == 1) {

            (VOID)KeFlushSingleTb (VaFlushList[0],
                                   FALSE,
                                   TRUE,
                                   (PHARDWARE_PTE)FlushPte,
                                   FlushContents.u.Flush);

        } else if (Count != MM_MAXIMUM_FLUSH_COUNT) {

            KeFlushMultipleTb (Count,
                               &VaFlushList[0],
                               FALSE,
                               TRUE,
                               NULL,
                               ZeroPte.u.Flush);

        } else {
            KeFlushEntireTb (FALSE, TRUE);
        }
    }
    UNLOCK_PFN2 (OldIrql);
    return Result;
}


BOOLEAN
MiCheckForContiguousMemory (
    IN PVOID BaseAddress,
    IN ULONG SizeInPages,
    IN ULONG HighestPfn
    )

/*++

Routine Description:

    This routine checks to see if the physical memory mapped
    by the specified BaseAddress for the specified size is
    contiguous and the last page of the physical memory is
    less than or equal to the specified HighestPfn.

Arguments:

    BaseAddress - Supplies the base address to start checking at.

    SizeInPages - Supplies the number of pages in the range.

    HighestPfn  - Supplies the highest PFN acceptable as a physical page.

Return Value:

    Returns TRUE if the physical memory is contiguous and less than
    or equal to the HighestPfn, FALSE otherwise.

Environment:

    Kernel mode, memory mangement internal.

--*/

{
    PMMPTE PointerPte;
    PMMPTE LastPte;
    ULONG PageFrameIndex;

    if (MI_IS_PHYSICAL_ADDRESS (BaseAddress)) {
        if (HighestPfn >=
                (MI_CONVERT_PHYSICAL_TO_PFN(BaseAddress) + SizeInPages - 1)) {
            return TRUE;
        } else {
            return FALSE;
        }
    } else {
        PointerPte = MiGetPteAddress (BaseAddress);
        LastPte = PointerPte + SizeInPages;
        PageFrameIndex = PointerPte->u.Hard.PageFrameNumber + 1;
        PointerPte += 1;

        //
        // Check to see if the range of physical addresses is contiguous.
        //

        while (PointerPte < LastPte) {
            if (PointerPte->u.Hard.PageFrameNumber != PageFrameIndex) {

                //
                // Memory is not physically contiguous.
                //

                return FALSE;
            }
            PageFrameIndex += 1;
            PointerPte += 1;
        }
    }

    if (PageFrameIndex <= HighestPfn) {
        return TRUE;
    }
    return FALSE;
}


VOID
MmLockPagableSectionByHandle (
    IN PVOID ImageSectionHandle
    )


/*++

Routine Description:

    This routine checks to see if the specified pages are resident in
    the process's working set and if so the reference count for the
    page is incremented.  The allows the virtual address to be accessed
    without getting a hard page fault (have to go to the disk... except
    for extremely rare case when the page table page is removed from the
    working set and migrates to the disk.

    If the virtual address is that of the system wide global "cache" the
    virtual adderss of the "locked" pages is always guarenteed to
    be valid.

    NOTE: This routine is not to be used for general locking of user
    addresses - use MmProbeAndLockPages.  This routine is intended for
    well behaved system code like the file system caches which allocates
    virtual addresses for mapping files AND guarantees that the mapping
    will not be modified (deleted or changed) while the pages are locked.

Arguments:

    ImageSectionHandle - Supplies the value returned by a previous call
        to MmLockPagableDataSection.  This is a pointer to the Section
        header for the image.

Return Value:

    None.

Environment:

    Kernel mode, IRQL of DISPATCH_LEVEL or below.

--*/

{
    PIMAGE_SECTION_HEADER NtSection;
    PVOID BaseAddress;
    ULONG SizeToLock;
    PMMPTE PointerPte;
    PMMPTE LastPte;
    KIRQL OldIrql;
    ULONG Collision;

    if (MI_IS_PHYSICAL_ADDRESS(ImageSectionHandle)) {

        //
        // No need to lock physical addresses.
        //

        return;
    }

    NtSection = (PIMAGE_SECTION_HEADER)ImageSectionHandle;

    BaseAddress = (PVOID)NtSection->PointerToLinenumbers;

    ASSERT ((BaseAddress < (PVOID)MM_SYSTEM_CACHE_START) ||
        (BaseAddress >= (PVOID)MM_SYSTEM_CACHE_END));
    ASSERT (BaseAddress >= (PVOID)MM_SYSTEM_RANGE_START);

    SizeToLock = NtSection->SizeOfRawData;
    PointerPte = MiGetPteAddress(BaseAddress);
    LastPte = MiGetPteAddress((PCHAR)BaseAddress + SizeToLock - 1);

    ASSERT (SizeToLock != 0);

    //
    // The address must be within the system space.
    //

RetryLock:

    LOCK_PFN2 (OldIrql);

    MiMakeSystemAddressValidPfn (&NtSection->NumberOfLinenumbers);

    //
    // The NumberOfLinenumbers field is used to store the
    // lock count.
    //
    //  Value of 0 means unlocked,
    //  Value of 1 means lock in progress by another thread.
    //  Value of 2 or more means locked.
    //
    //  If the value is 1, this thread must block until the other thread's
    //  lock operation is complete.
    //

    NtSection->NumberOfLinenumbers += 1;

    if (NtSection->NumberOfLinenumbers >= 3) {

        //
        // Already locked, increment counter and return.
        //

        UNLOCK_PFN2 (OldIrql);
        return;

    }

    if (NtSection->NumberOfLinenumbers == 2) {

        //
        // A lock is in progress.
        // Reset to back to 1 and wait.
        //

        NtSection->NumberOfLinenumbers = 1;
        MmCollidedLockWait = TRUE;

        KeEnterCriticalRegion();
        UNLOCK_PFN_AND_THEN_WAIT (OldIrql);

        KeWaitForSingleObject(&MmCollidedLockEvent,
                              WrVirtualMemory,
                              KernelMode,
                              FALSE,
                              (PLARGE_INTEGER)NULL);
        KeLeaveCriticalRegion();
        goto RetryLock;
    }

    //
    // Value was 0 when the lock was obtained.  It is now 1 indicating
    // a lock is in progress.
    //

    MiLockCode (PointerPte, LastPte, MM_LOCK_BY_REFCOUNT);

    //
    // Set lock count to 2 (it was 1 when this started) and check
    // to see if any other threads tried to lock while this was happening.
    //

    MiMakeSystemAddressValidPfn (&NtSection->NumberOfLinenumbers);
    NtSection->NumberOfLinenumbers += 1;

    ASSERT (NtSection->NumberOfLinenumbers == 2);

    Collision = MmCollidedLockWait;
    MmCollidedLockWait = FALSE;

    UNLOCK_PFN2 (OldIrql);

    if (Collision) {

        //
        // Wake up all waiters.
        //

        KePulseEvent (&MmCollidedLockEvent, 0, FALSE);
    }

    return;
}


VOID
MiLockCode (
    IN PMMPTE FirstPte,
    IN PMMPTE LastPte,
    IN ULONG LockType
    )

/*++

Routine Description:

    This routine checks to see if the specified pages are resident in
    the process's working set and if so the reference count for the
    page is incremented.  The allows the virtual address to be accessed
    without getting a hard page fault (have to go to the disk... except
    for extremely rare case when the page table page is removed from the
    working set and migrates to the disk.

    If the virtual address is that of the system wide global "cache" the
    virtual adderss of the "locked" pages is always guarenteed to
    be valid.

    NOTE: This routine is not to be used for general locking of user
    addresses - use MmProbeAndLockPages.  This routine is intended for
    well behaved system code like the file system caches which allocates
    virtual addresses for mapping files AND guarantees that the mapping
    will not be modified (deleted or changed) while the pages are locked.

Arguments:

    FirstPte - Supplies the base address to begin locking.

    LastPte - The last PTE to lock.

    LockType - Supplies either MM_LOCK_BY_REFCOUNT or MM_LOCK_NONPAGE.
               LOCK_BY_REFCOUNT increments the reference count to keep
               the page in memory, LOCK_NONPAGE removes the page from
               the working set so it's locked just like nonpaged pool.

Return Value:

    None.

Environment:

    Kernel mode, PFN LOCK held.

--*/

{
    PMMPFN Pfn1;
    PMMPTE PointerPte;
    MMPTE TempPte;
    MMPTE PteContents;
    ULONG WorkingSetIndex;
    ULONG PageFrameIndex;
    KIRQL OldIrql1;
    KIRQL OldIrql;

    MM_PFN_LOCK_ASSERT();

    ASSERT (!MI_IS_PHYSICAL_ADDRESS(MiGetVirtualAddressMappedByPte(FirstPte)));
    PointerPte = FirstPte;

    MmLockedCode += 1 + LastPte - FirstPte;

    do {

        PteContents = *PointerPte;
        ASSERT (PteContents.u.Long != ZeroKernelPte.u.Long);
        if (PteContents.u.Hard.Valid == 0) {

            ASSERT (PteContents.u.Soft.Prototype != 1);

            if (PteContents.u.Soft.Transition == 1) {

                PageFrameIndex = PteContents.u.Trans.PageFrameNumber;
                Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
                if ((Pfn1->u3.e1.ReadInProgress) ||
                    (Pfn1->u3.e1.InPageError)) {

                    //
                    // Page read is ongoing, wait for the read to
                    // complete then retest.
                    //

                    OldIrql = APC_LEVEL;
                    KeEnterCriticalRegion();
                    UNLOCK_PFN_AND_THEN_WAIT (OldIrql);
                    KeWaitForSingleObject( Pfn1->u1.Event,
                                           WrPageIn,
                                           KernelMode,
                                           FALSE,
                                           (PLARGE_INTEGER)NULL);
                    KeLeaveCriticalRegion();

                    //
                    // Need to delay so the faulting thread can
                    // perform the inpage completion.
                    //

                    KeDelayExecutionThread (KernelMode,
                                            FALSE,
                                            &MmShortTime);

                    LOCK_PFN (OldIrql);
                    continue;
                }

                MiUnlinkPageFromList (Pfn1);

                //
                // Set the reference count and share counts to 1.
                //

                Pfn1->u3.e2.ReferenceCount += 1;
                Pfn1->u2.ShareCount = 1;
                Pfn1->u3.e1.PageLocation = ActiveAndValid;

                MI_MAKE_VALID_PTE (TempPte,
                               PageFrameIndex,
                               Pfn1->OriginalPte.u.Soft.Protection,
                               PointerPte );

                *PointerPte = TempPte;

                //
                // Increment the reference count one for putting it the
                // working set list and one for locking it for I/O.
                //

                if (LockType == MM_LOCK_BY_REFCOUNT) {

                    //
                    // Lock the page in the working set by upping the
                    // refernece count.
                    //

                    Pfn1->u3.e2.ReferenceCount += 1;
                    Pfn1->u1.WsIndex = (ULONG)PsGetCurrentThread();

                    UNLOCK_PFN (APC_LEVEL);
                    LOCK_SYSTEM_WS (OldIrql);
                    WorkingSetIndex = MiLocateAndReserveWsle (&MmSystemCacheWs);

                    MiUpdateWsle (&WorkingSetIndex,
                                  MiGetVirtualAddressMappedByPte (PointerPte),
                                  MmSystemCacheWorkingSetList,
                                  Pfn1);
                    UNLOCK_SYSTEM_WS (OldIrql);
                    LOCK_PFN (OldIrql);
                } else {

                    //
                    // Set the wsindex field to zero, indicating that the
                    // page is not in the system working set.
                    //

                    ASSERT (Pfn1->u1.WsIndex == 0);
                }

            } else {

                //
                // Page is not in memory.
                //

                MiMakeSystemAddressValidPfn (
                        MiGetVirtualAddressMappedByPte(PointerPte));

                continue;
            }

        } else {

            //
            // This address is already in the system working set.
            //

            Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);

            //
            // Up the reference count so the page cannot be released.
            //

            Pfn1->u3.e2.ReferenceCount += 1;

            if (LockType != MM_LOCK_BY_REFCOUNT) {

                //
                // If the page is in the system working set, remove it.
                // The system working set lock MUST be owned to check to
                // see if this page is in the working set or not.  This
                // is because the pager may have just release the PFN lock,
                // acquired the system lock and is now trying to add the
                // page to the system working set.
                //

                UNLOCK_PFN (APC_LEVEL);
                LOCK_SYSTEM_WS (OldIrql1);

                if (Pfn1->u1.WsIndex != 0) {
                    MiRemoveWsle (Pfn1->u1.WsIndex,
                                  MmSystemCacheWorkingSetList );
                    MiReleaseWsle (Pfn1->u1.WsIndex, &MmSystemCacheWs);
                    Pfn1->u1.WsIndex = 0;
                }
                UNLOCK_SYSTEM_WS (OldIrql1);
                LOCK_PFN (OldIrql);
                ASSERT (Pfn1->u3.e2.ReferenceCount > 1);
                Pfn1->u3.e2.ReferenceCount -= 1;
            }
        }

        PointerPte += 1;
    } while (PointerPte <= LastPte);

    return;
}


PVOID
MmLockPagableDataSection(
    IN PVOID AddressWithinSection
    )

/*++

Routine Description:

    This functions locks the entire section that contains the specified
    section in memory.  This allows pagable code to be brought into
    memory and to be used as if the code was not really pagable.  This
    should not be done with a high degree of frequency.

Arguments:

    AddressWithinSection - Supplies the address of a function
        contained within a section that should be brought in and locked
        in memory.

Return Value:

    This function returns a value to be used in a subsequent call to
    MmUnlockPagableImageSection.

--*/

{
    PLDR_DATA_TABLE_ENTRY DataTableEntry;
    ULONG i;
    PIMAGE_NT_HEADERS NtHeaders;
    PIMAGE_SECTION_HEADER NtSection;
    PIMAGE_SECTION_HEADER FoundSection;
    ULONG Rva;

    PAGED_CODE();

    if (MI_IS_PHYSICAL_ADDRESS(AddressWithinSection)) {

        //
        // Physical address, just return that as the handle.
        //

        return AddressWithinSection;
    }

    //
    // Search the loaded module list for the data table entry that describes
    // the DLL that was just unloaded. It is possible an entry is not in the
    // list if a failure occured at a point in loading the DLL just before
    // the data table entry was generated.
    //

    FoundSection = NULL;

    KeEnterCriticalRegion();
    ExAcquireResourceShared (&PsLoadedModuleResource, TRUE);

    DataTableEntry = MiLookupDataTableEntry (AddressWithinSection, TRUE);

    Rva = (ULONG)((PUCHAR)AddressWithinSection - (ULONG)DataTableEntry->DllBase);

    NtHeaders = (PIMAGE_NT_HEADERS)RtlImageNtHeader(DataTableEntry->DllBase);

    NtSection = (PIMAGE_SECTION_HEADER)((ULONG)NtHeaders +
                        sizeof(ULONG) +
                        sizeof(IMAGE_FILE_HEADER) +
                        NtHeaders->FileHeader.SizeOfOptionalHeader
                        );

    for (i = 0; i < NtHeaders->FileHeader.NumberOfSections; i++) {

        if ( Rva >= NtSection->VirtualAddress &&
             Rva < NtSection->VirtualAddress + NtSection->SizeOfRawData ) {
            FoundSection = NtSection;

            if (NtSection->PointerToLinenumbers != (ULONG)((PUCHAR)DataTableEntry->DllBase +
                            NtSection->VirtualAddress)) {

                //
                // Stomp on the PointerToLineNumbers field so that it contains
                // the Va of this section and NumberOFLinenumbers so it contains
                // the Lock Count for the section.
                //

                NtSection->PointerToLinenumbers = (ULONG)((PUCHAR)DataTableEntry->DllBase +
                                        NtSection->VirtualAddress);
                NtSection->NumberOfLinenumbers = 0;
            }

            //
            // Now lock in the code
            //

#if DBG
            if (MmDebug & MM_DBG_LOCK_CODE) {
                DbgPrint("MM Lock %wZ %8s 0x%08x -> 0x%8x : 0x%08x %3ld.\n",
                        &DataTableEntry->BaseDllName,
                        NtSection->Name,
                        AddressWithinSection,
                        NtSection,
                        NtSection->PointerToLinenumbers,
                        NtSection->NumberOfLinenumbers);
            }
#endif //DBG

            MmLockPagableSectionByHandle ((PVOID)NtSection);

            goto found_the_section;
        }
        NtSection++;
    }

found_the_section:

    ExReleaseResource (&PsLoadedModuleResource);
    KeLeaveCriticalRegion();
    if (!FoundSection) {
        KeBugCheckEx (MEMORY_MANAGEMENT,
                      0x1234,
                      (ULONG)AddressWithinSection,
                      0,
                      0);
    }
    return (PVOID)FoundSection;
}


PLDR_DATA_TABLE_ENTRY
MiLookupDataTableEntry (
    IN PVOID AddressWithinSection,
    IN ULONG ResourceHeld
    )

/*++

Routine Description:

    This functions locks the entire section that contains the specified
    section in memory.  This allows pagable code to be brought into
    memory and to be used as if the code was not really pagable.  This
    should not be done with a high degree of frequency.

Arguments:

    AddressWithinSection - Supplies the address of a function
        contained within a section that should be brought in and locked
        in memory.

    ResourceHeld - Supplies true if the data table resource lock is
                   already held.

Return Value:

    This function returns a value to be used in a subsequent call to
    MmUnlockPagableImageSection.

--*/

{
    PLDR_DATA_TABLE_ENTRY DataTableEntry;
    PLDR_DATA_TABLE_ENTRY FoundEntry = NULL;
    PLIST_ENTRY NextEntry;

    PAGED_CODE();

    //
    // Search the loaded module list for the data table entry that describes
    // the DLL that was just unloaded. It is possible an entry is not in the
    // list if a failure occured at a point in loading the DLL just before
    // the data table entry was generated.
    //

    if (!ResourceHeld) {
        KeEnterCriticalRegion();
        ExAcquireResourceShared (&PsLoadedModuleResource, TRUE);
    }

    NextEntry = PsLoadedModuleList.Flink;
    do {

        DataTableEntry = CONTAINING_RECORD(NextEntry,
                                           LDR_DATA_TABLE_ENTRY,
                                           InLoadOrderLinks);

        //
        // Locate the loaded module that contains this address.
        //

        if ( AddressWithinSection >= DataTableEntry->DllBase &&
             AddressWithinSection < (PVOID)((PUCHAR)DataTableEntry->DllBase+DataTableEntry->SizeOfImage) ) {

            FoundEntry = DataTableEntry;
            break;
        }

        NextEntry = NextEntry->Flink;
    } while (NextEntry != &PsLoadedModuleList);

    if (!ResourceHeld) {
        ExReleaseResource (&PsLoadedModuleResource);
        KeLeaveCriticalRegion();
    }
    return FoundEntry;
}

VOID
MmUnlockPagableImageSection(
    IN PVOID ImageSectionHandle
    )

/*++

Routine Description:

    This function unlocks from memory, the pages locked by a preceding call to
    MmLockPagableDataSection.

Arguments:

    ImageSectionHandle - Supplies the value returned by a previous call
        to MmLockPagableDataSection.

Return Value:

    None.

--*/

{
    PIMAGE_SECTION_HEADER NtSection;
    PMMPTE PointerPte;
    PMMPTE LastPte;
    KIRQL OldIrql;
    PVOID BaseAddress;
    ULONG SizeToUnlock;
    ULONG Collision;

    if (MI_IS_PHYSICAL_ADDRESS(ImageSectionHandle)) {

        //
        // No need to lock physical addresses.
        //

        return;
    }

    NtSection = (PIMAGE_SECTION_HEADER)ImageSectionHandle;

    BaseAddress = (PVOID)NtSection->PointerToLinenumbers;
    SizeToUnlock = NtSection->SizeOfRawData;

    //DbgPrint("MM Unlock %s 0x%08x\n",NtSection->Name,NtSection->PointerToLinenumbers);

    PointerPte = MiGetPteAddress(BaseAddress);
    LastPte = MiGetPteAddress((PCHAR)BaseAddress + SizeToUnlock - 1);

    //
    // Address must be within the system cache.
    //

    LOCK_PFN2 (OldIrql);

    //
    // The NumberOfLinenumbers field is used to store the
    // lock count.
    //

    ASSERT (NtSection->NumberOfLinenumbers >= 2);
    NtSection->NumberOfLinenumbers -= 1;

    if (NtSection->NumberOfLinenumbers != 1) {
        UNLOCK_PFN2 (OldIrql);
        return;
    }

    do {

#if DBG
        {   PMMPFN Pfn;
            ASSERT (PointerPte->u.Hard.Valid == 1);
            Pfn = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
            ASSERT (Pfn->u3.e2.ReferenceCount > 1);
        }
#endif //DBG

        MiDecrementReferenceCount (PointerPte->u.Hard.PageFrameNumber);
        PointerPte += 1;
    } while (PointerPte <= LastPte);

    NtSection->NumberOfLinenumbers -= 1;
    ASSERT (NtSection->NumberOfLinenumbers == 0);
    Collision = MmCollidedLockWait;
    MmCollidedLockWait = FALSE;
    MmLockedCode -= SizeToUnlock;

    UNLOCK_PFN2 (OldIrql);

    if (Collision) {
        KePulseEvent (&MmCollidedLockEvent, 0, FALSE);
    }

    return;
}


BOOLEAN
MmIsRecursiveIoFault(
    VOID
    )

/*++

Routine Description:

    This function examines the thread's page fault clustering information
    and determines if the current page fault is occuring during an I/O
    operation.

Arguments:

    None.

Return Value:

    Returns TRUE if the fault is occuring during an I/O operation,
    FALSE otherwise.

--*/

{
    return PsGetCurrentThread()->DisablePageFaultClustering |
           PsGetCurrentThread()->ForwardClusterOnly;
}


VOID
MmMapMemoryDumpMdl(
    IN OUT PMDL MemoryDumpMdl
    )

/*++

Routine Description:

    For use by crash dump routine ONLY.  Maps an MDL into a fixed
    portion of the address space.  Only 1 mdl can be mapped at a
    time.

Arguments:

    MemoryDumpMdl - Supplies the MDL to map.

Return Value:

    None, fields in MDL updated.

--*/

{
    ULONG NumberOfPages;
    PMMPTE PointerPte;
    PCHAR BaseVa;
    MMPTE TempPte;
    PULONG Page;

    NumberOfPages = BYTES_TO_PAGES (MemoryDumpMdl->ByteCount + MemoryDumpMdl->ByteOffset);

    PointerPte = MmCrashDumpPte;
    BaseVa = (PCHAR)MiGetVirtualAddressMappedByPte(PointerPte);
    MemoryDumpMdl->MappedSystemVa = (PCHAR)BaseVa + MemoryDumpMdl->ByteOffset;
    TempPte = ValidKernelPte;
    Page = (PULONG)(MemoryDumpMdl + 1);

    do {

        KiFlushSingleTb (TRUE, BaseVa);
        ASSERT ((*Page <= MmHighestPhysicalPage) &&
                (*Page >= MmLowestPhysicalPage));

        TempPte.u.Hard.PageFrameNumber = *Page;
        *PointerPte = TempPte;

        Page++;
        PointerPte++;
        BaseVa += PAGE_SIZE;
        NumberOfPages -= 1;
    } while (NumberOfPages != 0);

    PointerPte->u.Long = MM_KERNEL_DEMAND_ZERO_PTE;
    return;
}


NTSTATUS
MmSetBankedSection (
    IN HANDLE ProcessHandle,
    IN PVOID VirtualAddress,
    IN ULONG BankLength,
    IN BOOLEAN ReadWriteBank,
    IN PBANKED_SECTION_ROUTINE BankRoutine,
    IN PVOID Context
    )

/*++

Routine Description:

    This function declares a mapped video buffer as a banked
    section.  This allows banked video devices (i.e., even
    though the video controller has a megabyte or so of memory,
    only a small bank (like 64k) can be mapped at any one time.

    In order to overcome this problem, the pager handles faults
    to this memory, unmaps the current bank, calls off to the
    video driver and then maps in the new bank.

    This function creates the neccessary structures to allow the
    video driver to be called from the pager.

 ********************* NOTE NOTE NOTE *************************
    At this time only read/write banks are supported!

Arguments:

    ProcessHandle - Supplies a handle to the process in which to
                    support the banked video function.

    VirtualAddress - Supplies the virtual address where the video
                     buffer is mapped in the specified process.

    BankLength - Supplies the size of the bank.

    ReadWriteBank - Supplies TRUE if the bank is read and write.

    BankRoutine - Supplies a pointer to the routine that should be
                  called by the pager.

    Context - Supplies a context to be passed by the pager to the
              BankRoutine.

Return Value:

    Returns the status of the function.

Environment:

    Kernel mode, APC_LEVEL or below.

--*/

{
    NTSTATUS Status;
    PEPROCESS Process;
    PMMVAD Vad;
    PMMPTE PointerPte;
    PMMPTE LastPte;
    MMPTE TempPte;
    ULONG size;
    LONG count;
    ULONG NumberOfPtes;
    PMMBANKED_SECTION Bank;

    PAGED_CODE ();

    //
    // Reference the specified process handle for VM_OPERATION access.
    //

    Status = ObReferenceObjectByHandle ( ProcessHandle,
                                         PROCESS_VM_OPERATION,
                                         PsProcessType,
                                         KernelMode,
                                         (PVOID *)&Process,
                                         NULL );

    if (!NT_SUCCESS(Status)) {
        return Status;
    }

    KeAttachProcess (&Process->Pcb);

    //
    // Get the address creation mutex to block multiple threads from
    // creating or deleting address space at the same time and
    // get the working set mutex so virtual address descriptors can
    // be inserted and walked.  Block APCs so an APC which takes a page
    // fault does not corrupt various structures.
    //

    LOCK_WS_AND_ADDRESS_SPACE (Process);

    //
    // Make sure the address space was not deleted, if so, return an error.
    //

    if (Process->AddressSpaceDeleted != 0) {
        Status = STATUS_PROCESS_IS_TERMINATING;
        goto ErrorReturn;
    }

    Vad = MiLocateAddress (VirtualAddress);

    if ((Vad == NULL) ||
        (Vad->StartingVa != VirtualAddress) ||
        (Vad->u.VadFlags.PhysicalMapping == 0)) {
        Status = STATUS_NOT_MAPPED_DATA;
        goto ErrorReturn;
    }

    size = 1 + (ULONG)Vad->EndingVa - (ULONG)Vad->StartingVa;
    if ((size % BankLength) != 0) {
        Status = STATUS_INVALID_VIEW_SIZE;
        goto ErrorReturn;
    }

    count = -1;
    NumberOfPtes = BankLength;

    do {
        NumberOfPtes = NumberOfPtes >> 1;
        count += 1;
    } while (NumberOfPtes != 0);

    //
    // Turn VAD into Banked VAD
    //

    NumberOfPtes = BankLength >> PAGE_SHIFT;

    Bank = ExAllocatePoolWithTag (NonPagedPool,
                                    sizeof (MMBANKED_SECTION) +
                                       (NumberOfPtes - 1) * sizeof(MMPTE),
                                    '  mM');
    if (Bank == NULL) {
        Status = STATUS_INSUFFICIENT_RESOURCES;
        goto ErrorReturn;
    }

    Bank->BankShift = PTE_SHIFT + count - PAGE_SHIFT;

    PointerPte = MiGetPteAddress(Vad->StartingVa);
    ASSERT (PointerPte->u.Hard.Valid == 1);

    Vad->Banked = Bank;
    Bank->BasePhysicalPage = PointerPte->u.Hard.PageFrameNumber;
    Bank->BasedPte = PointerPte;
    Bank->BankSize = BankLength;
    Bank->BankedRoutine = BankRoutine;
    Bank->Context = Context;
    Bank->CurrentMappedPte = PointerPte;

    //
    // Build template PTEs the structure.
    //

    count = 0;
    TempPte = ZeroPte;

    MI_MAKE_VALID_PTE (TempPte,
                       Bank->BasePhysicalPage,
                       MM_READWRITE,
                       PointerPte);

    if (TempPte.u.Hard.Write) {
        MI_SET_PTE_DIRTY (TempPte);
    }

    do {
        Bank->BankTemplate[count] = TempPte;
        TempPte.u.Hard.PageFrameNumber += 1;
        count += 1;
    } while ((ULONG)count < NumberOfPtes );

    LastPte = MiGetPteAddress (Vad->EndingVa);

    //
    // Set all PTEs within this range to zero.  Any faults within
    // this range will call the banked routine before making the
    // page valid.
    //

    RtlFillMemory (PointerPte,
                   (size >> (PAGE_SHIFT - PTE_SHIFT)),
                   (UCHAR)ZeroPte.u.Long);

    MiFlushTb ();

    Status = STATUS_SUCCESS;
ErrorReturn:

    UNLOCK_WS (Process);
    UNLOCK_ADDRESS_SPACE (Process);
    KeDetachProcess();
    return Status;
}

PVOID
MmMapVideoDisplay (
     IN PHYSICAL_ADDRESS PhysicalAddress,
     IN ULONG NumberOfBytes,
     IN MEMORY_CACHING_TYPE CacheType
     )

/*++

Routine Description:

    This function maps the specified physical address into the non-pageable
    portion of the system address space.

Arguments:

    PhysicalAddress - Supplies the starting physical address to map.

    NumberOfBytes - Supplies the number of bytes to map.

    CacheType - Supplies MmNonCached if the phyiscal address is to be mapped
                as non-cached, MmCached if the address should be cached, and
                MmCacheFrameBuffer if the address should be cached as a frame
                buffer. For I/O device registers, this is usually specified
                as MmNonCached.

Return Value:

    Returns the virtual address which maps the specified physical addresses.
    The value NULL is returned if sufficient virtual address space for
    the mapping could not be found.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{
    ULONG NumberOfPages;
    ULONG PageFrameIndex;
    PMMPTE PointerPte = NULL;
    PVOID BaseVa;
    MMPTE TempPte;
#ifdef LARGE_PAGES
    ULONG size;
    PMMPTE protoPte;
    PMMPTE largePte;
    ULONG pageSize;
    PSUBSECTION Subsection;
    ULONG Alignment;
#endif LARGE_PAGES
    ULONG LargePages = FALSE;

#ifdef i386
    ASSERT (PhysicalAddress.HighPart == 0);
#endif
#ifdef R4000
    ASSERT (PhysicalAddress.HighPart < 16);
#endif

    PAGED_CODE();

    ASSERT (NumberOfBytes != 0);

#ifdef LARGE_PAGES
    NumberOfPages = COMPUTE_PAGES_SPANNED (PhysicalAddress.LowPart,
                                           NumberOfBytes);

    TempPte = ValidKernelPte;
    MI_DISABLE_CACHING (TempPte);
    PageFrameIndex = (ULONG)(PhysicalAddress.QuadPart >> PAGE_SHIFT);
    TempPte.u.Hard.PageFrameNumber = PageFrameIndex;

    if ((NumberOfBytes > X64K) && (!MmLargeVideoMapped)) {
        size = (NumberOfBytes - 1) >> (PAGE_SHIFT + 1);
        pageSize = PAGE_SIZE;

        while (size != 0) {
            size = size >> 2;
            pageSize = pageSize << 2;
        }

        Alignment = pageSize << 1;
        if (Alignment < MM_VA_MAPPED_BY_PDE) {
            Alignment = MM_VA_MAPPED_BY_PDE;
        }
        NumberOfPages = Alignment >> PAGE_SHIFT;
        PointerPte = MiReserveSystemPtes(NumberOfPages,
                                         SystemPteSpace,
                                         Alignment,
                                         0,
                                         FALSE);
        protoPte = ExAllocatePoolWithTag (PagedPool,
                                           sizeof (MMPTE),
                                           'bSmM');
        if ((PointerPte != NULL) && (protoPte != NULL)) {

            RtlFillMemoryUlong (PointerPte,
                                Alignment >> (PAGE_SHIFT - PTE_SHIFT),
                                MM_ZERO_KERNEL_PTE);

            //
            // Build large page descriptor and fill in all the PTEs.
            //

            Subsection = ExAllocatePoolWithTag (NonPagedPoolMustSucceed,
                                         sizeof(SUBSECTION) + (4 * sizeof(MMPTE)),
                                         'bSmM');

            Subsection->StartingSector = pageSize;
            Subsection->EndingSector = (ULONG)NumberOfPages;
            Subsection->u.LongFlags = 0;
            Subsection->u.SubsectionFlags.LargePages = 1;
            Subsection->u.SubsectionFlags.Protection = MM_READWRITE | MM_NOCACHE;
            Subsection->PtesInSubsection = Alignment;
            Subsection->SubsectionBase = PointerPte;

            largePte = (PMMPTE)(Subsection + 1);

            //
            // Build the first 2 ptes as entries for the TLB to
            // map the specified physical address.
            //

            *largePte = TempPte;
            largePte += 1;

            if (NumberOfBytes > pageSize) {
                *largePte = TempPte;
                largePte->u.Hard.PageFrameNumber += (pageSize >> PAGE_SHIFT);
            } else {
                *largePte = ZeroKernelPte;
            }

            //
            // Build the first prototype PTE as a paging file format PTE
            // referring to the subsection.
            //

            protoPte->u.Long = (ULONG)MiGetSubsectionAddressForPte(Subsection);
            protoPte->u.Soft.Prototype = 1;
            protoPte->u.Soft.Protection = MM_READWRITE | MM_NOCACHE;

            //
            // Set the PTE up for all the user's PTE entries, proto pte
            // format pointing to the 3rd prototype PTE.
            //

            TempPte.u.Long = MiProtoAddressForPte (protoPte);
            MI_SET_GLOBAL_STATE (TempPte, 1);
            LargePages = TRUE;
            MmLargeVideoMapped = TRUE;
        }
    }
    BaseVa = (PVOID)MiGetVirtualAddressMappedByPte (PointerPte);
    BaseVa = (PVOID)((PCHAR)BaseVa + BYTE_OFFSET(PhysicalAddress.LowPart));

    if (PointerPte != NULL) {

        do {
            ASSERT (PointerPte->u.Hard.Valid == 0);
            *PointerPte = TempPte;
            PointerPte++;
            NumberOfPages -= 1;
        } while (NumberOfPages != 0);
    } else {
#endif //LARGE_PAGES

        BaseVa = MmMapIoSpace (PhysicalAddress,
                               NumberOfBytes,
                               CacheType);
#ifdef LARGE_PAGES
    }
#endif //LARGE_PAGES

    return BaseVa;
}

VOID
MmUnmapVideoDisplay (
     IN PVOID BaseAddress,
     IN ULONG NumberOfBytes
     )

/*++

Routine Description:

    This function unmaps a range of physical address which were previously
    mapped via an MmMapVideoDisplay function call.

Arguments:

    BaseAddress - Supplies the base virtual address where the physical
                  address was previously mapped.

    NumberOfBytes - Supplies the number of bytes which were mapped.

Return Value:

    None.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{

#ifdef LARGE_PAGES
    ULONG NumberOfPages;
    ULONG i;
    PMMPTE FirstPte;
    KIRQL OldIrql;
    PMMPTE LargePte;
    PSUBSECTION Subsection;

    PAGED_CODE();

    ASSERT (NumberOfBytes != 0);
    NumberOfPages = COMPUTE_PAGES_SPANNED (BaseAddress, NumberOfBytes);
    FirstPte = MiGetPteAddress (BaseAddress);

    if ((NumberOfBytes > X64K) && (FirstPte->u.Hard.Valid == 0)) {

        ASSERT (MmLargeVideoMapped);
        LargePte = MiPteToProto (FirstPte);
        Subsection = MiGetSubsectionAddress (LargePte);
        ASSERT (Subsection->SubsectionBase == FirstPte);
        NumberOfPages = Subsection->PtesInSubsection;
        ExFreePool (Subsection);
        ExFreePool (LargePte);
        MmLargeVideoMapped = FALSE;
        KeFillFixedEntryTb ((PHARDWARE_PTE)FirstPte, (PVOID)KSEG0_BASE, LARGE_ENTRY);
    }
    MiReleaseSystemPtes(FirstPte, NumberOfPages, SystemPteSpace);
    return;

#else // LARGE_PAGES

    MmUnmapIoSpace (BaseAddress, NumberOfBytes);
    return;
#endif //LARGE_PAGES
}


VOID
MiFlushTb (
    VOID
    )

/*++

Routine Description:

     Nonpagable wrapper.

Arguments:

Return Value:

    None.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{
    KIRQL OldIrql;

    KeRaiseIrql (DISPATCH_LEVEL, &OldIrql);
    KeFlushEntireTb (TRUE, TRUE);
    KeLowerIrql (OldIrql);
}


VOID
MmLockPagedPool (
    IN PVOID Address,
    IN ULONG Size
    )

/*++

Routine Description:

    Locks the specified address (which MUST reside in paged pool) into
    memory until MmUnlockPagedPool is called.

Arguments:

    Address - Supplies the address in paged pool to lock.

    Size - Supplies the size to lock.

Return Value:

    None.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{
    PMMPTE PointerPte;
    PMMPTE LastPte;
    KIRQL OldIrql;

    MmLockPagableSectionByHandle(ExPageLockHandle);
    PointerPte = MiGetPteAddress (Address);
    LastPte = MiGetPteAddress ((PVOID)((PCHAR)Address + (Size - 1)));
    LOCK_PFN (OldIrql);
    MiLockCode (PointerPte, LastPte, MM_LOCK_BY_REFCOUNT);
    UNLOCK_PFN (OldIrql);
    MmUnlockPagableImageSection(ExPageLockHandle);
    return;
}

NTKERNELAPI
VOID
MmUnlockPagedPool (
    IN PVOID Address,
    IN ULONG Size
    )

/*++

Routine Description:

    Unlocks paged pool that was locked with MmLockPagedPool.

Arguments:

    Address - Supplies the address in paged pool to unlock.

    Size - Supplies the size to unlock.

Return Value:

    None.

Environment:

    Kernel mode, IRQL of APC_LEVEL or below.

--*/

{
    PMMPTE PointerPte;
    PMMPTE LastPte;
    KIRQL OldIrql;

    MmLockPagableSectionByHandle(ExPageLockHandle);
    PointerPte = MiGetPteAddress (Address);
    LastPte = MiGetPteAddress ((PVOID)((PCHAR)Address + (Size - 1)));
    LOCK_PFN2 (OldIrql);

    do {
#if DBG
        {   PMMPFN Pfn;
            ASSERT (PointerPte->u.Hard.Valid == 1);
            Pfn = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
            ASSERT (Pfn->u3.e2.ReferenceCount > 1);
        }
#endif //DBG

        MiDecrementReferenceCount (PointerPte->u.Hard.PageFrameNumber);
        PointerPte += 1;
    } while (PointerPte <= LastPte);

    UNLOCK_PFN2 (OldIrql);
    MmUnlockPagableImageSection(ExPageLockHandle);
    return;
}