path: root/private/ntos/afd/fastio.c

/*++

Copyright (c) 1989  Microsoft Corporation

Module Name:

    fastio.c

Abstract:

    This module contains routines for handling fast ("turbo") IO
    in AFD.

Author:

    David Treadwell (davidtr)    12-Oct-1992

Revision History:

--*/

#include "afdp.h"

BOOLEAN
AfdFastDatagramIo (
    IN struct _FILE_OBJECT *FileObject,
    IN PVOID InputBuffer OPTIONAL,
    IN ULONG InputBufferLength,
    IN ULONG IoControlCode,
    OUT PIO_STATUS_BLOCK IoStatus
    );

BOOLEAN
AfdFastDatagramReceive (
    IN PAFD_ENDPOINT Endpoint,
    IN ULONG ReceiveFlags,
    IN ULONG AfdFlags,
    IN LPWSABUF BufferArray,
    IN ULONG BufferCount,
    IN ULONG ReceiveBufferLength,
    OUT PVOID SourceAddress,
    OUT PULONG SourceAddressLength,
    OUT PIO_STATUS_BLOCK IoStatus
    );

BOOLEAN
AfdFastDatagramSend (
    IN PAFD_BUFFER AfdBuffer,
    IN PAFD_ENDPOINT Endpoint,
    OUT PIO_STATUS_BLOCK IoStatus
    );

NTSTATUS
AfdRestartFastSendDatagram (
    IN PDEVICE_OBJECT DeviceObject,
    IN PIRP Irp,
    IN PVOID Context
    );

PAFD_BUFFER
CopyAddressToBuffer (
    IN PAFD_ENDPOINT Endpoint,
    IN ULONG OutputBufferLength
    );

#ifdef ALLOC_PRAGMA
#pragma alloc_text( PAGE, AfdFastDatagramIo )
#pragma alloc_text( PAGE, AfdFastDatagramSend )
#pragma alloc_text( PAGE, AfdFastIoRead )
#pragma alloc_text( PAGE, AfdFastIoWrite )
#pragma alloc_text( PAGEAFD, AfdFastIoDeviceControl )
#pragma alloc_text( PAGEAFD, AfdFastDatagramReceive )
#pragma alloc_text( PAGEAFD, AfdRestartFastSendDatagram )
#pragma alloc_text( PAGEAFD, CopyAddressToBuffer )
#pragma alloc_text( PAGEAFD, AfdShouldSendBlock )
#endif

BOOLEAN
AfdFastIoRead (
    IN struct _FILE_OBJECT *FileObject,
    IN PLARGE_INTEGER FileOffset,
    IN ULONG Length,
    IN BOOLEAN Wait,
    IN ULONG LockKey,
    OUT PVOID Buffer,
    OUT PIO_STATUS_BLOCK IoStatus,
    IN struct _DEVICE_OBJECT *DeviceObject
    )
{

    AFD_RECV_INFO recvInfo;
    WSABUF wsaBuf;

    PAGED_CODE( );

    UNREFERENCED_PARAMETER( FileOffset );
    UNREFERENCED_PARAMETER( LockKey );

    //
    // Build the (one and only) WSABUF.
    //

    wsaBuf.buf = Buffer;
    wsaBuf.len = Length;

    //
    // Setup the AFD_RECV_INFO structure.
    //

    recvInfo.BufferArray = &wsaBuf;
    recvInfo.BufferCount = 1;
    recvInfo.AfdFlags = AFD_OVERLAPPED;
    recvInfo.TdiFlags = TDI_RECEIVE_NORMAL;

    //
    // Fake an ioctl.
    //

    return AfdFastIoDeviceControl(
               FileObject,
               Wait,
               &recvInfo,
               sizeof(recvInfo),
               NULL,
               0,
               IOCTL_AFD_RECEIVE,
               IoStatus,
               DeviceObject
               );

} // AfdFastIoRead

BOOLEAN
AfdFastIoWrite (
    IN struct _FILE_OBJECT *FileObject,
    IN PLARGE_INTEGER FileOffset,
    IN ULONG Length,
    IN BOOLEAN Wait,
    IN ULONG LockKey,
    IN PVOID Buffer,
    OUT PIO_STATUS_BLOCK IoStatus,
    IN struct _DEVICE_OBJECT *DeviceObject
    )
{

    AFD_SEND_INFO sendInfo;
    WSABUF wsaBuf;

    PAGED_CODE( );

    UNREFERENCED_PARAMETER( FileOffset );
    UNREFERENCED_PARAMETER( LockKey );

    //
    // Build the (one and only) WSABUF.
    //

    wsaBuf.buf = Buffer;
    wsaBuf.len = Length;

    //
    // Setup the AFD_SEND_INFO structure.
    //

    sendInfo.BufferArray = &wsaBuf;
    sendInfo.BufferCount = 1;
    sendInfo.AfdFlags = AFD_OVERLAPPED;
    sendInfo.TdiFlags = 0;

    //
    // Fake an ioctl.
    //

    return AfdFastIoDeviceControl(
               FileObject,
               Wait,
               &sendInfo,
               sizeof(sendInfo),
               NULL,
               0,
               IOCTL_AFD_SEND,
               IoStatus,
               DeviceObject
               );

} // AfdFastIoWrite

#if AFD_PERF_DBG

BOOLEAN
AfdFastIoDeviceControlReal (
    IN struct _FILE_OBJECT *FileObject,
    IN BOOLEAN Wait,
    IN PVOID InputBuffer OPTIONAL,
    IN ULONG InputBufferLength,
    OUT PVOID OutputBuffer OPTIONAL,
    IN ULONG OutputBufferLength,
    IN ULONG IoControlCode,
    OUT PIO_STATUS_BLOCK IoStatus
    );


BOOLEAN
AfdFastIoDeviceControl (
    IN struct _FILE_OBJECT *FileObject,
    IN BOOLEAN Wait,
    IN PVOID InputBuffer OPTIONAL,
    IN ULONG InputBufferLength,
    OUT PVOID OutputBuffer OPTIONAL,
    IN ULONG OutputBufferLength,
    IN ULONG IoControlCode,
    OUT PIO_STATUS_BLOCK IoStatus,
    IN struct _DEVICE_OBJECT *DeviceObject
    )
{
    BOOLEAN success;

    if ( AfdDisableFastIo ) {
        return FALSE;
    }

    ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL );

    success = AfdFastIoDeviceControlReal (
                  FileObject,
                  Wait,
                  InputBuffer,
                  InputBufferLength,
                  OutputBuffer,
                  OutputBufferLength,
                  IoControlCode,
                  IoStatus
                  );

    ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL );

    switch ( IoControlCode ) {

    case IOCTL_AFD_SEND:

        if ( success ) {
            AfdFastSendsSucceeded++;
        } else {
            AfdFastSendsFailed++;
        }
        break;

    case IOCTL_AFD_RECEIVE:

        if ( success ) {
            AfdFastReceivesSucceeded++;
        } else {
            AfdFastReceivesFailed++;
        }
        break;

    case IOCTL_AFD_SEND_DATAGRAM:

        if ( success ) {
            AfdFastSendDatagramsSucceeded++;
        } else {
            AfdFastSendDatagramsFailed++;
        }
        break;

    case IOCTL_AFD_RECEIVE_DATAGRAM:

        if ( success ) {
            AfdFastReceiveDatagramsSucceeded++;
        } else {
            AfdFastReceiveDatagramsFailed++;
        }
        break;

    case IOCTL_AFD_POLL:

        if ( success ) {
            AfdFastPollsSucceeded++;
        } else {
            AfdFastPollsFailed++;
        }
        break;
    }

    return success;

} // AfdFastIoDeviceControl

BOOLEAN
AfdFastIoDeviceControlReal (
    IN struct _FILE_OBJECT *FileObject,
    IN BOOLEAN Wait,
    IN PVOID InputBuffer OPTIONAL,
    IN ULONG InputBufferLength,
    OUT PVOID OutputBuffer OPTIONAL,
    IN ULONG OutputBufferLength,
    IN ULONG IoControlCode,
    OUT PIO_STATUS_BLOCK IoStatus
    )
#else

BOOLEAN
AfdFastIoDeviceControl (
    IN struct _FILE_OBJECT *FileObject,
    IN BOOLEAN Wait,
    IN PVOID InputBuffer OPTIONAL,
    IN ULONG InputBufferLength,
    OUT PVOID OutputBuffer OPTIONAL,
    IN ULONG OutputBufferLength,
    IN ULONG IoControlCode,
    OUT PIO_STATUS_BLOCK IoStatus,
    IN struct _DEVICE_OBJECT *DeviceObject
    )
#endif
{
    PAFD_ENDPOINT endpoint;
    PAFD_CONNECTION connection;
    KIRQL oldIrql;
    PAFD_BUFFER afdBuffer;
    NTSTATUS status;
    PMDL MdlChain;

    //
    // All we want to do is pass the request through to the TDI provider
    // if possible.  First get the endpoint and connection pointers.
    //

    endpoint = FileObject->FsContext;
    ASSERT( IS_AFD_ENDPOINT_TYPE( endpoint ) );

    //
    // If the endpoint is shut down in any way, bail out of fast IO.
    //

    if ( endpoint->DisconnectMode != 0 ) {
        return FALSE;
    }

    //
    // If an OutputBuffer parameter was specified, then this is a more
    // complicated IO request, so bail on fast IO.
    //

    if( OutputBuffer != NULL ) {
        return FALSE;
    }

    //
    // Handle datagram fast IO in a subroutine.  This keeps this routine
    // cleaner and faster.
    //

    if ( IS_DGRAM_ENDPOINT(endpoint) ) {
        return AfdFastDatagramIo(
                   FileObject,
                   InputBuffer,
                   InputBufferLength,
                   IoControlCode,
                   IoStatus
                   );
    }

    //
    // If the endpoint isn't connected yet, then we don't want to
    // attempt fast IO on it.
    //

    if ( endpoint->State != AfdEndpointStateConnected ) {
        return FALSE;
    }

    ASSERT( endpoint->Type == AfdBlockTypeVcConnecting );
    ASSERT( endpoint->Common.VcConnecting.Connection != NULL );

    //
    // If the TDI provider for this endpoint supports bufferring,
    // don't use fast IO.
    //

    if ( endpoint->TdiBufferring ) {
        return FALSE;
    }

    connection = endpoint->Common.VcConnecting.Connection;
    ASSERT( connection->Type == AfdBlockTypeConnection );

    if( connection->CleanupBegun ) {
        return FALSE;
    }

    IF_DEBUG(FAST_IO) {
        KdPrint(( "AfdFastIoDeviceControl: attempting fast IO on endp %lx, "
                  "conn %lx, code %lx\n",
                      endpoint, connection, IoControlCode ));
    }

    //
    // Based on whether this is a send or receive, attempt to perform
    // fast IO.
    //

    switch ( IoControlCode ) {

    case IOCTL_AFD_SEND: {

        PAFD_SEND_INFO sendInfo;
        ULONG sendLength;
        ULONG afdFlags;

        //
        // If the connection has been aborted, then we don't want to try
        // fast IO on it.
        //

        if ( connection->AbortIndicated ) {
            return FALSE;
        }

        //
        // If the input structure isn't large enough, bail on fast IO.
        //

        if( InputBufferLength < sizeof(*sendInfo) ) {
            return FALSE;
        }

        try {

            //
            // Make a quick preliminary check of the input buffer. If it's
            // bogus (or if Fast IO is disabled), then fail the request.
            //

            sendInfo = (PAFD_SEND_INFO)InputBuffer;
            afdFlags = sendInfo->AfdFlags;

            if( (afdFlags & AFD_NO_FAST_IO) != 0 ||
                sendInfo->TdiFlags != 0 ||
                sendInfo->BufferArray == NULL ||
                sendInfo->BufferCount == 0 ) {

                return FALSE;

            }

            //
            // Calculate the length of the send buffer.
            //

            sendLength = AfdCalcBufferArrayByteLengthRead(
                             sendInfo->BufferArray,
                             sendInfo->BufferCount
                             );

        } except( EXCEPTION_EXECUTE_HANDLER ) {

            return FALSE;
        }

        //
        // Determine whether we can do fast IO with this send.  In order
        // to perform fast IO, there must be no other sends pended on this
        // connection and there must be enough space left for bufferring
        // the requested amount of data.
        //

        if ( AfdShouldSendBlock( endpoint, connection, sendLength ) ) {

            //
            // If this is a nonblocking endpoint, fail the request here and
            // save going through the regular path.
            //

            if ( endpoint->NonBlocking && !( afdFlags & AFD_OVERLAPPED ) ) {
                IoStatus->Status = STATUS_DEVICE_NOT_READY;
                return TRUE;
            }

            return FALSE;
        }

        //
        // Next get an AFD buffer structure that contains an IRP and a
        // buffer to hold the data.
        //

        afdBuffer = AfdGetBuffer( sendLength, 0 );

        if ( afdBuffer == NULL ) {

            AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql );
            connection->VcBufferredSendBytes -= sendLength;
            connection->VcBufferredSendCount -= 1;
            AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );

            return FALSE;
        }

        //
        // We have to rebuild the MDL in the AFD buffer structure to
        // represent exactly the number of bytes we're going to be
        // sending.
        //

        afdBuffer->Mdl->ByteCount = sendLength;
        SET_CHAIN_LENGTH( afdBuffer, sendLength );

        //
        // Remember the endpoint in the AFD buffer structure.  We need
        // this in order to access the endpoint in the restart routine.
        //

        afdBuffer->Context = endpoint;

        //
        // Copy the user's data into the AFD buffer.
        //

        if( sendLength > 0 ) {

            try {

                AfdCopyBufferArrayToBuffer(
                    afdBuffer->Buffer,
                    sendLength,
                    sendInfo->BufferArray,
                    sendInfo->BufferCount
                    );

            } except( EXCEPTION_EXECUTE_HANDLER ) {

                afdBuffer->Mdl->ByteCount = afdBuffer->BufferLength;
                RESET_CHAIN_LENGTH( afdBuffer );
                AfdReturnBuffer( afdBuffer );

                AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql );
                connection->VcBufferredSendBytes -= sendLength;
                connection->VcBufferredSendCount -= 1;
                AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );

                return FALSE;

            }

        }

        //
        // Use the IRP in the AFD buffer structure to give to the TDI
        // provider.  Build the TDI send request.
        //

        TdiBuildSend(
            afdBuffer->Irp,
            connection->DeviceObject,
            connection->FileObject,
            AfdRestartBufferSend,
            afdBuffer,
            afdBuffer->Mdl,
            0,
            sendLength
            );

        //
        // Add a reference to the connection object since the send
        // request will complete asynchronously.
        //

        REFERENCE_CONNECTION( connection );

        //
        // Call the transport to actually perform the send.
        //

        status = IoCallDriver(
                     connection->DeviceObject,
                     afdBuffer->Irp
                     );

        //
        // Complete the user's IRP as appropriate.  Note that we change the
        // status code from what was returned by the TDI provider into
        // STATUS_SUCCESS.  This is because we don't want to complete
        // the IRP with STATUS_PENDING etc.
        //

        if ( NT_SUCCESS(status) ) {
            IoStatus->Information = sendLength;
            IoStatus->Status = STATUS_SUCCESS;
            return TRUE;
        }

        //
        // The call failed for some reason.  Fail fast IO.
        //

        return FALSE;

    }

    case IOCTL_AFD_RECEIVE: {

        PLIST_ENTRY listEntry;
        PAFD_RECV_INFO recvInfo;
        ULONG recvLength;
        ULONG totalOffset;

        //
        // If the input structure isn't large enough, bail on fast IO.
        //

        if( InputBufferLength < sizeof(*recvInfo) ) {
            return FALSE;
        }

        try {

            //
            // Make a quick preliminary check of the input buffer. If it's
            // bogus (or if Fast IO is disabled), then fail the request.
            //

            recvInfo = (PAFD_RECV_INFO)InputBuffer;

            if( (recvInfo->AfdFlags & AFD_NO_FAST_IO) != 0 ||
                recvInfo->TdiFlags != TDI_RECEIVE_NORMAL ||
                recvInfo->BufferArray == NULL ||
                recvInfo->BufferCount == 0 ) {

                return FALSE;

            }

            //
            // Calculate the length of the receive buffer.
            //

            recvLength = AfdCalcBufferArrayByteLengthWrite(
                             recvInfo->BufferArray,
                             recvInfo->BufferCount
                             );

        } except( EXCEPTION_EXECUTE_HANDLER ) {

            return FALSE;
        }

        //
        // Determine whether we'll be able to perform fast IO.  In order
        // to do fast IO, there must be some bufferred data on the
        // connection, there must not be any pended receives on the
        // connection, and there must not be any bufferred expedited
        // data on the connection.  This last requirement is for
        // the sake of simplicity only.
        //

        AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql );

        if ( connection->VcBufferredReceiveCount == 0 ||
                 !IsListEmpty( &connection->VcReceiveIrpListHead ) ||
                 connection->VcBufferredExpeditedCount != 0 ) {

            AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );
            return FALSE;
        }

        ASSERT( !IsListEmpty( &connection->VcReceiveBufferListHead ) );

        //
        // Get a pointer to the first bufferred AFD buffer structure on
        // the connection.
        //

        afdBuffer = CONTAINING_RECORD(
                        connection->VcReceiveBufferListHead.Flink,
                        AFD_BUFFER,
                        BufferListEntry
                        );

        ASSERT( !afdBuffer->ExpeditedData );

        //
        // If the buffer contains a partial message, bail out of the
        // fast path.  We don't want the added complexity of handling
        // partial messages in the fast path.
        //

        if ( afdBuffer->PartialMessage &&
                 endpoint->EndpointType != AfdEndpointTypeStream ) {

            AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );
            return FALSE;
        }

        //
        // For simplicity, act differently based on whether the user's
        // buffer is large enough to hold the entire first AFD buffer
        // worth of data.
        //

        if ( recvLength >= afdBuffer->DataLength ) {

            LIST_ENTRY bufferListHead;

            IoStatus->Status = STATUS_SUCCESS;
            IoStatus->Information = 0;

            InitializeListHead( &bufferListHead );

            //
            // Loop getting AFD buffers that will fill in the user's
            // buffer with as much data as will fit, or else with a
            // single buffer if this is not a stream endpoint.  We don't
            // actually do the copy within this loop because this loop
            // must occur while holding a lock, and we cannot hold a
            // lock while copying the data into the user's buffer
            // because the user's buffer is not locked and we cannot
            // take a page fault at raised IRQL.
            //

            do {

                //
                // Update the count of bytes on the connection.
                //

                ASSERT( connection->VcBufferredReceiveBytes >= afdBuffer->DataLength );
                ASSERT( connection->VcBufferredReceiveCount > 0 );

                connection->VcBufferredReceiveBytes -= afdBuffer->DataLength;
                connection->VcBufferredReceiveCount -= 1;
                IoStatus->Information += afdBuffer->DataLength;

                //
                // Remove the AFD buffer from the connection's list of
                // buffers and place it on our local list of buffers.
                //

                RemoveEntryList( &afdBuffer->BufferListEntry );
                InsertTailList( &bufferListHead, &afdBuffer->BufferListEntry );

                //
                // If this is a stream endpoint and all of the data in
                // the next AFD buffer will fit in the user's buffer,
                // use this buffer for the IO as well.
                //

                if ( !IsListEmpty( &connection->VcReceiveBufferListHead ) ) {

                    afdBuffer = CONTAINING_RECORD(
                                    connection->VcReceiveBufferListHead.Flink,
                                    AFD_BUFFER,
                                    BufferListEntry
                                    );

                    ASSERT( !afdBuffer->ExpeditedData );
                    ASSERT( afdBuffer->DataOffset == 0 );

                    if ( endpoint->EndpointType == AfdEndpointTypeStream &&
                             IoStatus->Information + afdBuffer->DataLength <=
                                 recvLength ) {
                        continue;
                    } else {
                        break;
                    }

                } else {

                    break;
                }

            } while ( TRUE );

            //
            // If there is indicated but unreceived data in the TDI provider,
            // and we have available buffer space, fire off an IRP to receive
            // the data.
            //

            if ( connection->VcReceiveCountInTransport > 0

                 &&

                 connection->VcBufferredReceiveBytes <
                   connection->MaxBufferredReceiveBytes

                 &&

                 connection->VcBufferredReceiveCount <
                     connection->MaxBufferredReceiveCount ) {

                CLONG bytesToReceive;

                //
                // Remember the count of data that we're going to receive,
                // then reset the fields in the connection where we keep
                // track of how much data is available in the transport.
                // We reset it here before releasing the lock so that
                // another thread doesn't try to receive the data at the
                // same time as us.
                //

                if ( connection->VcReceiveBytesInTransport > AfdLargeBufferSize ) {
                    bytesToReceive = connection->VcReceiveBytesInTransport;
                } else {
                    bytesToReceive = AfdLargeBufferSize;
                }

                ASSERT( connection->VcReceiveCountInTransport == 1 );
                connection->VcReceiveBytesInTransport = 0;
                connection->VcReceiveCountInTransport = 0;

                AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );

                //
                // Get an AFD buffer structure to hold the data.
                //

                afdBuffer = AfdGetBuffer( bytesToReceive, 0 );

                if ( afdBuffer == NULL ) {

                    //
                    // If we were unable to get a buffer, abort the
                    // circuit.
                    //

                    AfdBeginAbort( connection );

                } else {

                    //
                    // We need to remember the connection in the AFD buffer
                    // because we'll need to access it in the completion
                    // routine.
                    //

                    afdBuffer->Context = connection;

                    //
                    // Finish building the receive IRP to give to the TDI provider.
                    //

                    TdiBuildReceive(
                        afdBuffer->Irp,
                        connection->DeviceObject,
                        connection->FileObject,
                        AfdRestartBufferReceive,
                        afdBuffer,
                        afdBuffer->Mdl,
                        TDI_RECEIVE_NORMAL,
                        bytesToReceive
                        );

                    //
                    // Hand off the IRP to the TDI provider.
                    //

                    (VOID)IoCallDriver(
                             connection->DeviceObject,
                             afdBuffer->Irp
                             );
                }

            } else {

               AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );
            }

            //
            // We have in a local list all the data we'll use for this
            // IO.  Start copying data to the user buffer.
            //

            totalOffset = 0;

            try {

                while ( !IsListEmpty( &bufferListHead ) ) {

                    //
                    // Take the first buffer from the list.
                    //

                    listEntry = RemoveHeadList( &bufferListHead );
                    afdBuffer = CONTAINING_RECORD(
                                    listEntry,
                                    AFD_BUFFER,
                                    BufferListEntry
                                    );

                    if( afdBuffer->DataLength > 0 ) {

                        //
                        // Copy the data in the buffer to the user buffer.
                        //

                        AfdCopyBufferToBufferArray(
                            recvInfo->BufferArray,
                            totalOffset,
                            recvInfo->BufferCount,
                            (PCHAR)afdBuffer->Buffer + afdBuffer->DataOffset,
                            afdBuffer->DataLength
                            );

                        totalOffset += afdBuffer->DataLength;

                    }

                    //
                    // We're done with the AFD buffer.
                    //

                    afdBuffer->DataOffset = 0;
                    RESET_CHAIN_LENGTH( afdBuffer );
                    AfdReturnBuffer( afdBuffer );
                }

            } except( EXCEPTION_EXECUTE_HANDLER ) {

                //
                // If an exception is hit, there is the possibility of
                // data corruption.  However, it is nearly impossible to
                // avoid this in all cases, so just throw out the
                // remainder of the data that we would have copied to
                // the user buffer.
                //

                afdBuffer->DataOffset = 0;
                RESET_CHAIN_LENGTH( afdBuffer );
                AfdReturnBuffer( afdBuffer );

                while ( !IsListEmpty( &bufferListHead ) ) {
                    listEntry = RemoveHeadList( &bufferListHead );
                    afdBuffer = CONTAINING_RECORD(
                                    listEntry,
                                    AFD_BUFFER,
                                    BufferListEntry
                                    );
                    RESET_CHAIN_LENGTH( afdBuffer );
                    AfdReturnBuffer( afdBuffer );
                }

                return FALSE;
            }

            //
            // Clear the receive data active bit. If there's more data
            // available, set the corresponding event.
            //

            AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql );

            endpoint->EventsActive &= ~AFD_POLL_RECEIVE;

            if( !IsListEmpty( &connection->VcReceiveBufferListHead ) ) {

                AfdIndicateEventSelectEvent(
                    endpoint,
                    AFD_POLL_RECEIVE_BIT,
                    STATUS_SUCCESS
                    );

            }

            AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );

            //
            // Fast IO succeeded!
            //

            ASSERT( IoStatus->Information <= recvLength );

            return TRUE;

        } else {

            PAFD_BUFFER tempAfdBuffer;

            //
            // If this is not a stream endpoint and the user's buffer
            // is insufficient to hold the entire message, then we cannot
            // use fast IO because this IO will need to fail with
            // STATUS_BUFFER_OVERFLOW.
            //

            if ( endpoint->EndpointType != AfdEndpointTypeStream ||
                 recvLength == 0 ) {
                AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );
                return FALSE;
            }

            //
            // This is a stream endpoint and the user buffer is
            // insufficient for the amount of data stored in the first
            // buffer.  So that we can perform fast IO and still
            // preserve data ordering, we're going to allocate a new AFD
            // buffer structure, copy the appropriate amount of data
            // into that buffer, then release locks and copy the data
            // into the user buffer.
            //
            // The extra copy incurred is still less than the IRP
            // overhead incurred in the normal IO path, so using fast IO
            // here is a win.  Also, applications which force us through
            // this path will typically be using very small receives,
            // like one or four bytes, so the copy will be short.
            //
            // First allocate an AFD buffer to hold the data we're
            // eventually going to give to the user.
            //

            tempAfdBuffer = AfdGetBuffer( recvLength, 0 );
            if ( tempAfdBuffer == NULL ) {
                AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );
                return FALSE;
            }

            //
            // Copy the first part of the bufferred data into our local
            // AFD buffer.
            //

            RtlCopyMemory(
                tempAfdBuffer->Buffer,
                (PCHAR)afdBuffer->Buffer + afdBuffer->DataOffset,
                recvLength
                );

            //
            // Update the data length and offset in the data bufferred
            // on the connection.
            //

            afdBuffer->DataLength -= recvLength;
            afdBuffer->DataOffset += recvLength;
            connection->VcBufferredReceiveBytes -= recvLength;

            //
            // If there is indicated but unreceived data in the TDI provider,
            // and we have available buffer space, fire off an IRP to receive
            // the data.
            //

            if ( connection->VcReceiveBytesInTransport > 0

                 &&

                 connection->VcBufferredReceiveBytes <
                   connection->MaxBufferredReceiveBytes

                 &&

                 connection->VcBufferredReceiveCount <
                     connection->MaxBufferredReceiveCount ) {

                CLONG bytesInTransport;

                //
                // Remember the count of data that we're going to receive,
                // then reset the fields in the connection where we keep
                // track of how much data is available in the transport.
                // We reset it here before releasing the lock so that
                // another thread doesn't try to receive the data at the
                // same time as us.
                //

                bytesInTransport = connection->VcReceiveBytesInTransport;

                ASSERT( connection->VcReceiveCountInTransport == 1 );
                connection->VcReceiveBytesInTransport = 0;
                connection->VcReceiveCountInTransport = 0;

                AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );

                //
                // Get an AFD buffer structure to hold the data.
                //

                afdBuffer = AfdGetBuffer( bytesInTransport, 0 );

                if ( afdBuffer == NULL ) {

                    //
                    // If we were unable to get a buffer, abort the
                    // circuit.
                    //

                    AfdBeginAbort( connection );

                } else {

                    //
                    // We need to remember the connection in the AFD buffer
                    // because we'll need to access it in the completion
                    // routine.
                    //

                    afdBuffer->Context = connection;

                    //
                    // Finish building the receive IRP to give to the TDI provider.
                    //

                    TdiBuildReceive(
                        afdBuffer->Irp,
                        connection->DeviceObject,
                        connection->FileObject,
                        AfdRestartBufferReceive,
                        afdBuffer,
                        afdBuffer->Mdl,
                        TDI_RECEIVE_NORMAL,
                        bytesInTransport
                        );

                    //
                    // Hand off the IRP to the TDI provider.
                    //

                    (VOID)IoCallDriver(
                             connection->DeviceObject,
                             afdBuffer->Irp
                             );
                }

            } else {

               AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );
            }

            //
            // Now copy the data to the user buffer inside an exception
            // handler.
            //

            try {

                AfdCopyBufferToBufferArray(
                    recvInfo->BufferArray,
                    0,
                    recvInfo->BufferCount,
                    tempAfdBuffer->Buffer,
                    recvLength
                    );

            } except( EXCEPTION_EXECUTE_HANDLER ) {

                RESET_CHAIN_LENGTH( tempAfdBuffer );
                AfdReturnBuffer( tempAfdBuffer );
                return FALSE;
            }

            //
            // Clear the receive data active bit. If there's more data
            // available, set the corresponding event.
            //

            AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql );

            endpoint->EventsActive &= ~AFD_POLL_RECEIVE;

            if( !IsListEmpty( &connection->VcReceiveBufferListHead ) ) {

                AfdIndicateEventSelectEvent(
                    endpoint,
                    AFD_POLL_RECEIVE_BIT,
                    STATUS_SUCCESS
                    );

            }

            AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql );

            //
            // Fast IO succeeded!
            //

            RESET_CHAIN_LENGTH( tempAfdBuffer );
            AfdReturnBuffer( tempAfdBuffer );

            IoStatus->Status = STATUS_SUCCESS;
            IoStatus->Information = recvLength;

            return TRUE;
        }
    }

    case IOCTL_AFD_TRANSMIT_FILE:

        return AfdFastTransmitFile(
                   FileObject,
                   InputBuffer,
                   InputBufferLength,
                   IoStatus
                   );


    default:

        return FALSE;
    }

    return FALSE;

} // AfdFastDeviceIoControlFile


BOOLEAN
AfdFastDatagramIo (
    IN struct _FILE_OBJECT *FileObject,
    IN PVOID InputBuffer OPTIONAL,
    IN ULONG InputBufferLength,
    IN ULONG IoControlCode,
    OUT PIO_STATUS_BLOCK IoStatus
    )
{
    PAFD_ENDPOINT endpoint;
    PAFD_BUFFER afdBuffer;

    PAGED_CODE( );

    //
    // All we want to do is pass the request through to the TDI provider
    // if possible.  First get the endpoint pointer.
    //

    endpoint = FileObject->FsContext;
    ASSERT( endpoint->Type == AfdBlockTypeDatagram );

    IF_DEBUG(FAST_IO) {
        KdPrint(( "AfdFastDatagramIo: attempting fast IO on endp %lx, "
                  "code %lx\n",
                      endpoint, IoControlCode ));
    }

    switch ( IoControlCode ) {

    case IOCTL_AFD_SEND: {

        PAFD_SEND_INFO sendInfo;
        ULONG sendLength;

        //
        // If the input structure isn't large enough, bail on fast IO.
        //

        if( InputBufferLength < sizeof(*sendInfo) ) {
            return FALSE;
        }

        try {

            //
            // Make a quick preliminary check of the input buffer. If it's
            // bogus (or if Fast IO is disabled), then fail the request.
            //

            sendInfo = (PAFD_SEND_INFO)InputBuffer;

            if( (sendInfo->AfdFlags & AFD_NO_FAST_IO) != 0 ||
                sendInfo->TdiFlags != 0 ||
                sendInfo->BufferArray == NULL ||
                sendInfo->BufferCount == 0 ) {

                return FALSE;

            }

            //
            // Calculate the length of the send buffer.
            //

            sendLength = AfdCalcBufferArrayByteLengthRead(
                             sendInfo->BufferArray,
                             sendInfo->BufferCount
                             );

        } except( EXCEPTION_EXECUTE_HANDLER ) {

            return FALSE;
        }

        //
        // If this is a send for more than the threshold number of
        // bytes, don't use the fast path.  We don't allow larger sends
        // in the fast path because of the extra data copy it entails,
        // which is more expensive for large buffers.  For smaller
        // buffers, however, the cost of the copy is small compared to
        // the IO system overhead of the slow path.
        //

        if ( sendLength > AfdFastSendDatagramThreshold ) {
            return FALSE;
        }

        //
        // In a subroutine, copy the destination address to the AFD
        // buffer.  We do this in a subroutine because it needs to
        // acquire a spin lock and we want this routine to be pageable.
        //

        afdBuffer = CopyAddressToBuffer( endpoint, sendLength );
        if ( afdBuffer == NULL ) {
            return FALSE;
        }

        //
        // Store the length of the data we're going to send.
        //

        afdBuffer->DataLength = sendLength;

        //
        // Copy the output buffer to the AFD buffer.
        //

        try {

            AfdCopyBufferArrayToBuffer(
                afdBuffer->Buffer,
                sendLength,
                sendInfo->BufferArray,
                sendInfo->BufferCount
                );

        } except( EXCEPTION_EXECUTE_HANDLER ) {

            RESET_CHAIN_LENGTH( afdBuffer );
            AfdReturnBuffer( afdBuffer );
            return FALSE;
        }

        //
        // Call a subroutine to complete the work of the fast path.
        //

        return AfdFastDatagramSend( afdBuffer, endpoint, IoStatus );

    }

    case IOCTL_AFD_SEND_DATAGRAM: {

        PTDI_REQUEST_SEND_DATAGRAM tdiRequest;
        ULONG destinationAddressLength;
        PAFD_SEND_DATAGRAM_INFO sendInfo;
        ULONG sendLength;

        //
        // If the input structure isn't large enough, bail on fast IO.
        //

        if( InputBufferLength < sizeof(*sendInfo) ) {
            return FALSE;
        }

        try {

            //
            // Make a quick preliminary check of the input buffer. If it's
            // bogus (or if Fast IO is disabled), then fail the request.
            //

            sendInfo = (PAFD_SEND_DATAGRAM_INFO)InputBuffer;

            if( (sendInfo->AfdFlags & AFD_NO_FAST_IO) != 0 ||
                sendInfo->BufferArray == NULL ||
                sendInfo->BufferCount == 0 ) {

                return FALSE;

            }

            //
            // Calculate the length of the send buffer.
            //

            sendLength = AfdCalcBufferArrayByteLengthRead(
                             sendInfo->BufferArray,
                             sendInfo->BufferCount
                             );

        } except( EXCEPTION_EXECUTE_HANDLER ) {

            return FALSE;
        }

        //
        // If this is a send for more than the threshold number of
        // bytes, don't use the fast path.  We don't allow larger sends
        // in the fast path because of the extra data copy it entails,
        // which is more expensive for large buffers.  For smaller
        // buffers, however, the cost of the copy is small compared to
        // the IO system overhead of the slow path.
        //

        if ( sendLength > AfdFastSendDatagramThreshold ) {
            return FALSE;
        }

        //
        // If the endpoint is not bound, fail.
        //

        if ( endpoint->State != AfdEndpointStateBound ) {
            return FALSE;
        }

        tdiRequest = &sendInfo->TdiRequest;

        try {
            destinationAddressLength =
                tdiRequest->SendDatagramInformation->RemoteAddressLength ;
        } except( EXCEPTION_EXECUTE_HANDLER ) {
            return FALSE;
        }

        //
        // Get an AFD buffer to use for the request.  We'll copy the
        // user's data to the AFD buffer then submit the IRP in the AFD
        // buffer to the TDI provider.
        //

        afdBuffer = AfdGetBuffer( sendLength, destinationAddressLength );
        if ( afdBuffer == NULL ) {
            return FALSE;
        }

        //
        // Store the length of the data and the address we're going to
        // send.
        //

        afdBuffer->DataLength = sendLength;
        afdBuffer->SourceAddressLength = destinationAddressLength;

        //
        // Copy the destination address and the output buffer to the
        // AFD buffer.
        //

        try {

            AfdCopyBufferArrayToBuffer(
                afdBuffer->Buffer,
                sendLength,
                sendInfo->BufferArray,
                sendInfo->BufferCount
                );

            RtlCopyMemory(
                afdBuffer->SourceAddress,
                tdiRequest->SendDatagramInformation->RemoteAddress,
                destinationAddressLength
                );

        } except( EXCEPTION_EXECUTE_HANDLER ) {

            RESET_CHAIN_LENGTH( afdBuffer );
            AfdReturnBuffer( afdBuffer );
            return FALSE;
        }

        //
        // Call a subroutine to complete the work of the fast path.
        //

        return AfdFastDatagramSend( afdBuffer, endpoint, IoStatus );

    }

    case IOCTL_AFD_RECEIVE: {

        AFD_RECV_INFO recvInfo;
        ULONG recvLength;

        //
        // If the input structure isn't large enough, bail on fast IO.
        //

        if( InputBufferLength < sizeof(recvInfo) ) {
            return FALSE;
        }

        //
        // Capture the input structure.
        //

        try {

            recvInfo = *(PAFD_RECV_INFO)InputBuffer;

            //
            // Make a quick preliminary check of the input buffer. If it's
            // bogus (or if Fast IO is disabled), then fail the request.
            //

            if( (recvInfo.AfdFlags & AFD_NO_FAST_IO) != 0 ||
                recvInfo.TdiFlags != TDI_RECEIVE_NORMAL ||
                recvInfo.BufferArray == NULL ||
                recvInfo.BufferCount == 0 ) {

                return FALSE;

            }

            //
            // Calculate the length of the receive buffer.
            //

            recvLength = AfdCalcBufferArrayByteLengthWrite(
                             recvInfo.BufferArray,
                             recvInfo.BufferCount
                             );

        } except( EXCEPTION_EXECUTE_HANDLER ) {

            return FALSE;

        }

        //
        // Attempt to perform fast IO on the endpoint.
        //

        return AfdFastDatagramReceive(
                   endpoint,
                   recvInfo.TdiFlags,
                   recvInfo.AfdFlags,
                   recvInfo.BufferArray,
                   recvInfo.BufferCount,
                   recvLength,
                   NULL,
                   NULL,
                   IoStatus
                   );

    }

    case IOCTL_AFD_RECEIVE_DATAGRAM: {

        AFD_RECV_DATAGRAM_INFO recvInfo;
        ULONG recvLength;

        //
        // If the input structure isn't large enough, bail on fast IO.
        //

        if( InputBufferLength < sizeof(recvInfo) ) {
            return FALSE;
        }

        //
        // Capture the input structure.
        //

        try {

            recvInfo = *(PAFD_RECV_DATAGRAM_INFO)InputBuffer;

            //
            // Make a quick preliminary check of the input buffer. If it's
            // bogus (or if Fast IO is disabled), then fail the request.
            //

            if( (recvInfo.AfdFlags & AFD_NO_FAST_IO) != 0 ||
                recvInfo.TdiFlags != TDI_RECEIVE_NORMAL ||
                recvInfo.BufferArray == NULL ||
                recvInfo.BufferCount == 0 ) {

                return FALSE;

            }

            //
            // Calculate the length of the receive buffer.
            //

            recvLength = AfdCalcBufferArrayByteLengthWrite(
                             recvInfo.BufferArray,
                             recvInfo.BufferCount
                             );

        } except( EXCEPTION_EXECUTE_HANDLER ) {

            return FALSE;

        }

        //
        // Attempt to perform fast IO on the endpoint.
        //

        return AfdFastDatagramReceive(
                   endpoint,
                   recvInfo.TdiFlags,
                   recvInfo.AfdFlags,
                   recvInfo.BufferArray,
                   recvInfo.BufferCount,
                   recvLength,
                   recvInfo.Address,
                   recvInfo.AddressLength,
                   IoStatus
                   );

    }

    default:

        return FALSE;
    }

} // AfdFastDatagramIo


BOOLEAN
AfdFastDatagramReceive (
    IN PAFD_ENDPOINT Endpoint,
    IN ULONG ReceiveFlags,
    IN ULONG AfdFlags,
    IN LPWSABUF BufferArray,
    IN ULONG BufferCount,
    IN ULONG ReceiveBufferLength,
    OUT PVOID SourceAddress,
    OUT PULONG SourceAddressLength,
    OUT PIO_STATUS_BLOCK IoStatus
    )
{
    KIRQL oldIrql;
    PLIST_ENTRY listEntry;
    PAFD_BUFFER afdBuffer;
    PTRANSPORT_ADDRESS tdiAddress;
    ULONG addressLength;

    //
    // If the receive flags has any unexpected bits set, fail fast IO.
    // We don't handle peeks or expedited data here.
    //

    if( ReceiveFlags != TDI_RECEIVE_NORMAL ) {
        return FALSE;
    }

    //
    // If the endpoint is neither bound nor connected, fail.
    //

    if ( Endpoint->State != AfdEndpointStateBound &&
             Endpoint->State != AfdEndpointStateConnected ) {
        return FALSE;
    }

    //
    // If there are no datagrams available to be received, don't
    // bother with the fast path.
    //

    AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql );

    if ( !ARE_DATAGRAMS_ON_ENDPOINT( Endpoint ) ) {

        //
        // If this is a nonblocking endpoint, fail the request here and
        // save going through the regular path.
        //

        if ( Endpoint->NonBlocking && !( AfdFlags & AFD_OVERLAPPED ) ) {
            Endpoint->EventsActive &= ~AFD_POLL_SEND;

            IF_DEBUG(EVENT_SELECT) {
                KdPrint((
                    "AfdFastDatagramReceive: Endp %08lX, Active %08lX\n",
                    Endpoint,
                    Endpoint->EventsActive
                    ));
            }

            IoStatus->Status = STATUS_DEVICE_NOT_READY;
            AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );
            return TRUE;
        }

        AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );
        return FALSE;
    }

    //
    // There is at least one datagram bufferred on the endpoint.  Use it
    // for this receive.
    //

    listEntry = RemoveHeadList( &Endpoint->ReceiveDatagramBufferListHead );
    afdBuffer = CONTAINING_RECORD( listEntry, AFD_BUFFER, BufferListEntry );

    //
    // If the datagram is too large, fail fast IO.
    //

    if ( afdBuffer->DataLength > ReceiveBufferLength ) {
        InsertHeadList(
            &Endpoint->ReceiveDatagramBufferListHead,
            &afdBuffer->BufferListEntry
            );
        AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );
        return FALSE;
    }

    //
    // Update counts of bufferred datagrams and bytes on the endpoint.
    //

    Endpoint->BufferredDatagramCount--;
    Endpoint->BufferredDatagramBytes -= afdBuffer->DataLength;

    //
    // Release the lock and copy the datagram into the user buffer.  We
    // can't continue to hold the lock, because it is not legal to take
    // an exception at raised IRQL.  Releasing the lock may result in a
    // misordered datagram if there is an exception in copying to the
    // user's buffer, but that is the user's fault for giving us a bogus
    // pointer.  Besides, datagram order is not guaranteed.
    //

    AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );

    try {

        AfdCopyBufferToBufferArray(
            BufferArray,
            0,
            BufferCount,
            afdBuffer->Buffer,
            afdBuffer->DataLength
            );

        //
        // If we need to return the source address, copy it to the
        // user's output buffer.
        //

        if ( SourceAddress != NULL ) {

            tdiAddress = afdBuffer->SourceAddress;

            addressLength = tdiAddress->Address[0].AddressLength +
                sizeof(u_short);    // sa_family

            if( *SourceAddressLength < addressLength ) {

                ExRaiseAccessViolation();

            }

            RtlCopyMemory(
                SourceAddress,
                &tdiAddress->Address[0].AddressType,
                addressLength
                );

            *SourceAddressLength = addressLength;

        }

        IoStatus->Information = afdBuffer->DataLength;
        IoStatus->Status = STATUS_SUCCESS;

    } except( EXCEPTION_EXECUTE_HANDLER ) {

        //
        // Put the buffer back on the endpoint's list.
        //

        AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql );

        InsertHeadList(
            &Endpoint->ReceiveDatagramBufferListHead,
            &afdBuffer->BufferListEntry
            );

        Endpoint->BufferredDatagramCount++;
        Endpoint->BufferredDatagramBytes += afdBuffer->DataLength;

        AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );

        return FALSE;
    }

    //
    // Clear the receive data active bit. If there's more data
    // available, set the corresponding event.
    //

    AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql );

    Endpoint->EventsActive &= ~AFD_POLL_RECEIVE;

    if( ARE_DATAGRAMS_ON_ENDPOINT( Endpoint ) ) {

        AfdIndicateEventSelectEvent(
            Endpoint,
            AFD_POLL_RECEIVE_BIT,
            STATUS_SUCCESS
            );

    }

    AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );

    //
    // The fast IO worked!  Clean up and return to the user.
    //

    RESET_CHAIN_LENGTH( afdBuffer );
    AfdReturnBuffer( afdBuffer );

    return TRUE;

} // AfdFastDatagramReceive


BOOLEAN
AfdFastDatagramSend (
    IN PAFD_BUFFER AfdBuffer,
    IN PAFD_ENDPOINT Endpoint,
    OUT PIO_STATUS_BLOCK IoStatus
    )
{
    NTSTATUS status;
    ULONG sendLength;

    PAGED_CODE( );

    //
    // Set up the input TDI information to point to the destination
    // address.
    //

    AfdBuffer->TdiInputInfo.RemoteAddressLength = AfdBuffer->SourceAddressLength;
    AfdBuffer->TdiInputInfo.RemoteAddress = AfdBuffer->SourceAddress;

    sendLength = AfdBuffer->DataLength;

    //
    // Initialize the IRP in the AFD buffer to do a fast datagram send.
    //

    TdiBuildSendDatagram(
        AfdBuffer->Irp,
        Endpoint->AddressDeviceObject,
        Endpoint->AddressFileObject,
        AfdRestartFastSendDatagram,
        AfdBuffer,
        AfdBuffer->Irp->MdlAddress,
        sendLength,
        &AfdBuffer->TdiInputInfo
        );

    //
    // Change the MDL in the AFD buffer to specify only the number
    // of bytes we're actually sending.  This is a requirement of TDI--
    // the MDL chain cannot describe a longer buffer than the send
    // request.
    //

    AfdBuffer->Mdl->ByteCount = sendLength;

    //
    // Reference the endpoint so that it does not go away until the send
    // completes.  This is necessary to ensure that a send which takes a
    // very long time and lasts longer than the process will not cause a
    // crash when the send datragram finally completes.
    //

    REFERENCE_ENDPOINT( Endpoint );
    AfdBuffer->Context = Endpoint;

    //
    // Give the IRP to the TDI provider.  If the request fails
    // immediately, then fail fast IO.  If the request fails later on,
    // there's nothing we can do about it.
    //

    status = IoCallDriver(
                 Endpoint->AddressDeviceObject,
                 AfdBuffer->Irp
                 );

    if ( NT_SUCCESS(status) ) {
        IoStatus->Information = sendLength;
        IoStatus->Status = STATUS_SUCCESS;
        return TRUE;
    } else {
        return FALSE;
    }

} // AfdFastDatagramSend


NTSTATUS
AfdRestartFastSendDatagram (
    IN PDEVICE_OBJECT DeviceObject,
    IN PIRP Irp,
    IN PVOID Context
    )
{
    PAFD_BUFFER afdBuffer;
    PAFD_ENDPOINT endpoint;

    afdBuffer = Context;

    //
    // Find the endpoint used for this request.
    //

    endpoint = afdBuffer->Context;
    ASSERT( endpoint->Type == AfdBlockTypeDatagram );

    //
    // Reset and free the AFD buffer structure.
    //

    ASSERT( afdBuffer->Irp == Irp );

    afdBuffer->TdiInputInfo.RemoteAddressLength = 0;
    afdBuffer->TdiInputInfo.RemoteAddress = NULL;

    afdBuffer->Mdl->ByteCount = afdBuffer->BufferLength;

    RESET_CHAIN_LENGTH( afdBuffer );
    AfdReturnBuffer( afdBuffer );

    //
    // Get rid of the reference we put on the endpoint when we started
    // this I/O.
    //

    DEREFERENCE_ENDPOINT( endpoint );

    //
    // Tell the IO system to stop processing this IRP.
    //

    return STATUS_MORE_PROCESSING_REQUIRED;

} // AfdRestartFastSendDatagram


PAFD_BUFFER
CopyAddressToBuffer (
    IN PAFD_ENDPOINT Endpoint,
    IN ULONG OutputBufferLength
    )
{
    KIRQL oldIrql;
    PAFD_BUFFER afdBuffer;

    ASSERT( Endpoint->Type == AfdBlockTypeDatagram );

    AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql );

    //
    // If the endpoint is not connected, fail.
    //

    if ( Endpoint->State != AfdEndpointStateConnected ) {
        AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );
        return NULL;
    }

    //
    // Get an AFD buffer to use for the request.  We'll copy the
    // user to the AFD buffer then submit the IRP in the AFD
    // buffer to the TDI provider.
    //

    afdBuffer = AfdGetBuffer(
                    OutputBufferLength,
                    Endpoint->Common.Datagram.RemoteAddressLength
                    );
    if ( afdBuffer == NULL ) {
        return NULL;
    }

    ASSERT( Endpoint->Common.Datagram.RemoteAddress != NULL );
    ASSERT( afdBuffer->AllocatedAddressLength >=
                Endpoint->Common.Datagram.RemoteAddressLength );

    //
    // Copy the address to the AFD buffer.
    //

    RtlCopyMemory(
        afdBuffer->SourceAddress,
        Endpoint->Common.Datagram.RemoteAddress,
        Endpoint->Common.Datagram.RemoteAddressLength
        );

    afdBuffer->SourceAddressLength = Endpoint->Common.Datagram.RemoteAddressLength;

    AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );

    return afdBuffer;

} // CopyAddressToBuffer


BOOLEAN
AfdShouldSendBlock (
    IN PAFD_ENDPOINT Endpoint,
    IN PAFD_CONNECTION Connection,
    IN ULONG SendLength
    )

/*++

Routine Description:

    Determines whether a nonblocking send can be performed on the
    connection, and if the send is possible, updates the connection's
    send tracking information.

Arguments:

    Endpoint - the AFD endpoint for the send.

    Connection - the AFD connection for the send.

    SendLength - the number of bytes that the caller wants to send.

Return Value:

    TRUE if the there is not too much data on the endpoint to perform
    the send; FALSE otherwise.

--*/

{
    KIRQL oldIrql;

    //
    // Determine whether we can do fast IO with this send.  In order
    // to perform fast IO, there must be no other sends pended on this
    // connection and there must be enough space left for bufferring
    // the requested amount of data.
    //

    AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql );

    if ( !IsListEmpty( &Connection->VcSendIrpListHead )

         ||

         Connection->VcBufferredSendBytes >= Connection->MaxBufferredSendBytes

         ||

         Connection->VcBufferredSendCount >= Connection->MaxBufferredSendCount

         ) {

        //
        // If this is a nonblocking endpoint, fail the request here and
        // save going through the regular path.
        //

        if ( Endpoint->NonBlocking ) {
            Endpoint->EventsActive &= ~AFD_POLL_SEND;

            IF_DEBUG(EVENT_SELECT) {
                KdPrint((
                    "AfdFastIoDeviceControl: Endp %08lX, Active %08lX\n",
                    Endpoint,
                    Endpoint->EventsActive
                    ));
            }
        }

        AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );
        return TRUE;
    }

    //
    // Update count of send bytes pending on the connection.
    //

    Connection->VcBufferredSendBytes += SendLength;
    Connection->VcBufferredSendCount += 1;

    AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql );

    //
    // Indicate to the caller that it is OK to proceed with the send.
    //

    return FALSE;

} // AfdShouldSendBlock