/*++ Copyright (c) 1989 Microsoft Corporation Module Name: fastio.c Abstract: This module contains routines for handling fast ("turbo") IO in AFD. Author: David Treadwell (davidtr) 12-Oct-1992 Revision History: --*/ #include "afdp.h" BOOLEAN AfdFastDatagramIo ( IN struct _FILE_OBJECT *FileObject, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus ); BOOLEAN AfdFastDatagramReceive ( IN PAFD_ENDPOINT Endpoint, IN ULONG ReceiveFlags, IN ULONG AfdFlags, IN LPWSABUF BufferArray, IN ULONG BufferCount, IN ULONG ReceiveBufferLength, OUT PVOID SourceAddress, OUT PULONG SourceAddressLength, OUT PIO_STATUS_BLOCK IoStatus ); BOOLEAN AfdFastDatagramSend ( IN PAFD_BUFFER AfdBuffer, IN PAFD_ENDPOINT Endpoint, OUT PIO_STATUS_BLOCK IoStatus ); NTSTATUS AfdRestartFastSendDatagram ( IN PDEVICE_OBJECT DeviceObject, IN PIRP Irp, IN PVOID Context ); PAFD_BUFFER CopyAddressToBuffer ( IN PAFD_ENDPOINT Endpoint, IN ULONG OutputBufferLength ); #ifdef ALLOC_PRAGMA #pragma alloc_text( PAGE, AfdFastDatagramIo ) #pragma alloc_text( PAGE, AfdFastDatagramSend ) #pragma alloc_text( PAGE, AfdFastIoRead ) #pragma alloc_text( PAGE, AfdFastIoWrite ) #pragma alloc_text( PAGEAFD, AfdFastIoDeviceControl ) #pragma alloc_text( PAGEAFD, AfdFastDatagramReceive ) #pragma alloc_text( PAGEAFD, AfdRestartFastSendDatagram ) #pragma alloc_text( PAGEAFD, CopyAddressToBuffer ) #pragma alloc_text( PAGEAFD, AfdShouldSendBlock ) #endif BOOLEAN AfdFastIoRead ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject ) { AFD_RECV_INFO recvInfo; WSABUF wsaBuf; PAGED_CODE( ); UNREFERENCED_PARAMETER( FileOffset ); UNREFERENCED_PARAMETER( LockKey ); // // Build the (one and only) WSABUF. // wsaBuf.buf = Buffer; wsaBuf.len = Length; // // Setup the AFD_RECV_INFO structure. // recvInfo.BufferArray = &wsaBuf; recvInfo.BufferCount = 1; recvInfo.AfdFlags = AFD_OVERLAPPED; recvInfo.TdiFlags = TDI_RECEIVE_NORMAL; // // Fake an ioctl. // return AfdFastIoDeviceControl( FileObject, Wait, &recvInfo, sizeof(recvInfo), NULL, 0, IOCTL_AFD_RECEIVE, IoStatus, DeviceObject ); } // AfdFastIoRead BOOLEAN AfdFastIoWrite ( IN struct _FILE_OBJECT *FileObject, IN PLARGE_INTEGER FileOffset, IN ULONG Length, IN BOOLEAN Wait, IN ULONG LockKey, IN PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject ) { AFD_SEND_INFO sendInfo; WSABUF wsaBuf; PAGED_CODE( ); UNREFERENCED_PARAMETER( FileOffset ); UNREFERENCED_PARAMETER( LockKey ); // // Build the (one and only) WSABUF. // wsaBuf.buf = Buffer; wsaBuf.len = Length; // // Setup the AFD_SEND_INFO structure. // sendInfo.BufferArray = &wsaBuf; sendInfo.BufferCount = 1; sendInfo.AfdFlags = AFD_OVERLAPPED; sendInfo.TdiFlags = 0; // // Fake an ioctl. // return AfdFastIoDeviceControl( FileObject, Wait, &sendInfo, sizeof(sendInfo), NULL, 0, IOCTL_AFD_SEND, IoStatus, DeviceObject ); } // AfdFastIoWrite #if AFD_PERF_DBG BOOLEAN AfdFastIoDeviceControlReal ( IN struct _FILE_OBJECT *FileObject, IN BOOLEAN Wait, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, OUT PVOID OutputBuffer OPTIONAL, IN ULONG OutputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus ); BOOLEAN AfdFastIoDeviceControl ( IN struct _FILE_OBJECT *FileObject, IN BOOLEAN Wait, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, OUT PVOID OutputBuffer OPTIONAL, IN ULONG OutputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject ) { BOOLEAN success; if ( AfdDisableFastIo ) { return FALSE; } ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL ); success = AfdFastIoDeviceControlReal ( FileObject, Wait, InputBuffer, InputBufferLength, OutputBuffer, OutputBufferLength, IoControlCode, IoStatus ); ASSERT( KeGetCurrentIrql( ) == LOW_LEVEL ); switch ( IoControlCode ) { case IOCTL_AFD_SEND: if ( success ) { AfdFastSendsSucceeded++; } else { AfdFastSendsFailed++; } break; case IOCTL_AFD_RECEIVE: if ( success ) { AfdFastReceivesSucceeded++; } else { AfdFastReceivesFailed++; } break; case IOCTL_AFD_SEND_DATAGRAM: if ( success ) { AfdFastSendDatagramsSucceeded++; } else { AfdFastSendDatagramsFailed++; } break; case IOCTL_AFD_RECEIVE_DATAGRAM: if ( success ) { AfdFastReceiveDatagramsSucceeded++; } else { AfdFastReceiveDatagramsFailed++; } break; case IOCTL_AFD_POLL: if ( success ) { AfdFastPollsSucceeded++; } else { AfdFastPollsFailed++; } break; } return success; } // AfdFastIoDeviceControl BOOLEAN AfdFastIoDeviceControlReal ( IN struct _FILE_OBJECT *FileObject, IN BOOLEAN Wait, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, OUT PVOID OutputBuffer OPTIONAL, IN ULONG OutputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus ) #else BOOLEAN AfdFastIoDeviceControl ( IN struct _FILE_OBJECT *FileObject, IN BOOLEAN Wait, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, OUT PVOID OutputBuffer OPTIONAL, IN ULONG OutputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus, IN struct _DEVICE_OBJECT *DeviceObject ) #endif { PAFD_ENDPOINT endpoint; PAFD_CONNECTION connection; KIRQL oldIrql; PAFD_BUFFER afdBuffer; NTSTATUS status; PMDL MdlChain; // // All we want to do is pass the request through to the TDI provider // if possible. First get the endpoint and connection pointers. // endpoint = FileObject->FsContext; ASSERT( IS_AFD_ENDPOINT_TYPE( endpoint ) ); // // If the endpoint is shut down in any way, bail out of fast IO. // if ( endpoint->DisconnectMode != 0 ) { return FALSE; } // // If an OutputBuffer parameter was specified, then this is a more // complicated IO request, so bail on fast IO. // if( OutputBuffer != NULL ) { return FALSE; } // // Handle datagram fast IO in a subroutine. This keeps this routine // cleaner and faster. // if ( IS_DGRAM_ENDPOINT(endpoint) ) { return AfdFastDatagramIo( FileObject, InputBuffer, InputBufferLength, IoControlCode, IoStatus ); } // // If the endpoint isn't connected yet, then we don't want to // attempt fast IO on it. // if ( endpoint->State != AfdEndpointStateConnected ) { return FALSE; } ASSERT( endpoint->Type == AfdBlockTypeVcConnecting ); ASSERT( endpoint->Common.VcConnecting.Connection != NULL ); // // If the TDI provider for this endpoint supports bufferring, // don't use fast IO. // if ( endpoint->TdiBufferring ) { return FALSE; } connection = endpoint->Common.VcConnecting.Connection; ASSERT( connection->Type == AfdBlockTypeConnection ); if( connection->CleanupBegun ) { return FALSE; } IF_DEBUG(FAST_IO) { KdPrint(( "AfdFastIoDeviceControl: attempting fast IO on endp %lx, " "conn %lx, code %lx\n", endpoint, connection, IoControlCode )); } // // Based on whether this is a send or receive, attempt to perform // fast IO. // switch ( IoControlCode ) { case IOCTL_AFD_SEND: { PAFD_SEND_INFO sendInfo; ULONG sendLength; ULONG afdFlags; // // If the connection has been aborted, then we don't want to try // fast IO on it. // if ( connection->AbortIndicated ) { return FALSE; } // // If the input structure isn't large enough, bail on fast IO. // if( InputBufferLength < sizeof(*sendInfo) ) { return FALSE; } try { // // Make a quick preliminary check of the input buffer. If it's // bogus (or if Fast IO is disabled), then fail the request. // sendInfo = (PAFD_SEND_INFO)InputBuffer; afdFlags = sendInfo->AfdFlags; if( (afdFlags & AFD_NO_FAST_IO) != 0 || sendInfo->TdiFlags != 0 || sendInfo->BufferArray == NULL || sendInfo->BufferCount == 0 ) { return FALSE; } // // Calculate the length of the send buffer. // sendLength = AfdCalcBufferArrayByteLengthRead( sendInfo->BufferArray, sendInfo->BufferCount ); } except( EXCEPTION_EXECUTE_HANDLER ) { return FALSE; } // // Determine whether we can do fast IO with this send. In order // to perform fast IO, there must be no other sends pended on this // connection and there must be enough space left for bufferring // the requested amount of data. // if ( AfdShouldSendBlock( endpoint, connection, sendLength ) ) { // // If this is a nonblocking endpoint, fail the request here and // save going through the regular path. // if ( endpoint->NonBlocking && !( afdFlags & AFD_OVERLAPPED ) ) { IoStatus->Status = STATUS_DEVICE_NOT_READY; return TRUE; } return FALSE; } // // Next get an AFD buffer structure that contains an IRP and a // buffer to hold the data. // afdBuffer = AfdGetBuffer( sendLength, 0 ); if ( afdBuffer == NULL ) { AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql ); connection->VcBufferredSendBytes -= sendLength; connection->VcBufferredSendCount -= 1; AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); return FALSE; } // // We have to rebuild the MDL in the AFD buffer structure to // represent exactly the number of bytes we're going to be // sending. // afdBuffer->Mdl->ByteCount = sendLength; SET_CHAIN_LENGTH( afdBuffer, sendLength ); // // Remember the endpoint in the AFD buffer structure. We need // this in order to access the endpoint in the restart routine. // afdBuffer->Context = endpoint; // // Copy the user's data into the AFD buffer. // if( sendLength > 0 ) { try { AfdCopyBufferArrayToBuffer( afdBuffer->Buffer, sendLength, sendInfo->BufferArray, sendInfo->BufferCount ); } except( EXCEPTION_EXECUTE_HANDLER ) { afdBuffer->Mdl->ByteCount = afdBuffer->BufferLength; RESET_CHAIN_LENGTH( afdBuffer ); AfdReturnBuffer( afdBuffer ); AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql ); connection->VcBufferredSendBytes -= sendLength; connection->VcBufferredSendCount -= 1; AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); return FALSE; } } // // Use the IRP in the AFD buffer structure to give to the TDI // provider. Build the TDI send request. // TdiBuildSend( afdBuffer->Irp, connection->DeviceObject, connection->FileObject, AfdRestartBufferSend, afdBuffer, afdBuffer->Mdl, 0, sendLength ); // // Add a reference to the connection object since the send // request will complete asynchronously. // REFERENCE_CONNECTION( connection ); // // Call the transport to actually perform the send. // status = IoCallDriver( connection->DeviceObject, afdBuffer->Irp ); // // Complete the user's IRP as appropriate. Note that we change the // status code from what was returned by the TDI provider into // STATUS_SUCCESS. This is because we don't want to complete // the IRP with STATUS_PENDING etc. // if ( NT_SUCCESS(status) ) { IoStatus->Information = sendLength; IoStatus->Status = STATUS_SUCCESS; return TRUE; } // // The call failed for some reason. Fail fast IO. // return FALSE; } case IOCTL_AFD_RECEIVE: { PLIST_ENTRY listEntry; PAFD_RECV_INFO recvInfo; ULONG recvLength; ULONG totalOffset; // // If the input structure isn't large enough, bail on fast IO. // if( InputBufferLength < sizeof(*recvInfo) ) { return FALSE; } try { // // Make a quick preliminary check of the input buffer. If it's // bogus (or if Fast IO is disabled), then fail the request. // recvInfo = (PAFD_RECV_INFO)InputBuffer; if( (recvInfo->AfdFlags & AFD_NO_FAST_IO) != 0 || recvInfo->TdiFlags != TDI_RECEIVE_NORMAL || recvInfo->BufferArray == NULL || recvInfo->BufferCount == 0 ) { return FALSE; } // // Calculate the length of the receive buffer. // recvLength = AfdCalcBufferArrayByteLengthWrite( recvInfo->BufferArray, recvInfo->BufferCount ); } except( EXCEPTION_EXECUTE_HANDLER ) { return FALSE; } // // Determine whether we'll be able to perform fast IO. In order // to do fast IO, there must be some bufferred data on the // connection, there must not be any pended receives on the // connection, and there must not be any bufferred expedited // data on the connection. This last requirement is for // the sake of simplicity only. // AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql ); if ( connection->VcBufferredReceiveCount == 0 || !IsListEmpty( &connection->VcReceiveIrpListHead ) || connection->VcBufferredExpeditedCount != 0 ) { AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); return FALSE; } ASSERT( !IsListEmpty( &connection->VcReceiveBufferListHead ) ); // // Get a pointer to the first bufferred AFD buffer structure on // the connection. // afdBuffer = CONTAINING_RECORD( connection->VcReceiveBufferListHead.Flink, AFD_BUFFER, BufferListEntry ); ASSERT( !afdBuffer->ExpeditedData ); // // If the buffer contains a partial message, bail out of the // fast path. We don't want the added complexity of handling // partial messages in the fast path. // if ( afdBuffer->PartialMessage && endpoint->EndpointType != AfdEndpointTypeStream ) { AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); return FALSE; } // // For simplicity, act differently based on whether the user's // buffer is large enough to hold the entire first AFD buffer // worth of data. // if ( recvLength >= afdBuffer->DataLength ) { LIST_ENTRY bufferListHead; IoStatus->Status = STATUS_SUCCESS; IoStatus->Information = 0; InitializeListHead( &bufferListHead ); // // Loop getting AFD buffers that will fill in the user's // buffer with as much data as will fit, or else with a // single buffer if this is not a stream endpoint. We don't // actually do the copy within this loop because this loop // must occur while holding a lock, and we cannot hold a // lock while copying the data into the user's buffer // because the user's buffer is not locked and we cannot // take a page fault at raised IRQL. // do { // // Update the count of bytes on the connection. // ASSERT( connection->VcBufferredReceiveBytes >= afdBuffer->DataLength ); ASSERT( connection->VcBufferredReceiveCount > 0 ); connection->VcBufferredReceiveBytes -= afdBuffer->DataLength; connection->VcBufferredReceiveCount -= 1; IoStatus->Information += afdBuffer->DataLength; // // Remove the AFD buffer from the connection's list of // buffers and place it on our local list of buffers. // RemoveEntryList( &afdBuffer->BufferListEntry ); InsertTailList( &bufferListHead, &afdBuffer->BufferListEntry ); // // If this is a stream endpoint and all of the data in // the next AFD buffer will fit in the user's buffer, // use this buffer for the IO as well. // if ( !IsListEmpty( &connection->VcReceiveBufferListHead ) ) { afdBuffer = CONTAINING_RECORD( connection->VcReceiveBufferListHead.Flink, AFD_BUFFER, BufferListEntry ); ASSERT( !afdBuffer->ExpeditedData ); ASSERT( afdBuffer->DataOffset == 0 ); if ( endpoint->EndpointType == AfdEndpointTypeStream && IoStatus->Information + afdBuffer->DataLength <= recvLength ) { continue; } else { break; } } else { break; } } while ( TRUE ); // // If there is indicated but unreceived data in the TDI provider, // and we have available buffer space, fire off an IRP to receive // the data. // if ( connection->VcReceiveCountInTransport > 0 && connection->VcBufferredReceiveBytes < connection->MaxBufferredReceiveBytes && connection->VcBufferredReceiveCount < connection->MaxBufferredReceiveCount ) { CLONG bytesToReceive; // // Remember the count of data that we're going to receive, // then reset the fields in the connection where we keep // track of how much data is available in the transport. // We reset it here before releasing the lock so that // another thread doesn't try to receive the data at the // same time as us. // if ( connection->VcReceiveBytesInTransport > AfdLargeBufferSize ) { bytesToReceive = connection->VcReceiveBytesInTransport; } else { bytesToReceive = AfdLargeBufferSize; } ASSERT( connection->VcReceiveCountInTransport == 1 ); connection->VcReceiveBytesInTransport = 0; connection->VcReceiveCountInTransport = 0; AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); // // Get an AFD buffer structure to hold the data. // afdBuffer = AfdGetBuffer( bytesToReceive, 0 ); if ( afdBuffer == NULL ) { // // If we were unable to get a buffer, abort the // circuit. // AfdBeginAbort( connection ); } else { // // We need to remember the connection in the AFD buffer // because we'll need to access it in the completion // routine. // afdBuffer->Context = connection; // // Finish building the receive IRP to give to the TDI provider. // TdiBuildReceive( afdBuffer->Irp, connection->DeviceObject, connection->FileObject, AfdRestartBufferReceive, afdBuffer, afdBuffer->Mdl, TDI_RECEIVE_NORMAL, bytesToReceive ); // // Hand off the IRP to the TDI provider. // (VOID)IoCallDriver( connection->DeviceObject, afdBuffer->Irp ); } } else { AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); } // // We have in a local list all the data we'll use for this // IO. Start copying data to the user buffer. // totalOffset = 0; try { while ( !IsListEmpty( &bufferListHead ) ) { // // Take the first buffer from the list. // listEntry = RemoveHeadList( &bufferListHead ); afdBuffer = CONTAINING_RECORD( listEntry, AFD_BUFFER, BufferListEntry ); if( afdBuffer->DataLength > 0 ) { // // Copy the data in the buffer to the user buffer. // AfdCopyBufferToBufferArray( recvInfo->BufferArray, totalOffset, recvInfo->BufferCount, (PCHAR)afdBuffer->Buffer + afdBuffer->DataOffset, afdBuffer->DataLength ); totalOffset += afdBuffer->DataLength; } // // We're done with the AFD buffer. // afdBuffer->DataOffset = 0; RESET_CHAIN_LENGTH( afdBuffer ); AfdReturnBuffer( afdBuffer ); } } except( EXCEPTION_EXECUTE_HANDLER ) { // // If an exception is hit, there is the possibility of // data corruption. However, it is nearly impossible to // avoid this in all cases, so just throw out the // remainder of the data that we would have copied to // the user buffer. // afdBuffer->DataOffset = 0; RESET_CHAIN_LENGTH( afdBuffer ); AfdReturnBuffer( afdBuffer ); while ( !IsListEmpty( &bufferListHead ) ) { listEntry = RemoveHeadList( &bufferListHead ); afdBuffer = CONTAINING_RECORD( listEntry, AFD_BUFFER, BufferListEntry ); RESET_CHAIN_LENGTH( afdBuffer ); AfdReturnBuffer( afdBuffer ); } return FALSE; } // // Clear the receive data active bit. If there's more data // available, set the corresponding event. // AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql ); endpoint->EventsActive &= ~AFD_POLL_RECEIVE; if( !IsListEmpty( &connection->VcReceiveBufferListHead ) ) { AfdIndicateEventSelectEvent( endpoint, AFD_POLL_RECEIVE_BIT, STATUS_SUCCESS ); } AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); // // Fast IO succeeded! // ASSERT( IoStatus->Information <= recvLength ); return TRUE; } else { PAFD_BUFFER tempAfdBuffer; // // If this is not a stream endpoint and the user's buffer // is insufficient to hold the entire message, then we cannot // use fast IO because this IO will need to fail with // STATUS_BUFFER_OVERFLOW. // if ( endpoint->EndpointType != AfdEndpointTypeStream || recvLength == 0 ) { AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); return FALSE; } // // This is a stream endpoint and the user buffer is // insufficient for the amount of data stored in the first // buffer. So that we can perform fast IO and still // preserve data ordering, we're going to allocate a new AFD // buffer structure, copy the appropriate amount of data // into that buffer, then release locks and copy the data // into the user buffer. // // The extra copy incurred is still less than the IRP // overhead incurred in the normal IO path, so using fast IO // here is a win. Also, applications which force us through // this path will typically be using very small receives, // like one or four bytes, so the copy will be short. // // First allocate an AFD buffer to hold the data we're // eventually going to give to the user. // tempAfdBuffer = AfdGetBuffer( recvLength, 0 ); if ( tempAfdBuffer == NULL ) { AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); return FALSE; } // // Copy the first part of the bufferred data into our local // AFD buffer. // RtlCopyMemory( tempAfdBuffer->Buffer, (PCHAR)afdBuffer->Buffer + afdBuffer->DataOffset, recvLength ); // // Update the data length and offset in the data bufferred // on the connection. // afdBuffer->DataLength -= recvLength; afdBuffer->DataOffset += recvLength; connection->VcBufferredReceiveBytes -= recvLength; // // If there is indicated but unreceived data in the TDI provider, // and we have available buffer space, fire off an IRP to receive // the data. // if ( connection->VcReceiveBytesInTransport > 0 && connection->VcBufferredReceiveBytes < connection->MaxBufferredReceiveBytes && connection->VcBufferredReceiveCount < connection->MaxBufferredReceiveCount ) { CLONG bytesInTransport; // // Remember the count of data that we're going to receive, // then reset the fields in the connection where we keep // track of how much data is available in the transport. // We reset it here before releasing the lock so that // another thread doesn't try to receive the data at the // same time as us. // bytesInTransport = connection->VcReceiveBytesInTransport; ASSERT( connection->VcReceiveCountInTransport == 1 ); connection->VcReceiveBytesInTransport = 0; connection->VcReceiveCountInTransport = 0; AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); // // Get an AFD buffer structure to hold the data. // afdBuffer = AfdGetBuffer( bytesInTransport, 0 ); if ( afdBuffer == NULL ) { // // If we were unable to get a buffer, abort the // circuit. // AfdBeginAbort( connection ); } else { // // We need to remember the connection in the AFD buffer // because we'll need to access it in the completion // routine. // afdBuffer->Context = connection; // // Finish building the receive IRP to give to the TDI provider. // TdiBuildReceive( afdBuffer->Irp, connection->DeviceObject, connection->FileObject, AfdRestartBufferReceive, afdBuffer, afdBuffer->Mdl, TDI_RECEIVE_NORMAL, bytesInTransport ); // // Hand off the IRP to the TDI provider. // (VOID)IoCallDriver( connection->DeviceObject, afdBuffer->Irp ); } } else { AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); } // // Now copy the data to the user buffer inside an exception // handler. // try { AfdCopyBufferToBufferArray( recvInfo->BufferArray, 0, recvInfo->BufferCount, tempAfdBuffer->Buffer, recvLength ); } except( EXCEPTION_EXECUTE_HANDLER ) { RESET_CHAIN_LENGTH( tempAfdBuffer ); AfdReturnBuffer( tempAfdBuffer ); return FALSE; } // // Clear the receive data active bit. If there's more data // available, set the corresponding event. // AfdAcquireSpinLock( &endpoint->SpinLock, &oldIrql ); endpoint->EventsActive &= ~AFD_POLL_RECEIVE; if( !IsListEmpty( &connection->VcReceiveBufferListHead ) ) { AfdIndicateEventSelectEvent( endpoint, AFD_POLL_RECEIVE_BIT, STATUS_SUCCESS ); } AfdReleaseSpinLock( &endpoint->SpinLock, oldIrql ); // // Fast IO succeeded! // RESET_CHAIN_LENGTH( tempAfdBuffer ); AfdReturnBuffer( tempAfdBuffer ); IoStatus->Status = STATUS_SUCCESS; IoStatus->Information = recvLength; return TRUE; } } case IOCTL_AFD_TRANSMIT_FILE: return AfdFastTransmitFile( FileObject, InputBuffer, InputBufferLength, IoStatus ); default: return FALSE; } return FALSE; } // AfdFastDeviceIoControlFile BOOLEAN AfdFastDatagramIo ( IN struct _FILE_OBJECT *FileObject, IN PVOID InputBuffer OPTIONAL, IN ULONG InputBufferLength, IN ULONG IoControlCode, OUT PIO_STATUS_BLOCK IoStatus ) { PAFD_ENDPOINT endpoint; PAFD_BUFFER afdBuffer; PAGED_CODE( ); // // All we want to do is pass the request through to the TDI provider // if possible. First get the endpoint pointer. // endpoint = FileObject->FsContext; ASSERT( endpoint->Type == AfdBlockTypeDatagram ); IF_DEBUG(FAST_IO) { KdPrint(( "AfdFastDatagramIo: attempting fast IO on endp %lx, " "code %lx\n", endpoint, IoControlCode )); } switch ( IoControlCode ) { case IOCTL_AFD_SEND: { PAFD_SEND_INFO sendInfo; ULONG sendLength; // // If the input structure isn't large enough, bail on fast IO. // if( InputBufferLength < sizeof(*sendInfo) ) { return FALSE; } try { // // Make a quick preliminary check of the input buffer. If it's // bogus (or if Fast IO is disabled), then fail the request. // sendInfo = (PAFD_SEND_INFO)InputBuffer; if( (sendInfo->AfdFlags & AFD_NO_FAST_IO) != 0 || sendInfo->TdiFlags != 0 || sendInfo->BufferArray == NULL || sendInfo->BufferCount == 0 ) { return FALSE; } // // Calculate the length of the send buffer. // sendLength = AfdCalcBufferArrayByteLengthRead( sendInfo->BufferArray, sendInfo->BufferCount ); } except( EXCEPTION_EXECUTE_HANDLER ) { return FALSE; } // // If this is a send for more than the threshold number of // bytes, don't use the fast path. We don't allow larger sends // in the fast path because of the extra data copy it entails, // which is more expensive for large buffers. For smaller // buffers, however, the cost of the copy is small compared to // the IO system overhead of the slow path. // if ( sendLength > AfdFastSendDatagramThreshold ) { return FALSE; } // // In a subroutine, copy the destination address to the AFD // buffer. We do this in a subroutine because it needs to // acquire a spin lock and we want this routine to be pageable. // afdBuffer = CopyAddressToBuffer( endpoint, sendLength ); if ( afdBuffer == NULL ) { return FALSE; } // // Store the length of the data we're going to send. // afdBuffer->DataLength = sendLength; // // Copy the output buffer to the AFD buffer. // try { AfdCopyBufferArrayToBuffer( afdBuffer->Buffer, sendLength, sendInfo->BufferArray, sendInfo->BufferCount ); } except( EXCEPTION_EXECUTE_HANDLER ) { RESET_CHAIN_LENGTH( afdBuffer ); AfdReturnBuffer( afdBuffer ); return FALSE; } // // Call a subroutine to complete the work of the fast path. // return AfdFastDatagramSend( afdBuffer, endpoint, IoStatus ); } case IOCTL_AFD_SEND_DATAGRAM: { PTDI_REQUEST_SEND_DATAGRAM tdiRequest; ULONG destinationAddressLength; PAFD_SEND_DATAGRAM_INFO sendInfo; ULONG sendLength; // // If the input structure isn't large enough, bail on fast IO. // if( InputBufferLength < sizeof(*sendInfo) ) { return FALSE; } try { // // Make a quick preliminary check of the input buffer. If it's // bogus (or if Fast IO is disabled), then fail the request. // sendInfo = (PAFD_SEND_DATAGRAM_INFO)InputBuffer; if( (sendInfo->AfdFlags & AFD_NO_FAST_IO) != 0 || sendInfo->BufferArray == NULL || sendInfo->BufferCount == 0 ) { return FALSE; } // // Calculate the length of the send buffer. // sendLength = AfdCalcBufferArrayByteLengthRead( sendInfo->BufferArray, sendInfo->BufferCount ); } except( EXCEPTION_EXECUTE_HANDLER ) { return FALSE; } // // If this is a send for more than the threshold number of // bytes, don't use the fast path. We don't allow larger sends // in the fast path because of the extra data copy it entails, // which is more expensive for large buffers. For smaller // buffers, however, the cost of the copy is small compared to // the IO system overhead of the slow path. // if ( sendLength > AfdFastSendDatagramThreshold ) { return FALSE; } // // If the endpoint is not bound, fail. // if ( endpoint->State != AfdEndpointStateBound ) { return FALSE; } tdiRequest = &sendInfo->TdiRequest; try { destinationAddressLength = tdiRequest->SendDatagramInformation->RemoteAddressLength ; } except( EXCEPTION_EXECUTE_HANDLER ) { return FALSE; } // // Get an AFD buffer to use for the request. We'll copy the // user's data to the AFD buffer then submit the IRP in the AFD // buffer to the TDI provider. // afdBuffer = AfdGetBuffer( sendLength, destinationAddressLength ); if ( afdBuffer == NULL ) { return FALSE; } // // Store the length of the data and the address we're going to // send. // afdBuffer->DataLength = sendLength; afdBuffer->SourceAddressLength = destinationAddressLength; // // Copy the destination address and the output buffer to the // AFD buffer. // try { AfdCopyBufferArrayToBuffer( afdBuffer->Buffer, sendLength, sendInfo->BufferArray, sendInfo->BufferCount ); RtlCopyMemory( afdBuffer->SourceAddress, tdiRequest->SendDatagramInformation->RemoteAddress, destinationAddressLength ); } except( EXCEPTION_EXECUTE_HANDLER ) { RESET_CHAIN_LENGTH( afdBuffer ); AfdReturnBuffer( afdBuffer ); return FALSE; } // // Call a subroutine to complete the work of the fast path. // return AfdFastDatagramSend( afdBuffer, endpoint, IoStatus ); } case IOCTL_AFD_RECEIVE: { AFD_RECV_INFO recvInfo; ULONG recvLength; // // If the input structure isn't large enough, bail on fast IO. // if( InputBufferLength < sizeof(recvInfo) ) { return FALSE; } // // Capture the input structure. // try { recvInfo = *(PAFD_RECV_INFO)InputBuffer; // // Make a quick preliminary check of the input buffer. If it's // bogus (or if Fast IO is disabled), then fail the request. // if( (recvInfo.AfdFlags & AFD_NO_FAST_IO) != 0 || recvInfo.TdiFlags != TDI_RECEIVE_NORMAL || recvInfo.BufferArray == NULL || recvInfo.BufferCount == 0 ) { return FALSE; } // // Calculate the length of the receive buffer. // recvLength = AfdCalcBufferArrayByteLengthWrite( recvInfo.BufferArray, recvInfo.BufferCount ); } except( EXCEPTION_EXECUTE_HANDLER ) { return FALSE; } // // Attempt to perform fast IO on the endpoint. // return AfdFastDatagramReceive( endpoint, recvInfo.TdiFlags, recvInfo.AfdFlags, recvInfo.BufferArray, recvInfo.BufferCount, recvLength, NULL, NULL, IoStatus ); } case IOCTL_AFD_RECEIVE_DATAGRAM: { AFD_RECV_DATAGRAM_INFO recvInfo; ULONG recvLength; // // If the input structure isn't large enough, bail on fast IO. // if( InputBufferLength < sizeof(recvInfo) ) { return FALSE; } // // Capture the input structure. // try { recvInfo = *(PAFD_RECV_DATAGRAM_INFO)InputBuffer; // // Make a quick preliminary check of the input buffer. If it's // bogus (or if Fast IO is disabled), then fail the request. // if( (recvInfo.AfdFlags & AFD_NO_FAST_IO) != 0 || recvInfo.TdiFlags != TDI_RECEIVE_NORMAL || recvInfo.BufferArray == NULL || recvInfo.BufferCount == 0 ) { return FALSE; } // // Calculate the length of the receive buffer. // recvLength = AfdCalcBufferArrayByteLengthWrite( recvInfo.BufferArray, recvInfo.BufferCount ); } except( EXCEPTION_EXECUTE_HANDLER ) { return FALSE; } // // Attempt to perform fast IO on the endpoint. // return AfdFastDatagramReceive( endpoint, recvInfo.TdiFlags, recvInfo.AfdFlags, recvInfo.BufferArray, recvInfo.BufferCount, recvLength, recvInfo.Address, recvInfo.AddressLength, IoStatus ); } default: return FALSE; } } // AfdFastDatagramIo BOOLEAN AfdFastDatagramReceive ( IN PAFD_ENDPOINT Endpoint, IN ULONG ReceiveFlags, IN ULONG AfdFlags, IN LPWSABUF BufferArray, IN ULONG BufferCount, IN ULONG ReceiveBufferLength, OUT PVOID SourceAddress, OUT PULONG SourceAddressLength, OUT PIO_STATUS_BLOCK IoStatus ) { KIRQL oldIrql; PLIST_ENTRY listEntry; PAFD_BUFFER afdBuffer; PTRANSPORT_ADDRESS tdiAddress; ULONG addressLength; // // If the receive flags has any unexpected bits set, fail fast IO. // We don't handle peeks or expedited data here. // if( ReceiveFlags != TDI_RECEIVE_NORMAL ) { return FALSE; } // // If the endpoint is neither bound nor connected, fail. // if ( Endpoint->State != AfdEndpointStateBound && Endpoint->State != AfdEndpointStateConnected ) { return FALSE; } // // If there are no datagrams available to be received, don't // bother with the fast path. // AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql ); if ( !ARE_DATAGRAMS_ON_ENDPOINT( Endpoint ) ) { // // If this is a nonblocking endpoint, fail the request here and // save going through the regular path. // if ( Endpoint->NonBlocking && !( AfdFlags & AFD_OVERLAPPED ) ) { Endpoint->EventsActive &= ~AFD_POLL_SEND; IF_DEBUG(EVENT_SELECT) { KdPrint(( "AfdFastDatagramReceive: Endp %08lX, Active %08lX\n", Endpoint, Endpoint->EventsActive )); } IoStatus->Status = STATUS_DEVICE_NOT_READY; AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); return TRUE; } AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); return FALSE; } // // There is at least one datagram bufferred on the endpoint. Use it // for this receive. // listEntry = RemoveHeadList( &Endpoint->ReceiveDatagramBufferListHead ); afdBuffer = CONTAINING_RECORD( listEntry, AFD_BUFFER, BufferListEntry ); // // If the datagram is too large, fail fast IO. // if ( afdBuffer->DataLength > ReceiveBufferLength ) { InsertHeadList( &Endpoint->ReceiveDatagramBufferListHead, &afdBuffer->BufferListEntry ); AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); return FALSE; } // // Update counts of bufferred datagrams and bytes on the endpoint. // Endpoint->BufferredDatagramCount--; Endpoint->BufferredDatagramBytes -= afdBuffer->DataLength; // // Release the lock and copy the datagram into the user buffer. We // can't continue to hold the lock, because it is not legal to take // an exception at raised IRQL. Releasing the lock may result in a // misordered datagram if there is an exception in copying to the // user's buffer, but that is the user's fault for giving us a bogus // pointer. Besides, datagram order is not guaranteed. // AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); try { AfdCopyBufferToBufferArray( BufferArray, 0, BufferCount, afdBuffer->Buffer, afdBuffer->DataLength ); // // If we need to return the source address, copy it to the // user's output buffer. // if ( SourceAddress != NULL ) { tdiAddress = afdBuffer->SourceAddress; addressLength = tdiAddress->Address[0].AddressLength + sizeof(u_short); // sa_family if( *SourceAddressLength < addressLength ) { ExRaiseAccessViolation(); } RtlCopyMemory( SourceAddress, &tdiAddress->Address[0].AddressType, addressLength ); *SourceAddressLength = addressLength; } IoStatus->Information = afdBuffer->DataLength; IoStatus->Status = STATUS_SUCCESS; } except( EXCEPTION_EXECUTE_HANDLER ) { // // Put the buffer back on the endpoint's list. // AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql ); InsertHeadList( &Endpoint->ReceiveDatagramBufferListHead, &afdBuffer->BufferListEntry ); Endpoint->BufferredDatagramCount++; Endpoint->BufferredDatagramBytes += afdBuffer->DataLength; AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); return FALSE; } // // Clear the receive data active bit. If there's more data // available, set the corresponding event. // AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql ); Endpoint->EventsActive &= ~AFD_POLL_RECEIVE; if( ARE_DATAGRAMS_ON_ENDPOINT( Endpoint ) ) { AfdIndicateEventSelectEvent( Endpoint, AFD_POLL_RECEIVE_BIT, STATUS_SUCCESS ); } AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); // // The fast IO worked! Clean up and return to the user. // RESET_CHAIN_LENGTH( afdBuffer ); AfdReturnBuffer( afdBuffer ); return TRUE; } // AfdFastDatagramReceive BOOLEAN AfdFastDatagramSend ( IN PAFD_BUFFER AfdBuffer, IN PAFD_ENDPOINT Endpoint, OUT PIO_STATUS_BLOCK IoStatus ) { NTSTATUS status; ULONG sendLength; PAGED_CODE( ); // // Set up the input TDI information to point to the destination // address. // AfdBuffer->TdiInputInfo.RemoteAddressLength = AfdBuffer->SourceAddressLength; AfdBuffer->TdiInputInfo.RemoteAddress = AfdBuffer->SourceAddress; sendLength = AfdBuffer->DataLength; // // Initialize the IRP in the AFD buffer to do a fast datagram send. // TdiBuildSendDatagram( AfdBuffer->Irp, Endpoint->AddressDeviceObject, Endpoint->AddressFileObject, AfdRestartFastSendDatagram, AfdBuffer, AfdBuffer->Irp->MdlAddress, sendLength, &AfdBuffer->TdiInputInfo ); // // Change the MDL in the AFD buffer to specify only the number // of bytes we're actually sending. This is a requirement of TDI-- // the MDL chain cannot describe a longer buffer than the send // request. // AfdBuffer->Mdl->ByteCount = sendLength; // // Reference the endpoint so that it does not go away until the send // completes. This is necessary to ensure that a send which takes a // very long time and lasts longer than the process will not cause a // crash when the send datragram finally completes. // REFERENCE_ENDPOINT( Endpoint ); AfdBuffer->Context = Endpoint; // // Give the IRP to the TDI provider. If the request fails // immediately, then fail fast IO. If the request fails later on, // there's nothing we can do about it. // status = IoCallDriver( Endpoint->AddressDeviceObject, AfdBuffer->Irp ); if ( NT_SUCCESS(status) ) { IoStatus->Information = sendLength; IoStatus->Status = STATUS_SUCCESS; return TRUE; } else { return FALSE; } } // AfdFastDatagramSend NTSTATUS AfdRestartFastSendDatagram ( IN PDEVICE_OBJECT DeviceObject, IN PIRP Irp, IN PVOID Context ) { PAFD_BUFFER afdBuffer; PAFD_ENDPOINT endpoint; afdBuffer = Context; // // Find the endpoint used for this request. // endpoint = afdBuffer->Context; ASSERT( endpoint->Type == AfdBlockTypeDatagram ); // // Reset and free the AFD buffer structure. // ASSERT( afdBuffer->Irp == Irp ); afdBuffer->TdiInputInfo.RemoteAddressLength = 0; afdBuffer->TdiInputInfo.RemoteAddress = NULL; afdBuffer->Mdl->ByteCount = afdBuffer->BufferLength; RESET_CHAIN_LENGTH( afdBuffer ); AfdReturnBuffer( afdBuffer ); // // Get rid of the reference we put on the endpoint when we started // this I/O. // DEREFERENCE_ENDPOINT( endpoint ); // // Tell the IO system to stop processing this IRP. // return STATUS_MORE_PROCESSING_REQUIRED; } // AfdRestartFastSendDatagram PAFD_BUFFER CopyAddressToBuffer ( IN PAFD_ENDPOINT Endpoint, IN ULONG OutputBufferLength ) { KIRQL oldIrql; PAFD_BUFFER afdBuffer; ASSERT( Endpoint->Type == AfdBlockTypeDatagram ); AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql ); // // If the endpoint is not connected, fail. // if ( Endpoint->State != AfdEndpointStateConnected ) { AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); return NULL; } // // Get an AFD buffer to use for the request. We'll copy the // user to the AFD buffer then submit the IRP in the AFD // buffer to the TDI provider. // afdBuffer = AfdGetBuffer( OutputBufferLength, Endpoint->Common.Datagram.RemoteAddressLength ); if ( afdBuffer == NULL ) { return NULL; } ASSERT( Endpoint->Common.Datagram.RemoteAddress != NULL ); ASSERT( afdBuffer->AllocatedAddressLength >= Endpoint->Common.Datagram.RemoteAddressLength ); // // Copy the address to the AFD buffer. // RtlCopyMemory( afdBuffer->SourceAddress, Endpoint->Common.Datagram.RemoteAddress, Endpoint->Common.Datagram.RemoteAddressLength ); afdBuffer->SourceAddressLength = Endpoint->Common.Datagram.RemoteAddressLength; AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); return afdBuffer; } // CopyAddressToBuffer BOOLEAN AfdShouldSendBlock ( IN PAFD_ENDPOINT Endpoint, IN PAFD_CONNECTION Connection, IN ULONG SendLength ) /*++ Routine Description: Determines whether a nonblocking send can be performed on the connection, and if the send is possible, updates the connection's send tracking information. Arguments: Endpoint - the AFD endpoint for the send. Connection - the AFD connection for the send. SendLength - the number of bytes that the caller wants to send. Return Value: TRUE if the there is not too much data on the endpoint to perform the send; FALSE otherwise. --*/ { KIRQL oldIrql; // // Determine whether we can do fast IO with this send. In order // to perform fast IO, there must be no other sends pended on this // connection and there must be enough space left for bufferring // the requested amount of data. // AfdAcquireSpinLock( &Endpoint->SpinLock, &oldIrql ); if ( !IsListEmpty( &Connection->VcSendIrpListHead ) || Connection->VcBufferredSendBytes >= Connection->MaxBufferredSendBytes || Connection->VcBufferredSendCount >= Connection->MaxBufferredSendCount ) { // // If this is a nonblocking endpoint, fail the request here and // save going through the regular path. // if ( Endpoint->NonBlocking ) { Endpoint->EventsActive &= ~AFD_POLL_SEND; IF_DEBUG(EVENT_SELECT) { KdPrint(( "AfdFastIoDeviceControl: Endp %08lX, Active %08lX\n", Endpoint, Endpoint->EventsActive )); } } AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); return TRUE; } // // Update count of send bytes pending on the connection. // Connection->VcBufferredSendBytes += SendLength; Connection->VcBufferredSendCount += 1; AfdReleaseSpinLock( &Endpoint->SpinLock, oldIrql ); // // Indicate to the caller that it is OK to proceed with the send. // return FALSE; } // AfdShouldSendBlock