Mirror-bot/yttrium-virtio-gpu
1
0 Fork 0
mirror of https://github.com/arehnman/yttrium-virtio-gpu.git synced 2026-11-07 02:06:32 +00:00
Code Issues Projects Releases Activity
Files
e941c70b7513d7cbbf54aed3eb00a6367f213913
yttrium-virtio-gpu/NetKVM/wlh/ParaNdis6-Impl.cpp
T
Yuri Benditovich 3529c23ce6 netkvm(arm64): copy RX data to cached RAM before indication 
On ARM there is problem of unaligned access to the network data
buffer, which is received by allocation of shared memory.
Until the root cause solved we can copy RX data from original
shared buffer to cached system RAM to prevent the exception
in case of unalignhed access.

Signed-off-by: Yuri Benditovich <[email protected]>
2019-12-23 09:57:33 +02:00

1110 lines
39 KiB
C++
Executable File
Raw Blame History

/*
* This file contains NDIS6-specific implementation of driver's procedures.
*
* Copyright (c) 2008-2017 Red Hat, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met :
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and / or other materials provided with the distribution.
* 3. Neither the names of the copyright holders nor the names of their contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include "ParaNdis6.h"
#include "kdebugprint.h"
#include "ParaNdis_DebugHistory.h"
#include "Trace.h"
#ifdef NETKVM_WPP_ENABLED
#include "ParaNdis6-Impl.tmh"
#endif
static MINIPORT_DISABLE_INTERRUPT MiniportDisableInterruptEx;
static MINIPORT_ENABLE_INTERRUPT MiniportEnableInterruptEx;
static MINIPORT_INTERRUPT_DPC MiniportInterruptDPC;
static MINIPORT_ISR MiniportInterrupt;
static MINIPORT_ENABLE_MESSAGE_INTERRUPT MiniportEnableMSIInterrupt;
static MINIPORT_DISABLE_MESSAGE_INTERRUPT MiniportDisableMSIInterrupt;
static MINIPORT_MESSAGE_INTERRUPT MiniportMSIInterrupt;
static MINIPORT_MESSAGE_INTERRUPT_DPC MiniportMSIInterruptDpc;
static MINIPORT_PROCESS_SG_LIST ProcessSGListHandler;
static MINIPORT_ALLOCATE_SHARED_MEM_COMPLETE SharedMemAllocateCompleteHandler;
static MINIPORT_PROCESS_SG_LIST ProcessSGListHandler;
/**********************************************************
Implements general-purpose memory allocation routine
Parameters:
ULONG ulRequiredSize: block size
Return value:
PVOID allocated memory block
NULL on error
***********************************************************/
PVOID ParaNdis_AllocateMemoryRaw(NDIS_HANDLE MiniportHandle, ULONG ulRequiredSize)
{
return NdisAllocateMemoryWithTagPriority(
MiniportHandle,
ulRequiredSize,
PARANDIS_MEMORY_TAG,
NormalPoolPriority);
}
PVOID ParaNdis_AllocateMemory(PARANDIS_ADAPTER *pContext, ULONG ulRequiredSize)
{
return ParaNdis_AllocateMemoryRaw(pContext->MiniportHandle, ulRequiredSize);
}
/**********************************************************
Implements opening of adapter-specific configuration
Parameters:
Return value:
NDIS_HANDLE Handle of open configuration
NULL on error
***********************************************************/
NDIS_HANDLE ParaNdis_OpenNICConfiguration(PARANDIS_ADAPTER *pContext)
{
NDIS_CONFIGURATION_OBJECT co;
NDIS_HANDLE cfg;
NDIS_STATUS status;
DEBUG_ENTRY(2);
co.Header.Type = NDIS_OBJECT_TYPE_CONFIGURATION_OBJECT;
co.Header.Revision = NDIS_CONFIGURATION_OBJECT_REVISION_1;
co.Header.Size = sizeof(co);
co.Flags = 0;
co.NdisHandle = pContext->MiniportHandle;
status = NdisOpenConfigurationEx(&co, &cfg);
if (status != NDIS_STATUS_SUCCESS)
cfg = NULL;
DEBUG_EXIT_STATUS(status == NDIS_STATUS_SUCCESS ? 2 : 0, status);
return cfg;
}
/**********************************************************
NDIS6 implementation of shared memory allocation
Parameters:
context
tCompletePhysicalAddress *pAddresses
the structure accumulates all our knowledge
about the allocation (size, addresses, cacheability etc)
Return value:
TRUE if the allocation was successful
***********************************************************/
BOOLEAN ParaNdis_InitialAllocatePhysicalMemory(
PARANDIS_ADAPTER *pContext,
ULONG ulSize,
tCompletePhysicalAddress *pAddresses)
{
NdisMAllocateSharedMemory(
pContext->MiniportHandle,
ulSize,
TRUE,
&pAddresses->Virtual,
&pAddresses->Physical);
if (pAddresses->Virtual != NULL)
{
pAddresses->size = ulSize;
return TRUE;
}
return FALSE;
}
/**********************************************************
NDIS6 implementation of shared memory freeing
Parameters:
context
tCompletePhysicalAddress *pAddresses
the structure accumulates all our knowledge
about the allocation (size, addresses, cacheability etc)
filled by ParaNdis_InitialAllocatePhysicalMemory
***********************************************************/
VOID ParaNdis_FreePhysicalMemory(
PARANDIS_ADAPTER *pContext,
tCompletePhysicalAddress *pAddresses)
{
NdisMFreeSharedMemory(
pContext->MiniportHandle,
pAddresses->size,
TRUE,
pAddresses->Virtual,
pAddresses->Physical);
}
#if (NDIS_SUPPORT_NDIS620)
typedef MINIPORT_SYNCHRONIZE_INTERRUPT_HANDLER NDIS_SYNC_PROC_TYPE;
#else
typedef PVOID NDIS_SYNC_PROC_TYPE;
#endif
BOOLEAN ParaNdis_SynchronizeWithInterrupt(
PARANDIS_ADAPTER *pContext,
ULONG messageId,
tSynchronizedProcedure procedure,
PVOID parameter)
{
NDIS_SYNC_PROC_TYPE syncProc;
syncProc = (NDIS_SYNC_PROC_TYPE) procedure;
return NdisMSynchronizeWithInterruptEx(pContext->InterruptHandle, messageId, syncProc, parameter);
}
/**********************************************************
NDIS-required procedure for hardware interrupt registration
Parameters:
IN PVOID MiniportInterruptContext (actually Adapter context)
***********************************************************/
static VOID MiniportDisableInterruptEx(IN PVOID MiniportInterruptContext)
{
DEBUG_ENTRY(0);
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)MiniportInterruptContext;
/* TODO - make sure that interrups are not reenabled by the DPC callback*/
for (UINT i = 0; i < pContext->nPathBundles; i++)
{
pContext->pPathBundles[i].txPath.DisableInterrupts();
pContext->pPathBundles[i].rxPath.DisableInterrupts();
}
if (pContext->bCXPathCreated)
{
pContext->CXPath.DisableInterrupts();
}
}
/**********************************************************
NDIS-required procedure for hardware interrupt registration
Parameters:
IN PVOID MiniportInterruptContext (actually Adapter context)
***********************************************************/
static VOID MiniportEnableInterruptEx(IN PVOID MiniportInterruptContext)
{
DEBUG_ENTRY(0);
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)MiniportInterruptContext;
for (UINT i = 0; i < pContext->nPathBundles; i++)
{
pContext->pPathBundles[i].txPath.EnableInterrupts();
pContext->pPathBundles[i].rxPath.EnableInterrupts();
}
if (pContext->bCXPathCreated)
{
pContext->CXPath.EnableInterrupts();
}
}
/**********************************************************
NDIS-required procedure for hardware interrupt handling
Parameters:
IN PVOID MiniportInterruptContext (actually Adapter context)
OUT PBOOLEAN QueueDefaultInterruptDpc - set to TRUE for default DPC spawning
OUT PULONG TargetProcessors
Return value:
TRUE if recognized
***********************************************************/
static BOOLEAN MiniportInterrupt(
IN PVOID MiniportInterruptContext,
OUT PBOOLEAN QueueDefaultInterruptDpc,
OUT PULONG TargetProcessors
)
{
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)MiniportInterruptContext;
ULONG status = virtio_read_isr_status(&pContext->IODevice);
*TargetProcessors = 0;
if((status == 0) ||
(status == VIRTIO_NET_INVALID_INTERRUPT_STATUS))
{
*QueueDefaultInterruptDpc = FALSE;
return FALSE;
}
PARANDIS_STORE_LAST_INTERRUPT_TIMESTAMP(pContext);
if(!pContext->bDeviceInitialized) {
*QueueDefaultInterruptDpc = FALSE;
return TRUE;
}
for (UINT i = 0; i < pContext->nPathBundles; i++)
{
pContext->pPathBundles[i].txPath.DisableInterrupts();
pContext->pPathBundles[i].rxPath.DisableInterrupts();
}
if (pContext->bCXPathCreated)
{
pContext->CXPath.DisableInterrupts();
}
*QueueDefaultInterruptDpc = TRUE;
pContext->ulIrqReceived += 1;
return true;
}
// This procedure must work the same way as
// ParaNdis_ConfigureMSIXVectors when spreads vectors over RX/TX/CX pathes.
// Returns respective TX or RX path if exists, then CX path if exists
// (i.e. returns CX path only if it has dedicated vector)
// otherwise (unlikely) returns NULL
static CParaNdisAbstractPath *GetPathByMessageId(PARANDIS_ADAPTER *pContext, ULONG MessageId)
{
CParaNdisAbstractPath *path;
UINT bundleId = MessageId / 2;
if (bundleId < pContext->nPathBundles)
{
if (MessageId & 1)
{
path = &(pContext->pPathBundles[bundleId].rxPath);
}
else
{
path = &(pContext->pPathBundles[bundleId].txPath);
}
}
else if (pContext->CXPath.getMessageIndex() == MessageId)
{
path = &pContext->CXPath;
}
else
{
path = NULL;
}
return path;
}
/**********************************************************
NDIS-required procedure for MSI hardware interrupt handling
Parameters:
IN PVOID MiniportInterruptContext (actually Adapter context)
IN ULONG MessageId - specific interrupt index
OUT PBOOLEAN QueueDefaultInterruptDpc - - set to TRUE for default DPC spawning
OUT PULONG TargetProcessors
Return value:
TRUE if recognized
***********************************************************/
static BOOLEAN MiniportMSIInterrupt(
IN PVOID MiniportInterruptContext,
IN ULONG MessageId,
OUT PBOOLEAN QueueDefaultInterruptDpc,
OUT PULONG TargetProcessors
)
{
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)MiniportInterruptContext;
PARANDIS_STORE_LAST_INTERRUPT_TIMESTAMP(pContext);
*TargetProcessors = 0;
*QueueDefaultInterruptDpc = FALSE;
if (!pContext->bDeviceInitialized) {
return TRUE;
}
CParaNdisAbstractPath *path = GetPathByMessageId(pContext, MessageId);
if (NULL == path) {
*QueueDefaultInterruptDpc = FALSE;
return TRUE;
}
path->SetLastInterruptTimestamp(pContext->LastInterruptTimeStamp);
path->DisableInterrupts();
// emit DPC for processing of TX/RX or DPC for processing of CX path
// note that in case the CX shares vector with TX/RX the CX DPC is not
// scheduled and CX path is processed by ParaNdis_RXTXDPCWorkBody
// (see pContext->bSharedVectors for such case)
if (!path->FireDPC(MessageId))
{
*QueueDefaultInterruptDpc = TRUE;
#if !NDIS_SUPPORT_NDIS620
*TargetProcessors = (ULONG)path->DPCTargetProcessor;
#endif
}
pContext->ulIrqReceived += 1;
return true;
}
#if NDIS_SUPPORT_NDIS620
static __inline
VOID GetAffinityForCurrentCpu(PGROUP_AFFINITY pAffinity)
{
PROCESSOR_NUMBER ProcNum;
KeGetCurrentProcessorNumberEx(&ProcNum);
pAffinity->Group = ProcNum.Group;
pAffinity->Mask = 1;
pAffinity->Mask <<= ProcNum.Number;
}
#endif
/**********************************************************
NDIS-required procedure for DPC handling
Parameters:
PVOID MiniportInterruptContext (Adapter context)
***********************************************************/
static VOID MiniportInterruptDPC(
IN NDIS_HANDLE MiniportInterruptContext,
IN PVOID MiniportDpcContext,
IN PVOID ReceiveThrottleParameters,
IN PVOID NdisReserved2
)
{
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)MiniportInterruptContext;
bool requiresDPCRescheduling;
#if NDIS_SUPPORT_NDIS620
PNDIS_RECEIVE_THROTTLE_PARAMETERS RxThrottleParameters = (PNDIS_RECEIVE_THROTTLE_PARAMETERS)ReceiveThrottleParameters;
DEBUG_ENTRY(5);
RxThrottleParameters->MoreNblsPending = 0;
requiresDPCRescheduling = ParaNdis_RXTXDPCWorkBody(pContext, RxThrottleParameters->MaxNblsToIndicate);
if (requiresDPCRescheduling)
{
GROUP_AFFINITY Affinity;
GetAffinityForCurrentCpu(&Affinity);
NdisMQueueDpcEx(pContext->InterruptHandle, 0, &Affinity, MiniportDpcContext);
}
#else /* NDIS 6.0*/
DEBUG_ENTRY(5);
UNREFERENCED_PARAMETER(ReceiveThrottleParameters);
requiresDPCRescheduling = ParaNdis_RXTXDPCWorkBody(pContext, PARANDIS_UNLIMITED_PACKETS_TO_INDICATE);
if (requiresDPCRescheduling)
{
DPrintf(4, "[%s] Queued additional DPC for %d\n", __FUNCTION__, requiresDPCRescheduling);
NdisMQueueDpc(pContext->InterruptHandle, 0, 1 << KeGetCurrentProcessorNumber(), MiniportDpcContext);
}
#endif /* NDIS_SUPPORT_NDIS620 */
UNREFERENCED_PARAMETER(NdisReserved2);
}
/**********************************************************
A CX procedure for MSI DPC handling
Parameters:
KDPC * Dpc - The dpc structure for CX
IN ULONG MessageId - specific interrupt index
***********************************************************/
VOID _Function_class_(KDEFERRED_ROUTINE) MiniportMSIInterruptCXDpc(
struct _KDPC *Dpc,
IN PVOID MiniportInterruptContext,
IN PVOID NdisReserved1,
IN PVOID NdisReserved2
)
{
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)MiniportInterruptContext;
ParaNdis_CXDPCWorkBody(pContext);
UNREFERENCED_PARAMETER(Dpc);
UNREFERENCED_PARAMETER(NdisReserved1);
UNREFERENCED_PARAMETER(NdisReserved2);
}
/**********************************************************
NDIS-required procedure for MSI DPC handling
Parameters:
PVOID MiniportInterruptContext (Adapter context)
IN ULONG MessageId - specific interrupt index
***********************************************************/
static VOID MiniportMSIInterruptDpc(
IN PVOID MiniportInterruptContext,
IN ULONG MessageId,
IN PVOID MiniportDpcContext,
#if NDIS_SUPPORT_NDIS620
IN PVOID ReceiveThrottleParameters,
IN PVOID NdisReserved2
#else
IN PULONG NdisReserved1,
IN PULONG NdisReserved2
#endif
)
{
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)MiniportInterruptContext;
bool requireDPCRescheduling;
#if NDIS_SUPPORT_NDIS620
PNDIS_RECEIVE_THROTTLE_PARAMETERS RxThrottleParameters = (PNDIS_RECEIVE_THROTTLE_PARAMETERS)ReceiveThrottleParameters;
RxThrottleParameters->MoreNblsPending = 0;
requireDPCRescheduling = ParaNdis_RXTXDPCWorkBody(pContext, RxThrottleParameters->MaxNblsToIndicate);
if (requireDPCRescheduling)
{
GROUP_AFFINITY Affinity;
GetAffinityForCurrentCpu(&Affinity);
NdisMQueueDpcEx(pContext->InterruptHandle, MessageId, &Affinity, MiniportDpcContext);
}
#else
UNREFERENCED_PARAMETER(NdisReserved1);
requireDPCRescheduling = ParaNdis_RXTXDPCWorkBody(pContext, PARANDIS_UNLIMITED_PACKETS_TO_INDICATE);
if (requireDPCRescheduling)
{
NdisMQueueDpc(pContext->InterruptHandle, MessageId, 1 << KeGetCurrentProcessorNumber(), MiniportDpcContext);
}
#endif
UNREFERENCED_PARAMETER(NdisReserved2);
}
static VOID MiniportDisableMSIInterrupt(
IN PVOID MiniportInterruptContext,
IN ULONG MessageId
)
{
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)MiniportInterruptContext;
/* TODO - How we prevent DPC procedure from re-enabling interrupt? */
CParaNdisAbstractPath *path = GetPathByMessageId(pContext, MessageId);
path->DisableInterrupts();
}
static VOID MiniportEnableMSIInterrupt(
IN PVOID MiniportInterruptContext,
IN ULONG MessageId
)
{
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)MiniportInterruptContext;
CParaNdisAbstractPath *path = GetPathByMessageId(pContext, MessageId);
path->EnableInterrupts();
}
/**********************************************************
NDIS required handler for run-time allocation of physical memory
Parameters:
Return value:
***********************************************************/
static VOID SharedMemAllocateCompleteHandler(
IN NDIS_HANDLE MiniportAdapterContext,
IN PVOID VirtualAddress,
IN PNDIS_PHYSICAL_ADDRESS PhysicalAddress,
IN ULONG Length,
IN PVOID Context
)
{
UNREFERENCED_PARAMETER(MiniportAdapterContext);
UNREFERENCED_PARAMETER(VirtualAddress);
UNREFERENCED_PARAMETER(PhysicalAddress);
UNREFERENCED_PARAMETER(Length);
UNREFERENCED_PARAMETER(Context);
}
/*
We enter this procedure in two cases:
- number of MSIX vectors is at least nPathBundles*2 + 1
- there is single MSIX vector and one path bundle
*/
NDIS_STATUS ParaNdis_ConfigureMSIXVectors(PARANDIS_ADAPTER *pContext)
{
NDIS_STATUS status = NDIS_STATUS_RESOURCES;
UINT i;
u16 nVectors = (u16)pContext->pMSIXInfoTable->MessageCount;
PIO_INTERRUPT_MESSAGE_INFO pTable = pContext->pMSIXInfoTable;
if (pTable && pTable->MessageCount)
{
status = NDIS_STATUS_SUCCESS;
DPrintf(0, "[%s] Using MSIX interrupts (%d messages, irql %d)\n",
__FUNCTION__, pTable->MessageCount, pTable->UnifiedIrql);
for (i = 0; i < pContext->pMSIXInfoTable->MessageCount; ++i)
{
DPrintf(0, "[%s] MSIX message%d=%08X=>%I64X\n",
__FUNCTION__, i,
pTable->MessageInfo[i].MessageData,
pTable->MessageInfo[i].MessageAddress.QuadPart);
}
for (UINT j = 0; j < pContext->nPathBundles && status == NDIS_STATUS_SUCCESS; ++j)
{
u16 vector = 2 * u16(j);
status = pContext->pPathBundles[j].txPath.SetupMessageIndex(vector);
if (status == NDIS_STATUS_SUCCESS)
{
if (nVectors > 1) vector++;
status = pContext->pPathBundles[j].rxPath.SetupMessageIndex(vector);
}
DPrintf(0, "[%s] Using messages %u/%u for RX/TX queue %u\n", __FUNCTION__,
pContext->pPathBundles[j].rxPath.getMessageIndex(),
pContext->pPathBundles[j].txPath.getMessageIndex(),
j);
}
if (status == NDIS_STATUS_SUCCESS && pContext->bCXPathCreated)
{
u16 nVector;
/*
Usually there is own vector for control queue.
In corner case of one or two vectors control queue uses the same vector as RX or TX
and does not spawn its own DPC for processing
*/
if (nVectors < 3)
{
nVector = nVectors - 1;
pContext->bSharedVectors = TRUE;
}
else
{
nVector = 2 * u16(pContext->nPathBundles);
}
status = pContext->CXPath.SetupMessageIndex(nVector);
}
}
DEBUG_EXIT_STATUS(0, status);
return status;
}
void ParaNdis_RestoreDeviceConfigurationAfterReset(
PARANDIS_ADAPTER *pContext)
{
if (pContext->bUsingMSIX)
{
ParaNdis_ConfigureMSIXVectors(pContext);
}
}
static void DebugParseOffloadBits()
{
NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO info;
tChecksumCheckResult res;
ULONG val = 1;
int level = 1;
while (val)
{
info.Value = (PVOID)(ULONG_PTR)val;
if (info.Receive.IpChecksumFailed) DPrintf(level, "W.%X=IPCS failed\n", val);
if (info.Receive.IpChecksumSucceeded) DPrintf(level, "W.%X=IPCS OK\n", val);
if (info.Receive.TcpChecksumFailed) DPrintf(level, "W.%X=TCPCS failed\n", val);
if (info.Receive.TcpChecksumSucceeded) DPrintf(level, "W.%X=TCPCS OK\n", val);
if (info.Receive.UdpChecksumFailed) DPrintf(level, "W.%X=UDPCS failed\n", val);
if (info.Receive.UdpChecksumSucceeded) DPrintf(level, "W.%X=UDPCS OK\n", val);
val = val << 1;
}
val = 1;
while (val)
{
res.value = val;
if (res.flags.IpFailed) DPrintf(level, "C.%X=IPCS failed\n", val);
if (res.flags.IpOK) DPrintf(level, "C.%X=IPCS OK\n", val);
if (res.flags.TcpFailed) DPrintf(level, "C.%X=TCPCS failed\n", val);
if (res.flags.TcpOK) DPrintf(level, "C.%X=TCPCS OK\n", val);
if (res.flags.UdpFailed) DPrintf(level, "C.%X=UDPCS failed\n", val);
if (res.flags.UdpOK) DPrintf(level, "C.%X=UDPCS OK\n", val);
val = val << 1;
}
}
/**********************************************************
NDIS6-related final initialization:
Installing interrupt handler
Allocate buffer list pool
Parameters:
Return value:
***********************************************************/
NDIS_STATUS ParaNdis_FinishSpecificInitialization(PARANDIS_ADAPTER *pContext)
{
NDIS_STATUS status = NDIS_STATUS_SUCCESS;
NET_BUFFER_LIST_POOL_PARAMETERS PoolParams;
NDIS_MINIPORT_INTERRUPT_CHARACTERISTICS mic;
DEBUG_ENTRY(0);
NdisZeroMemory(&mic, sizeof(mic));
mic.Header.Type = NDIS_OBJECT_TYPE_MINIPORT_INTERRUPT;
mic.Header.Revision = NDIS_MINIPORT_INTERRUPT_REVISION_1;
mic.Header.Size = NDIS_SIZEOF_MINIPORT_INTERRUPT_CHARACTERISTICS_REVISION_1;
mic.DisableInterruptHandler = MiniportDisableInterruptEx;
mic.EnableInterruptHandler = MiniportEnableInterruptEx;
mic.InterruptDpcHandler = MiniportInterruptDPC;
mic.InterruptHandler = MiniportInterrupt;
if (pContext->bUsingMSIX)
{
mic.MsiSupported = TRUE;
mic.MsiSyncWithAllMessages = TRUE;
mic.EnableMessageInterruptHandler = MiniportEnableMSIInterrupt;
mic.DisableMessageInterruptHandler = MiniportDisableMSIInterrupt;
mic.MessageInterruptHandler = MiniportMSIInterrupt;
mic.MessageInterruptDpcHandler = MiniportMSIInterruptDpc;
}
PoolParams.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
PoolParams.Header.Size = sizeof(PoolParams);
PoolParams.Header.Revision = NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
PoolParams.ProtocolId = NDIS_PROTOCOL_ID_DEFAULT;
PoolParams.fAllocateNetBuffer = TRUE;
PoolParams.ContextSize = 0;
PoolParams.PoolTag = PARANDIS_MEMORY_TAG;
PoolParams.DataSize = 0;
pContext->BufferListsPool = NdisAllocateNetBufferListPool(pContext->MiniportHandle, &PoolParams);
if (!pContext->BufferListsPool)
{
status = NDIS_STATUS_RESOURCES;
}
#if defined(NETKVM_COPY_RX_DATA)
if (status == NDIS_STATUS_SUCCESS)
{
PoolParams.DataSize = pContext->MaxPacketSize.nMaxFullSizeOsRx;
pContext->BufferListsPoolForArm = NdisAllocateNetBufferListPool(pContext->MiniportHandle, &PoolParams);
if (!pContext->BufferListsPoolForArm)
{
status = NDIS_STATUS_RESOURCES;
}
}
#endif
if (status == NDIS_STATUS_SUCCESS)
{
status = NdisMRegisterInterruptEx(pContext->MiniportHandle, pContext, &mic, &pContext->InterruptHandle);
}
if (pContext->bUsingMSIX)
{
DPrintf(0, "[%s] MSIX message table %savailable, count = %u\n", __FUNCTION__, (mic.MessageInfoTable == nullptr ? "not " : ""),
(mic.MessageInfoTable == nullptr ? 0 : mic.MessageInfoTable->MessageCount));
}
else
{
DPrintf(0, "[%s] Not using MSIX\n", __FUNCTION__);
}
if (status == NDIS_STATUS_SUCCESS)
{
NDIS_SG_DMA_DESCRIPTION sgDesc;
sgDesc.Header.Type = NDIS_OBJECT_TYPE_SG_DMA_DESCRIPTION;
sgDesc.Header.Revision = NDIS_SG_DMA_DESCRIPTION_REVISION_1;
sgDesc.Header.Size = sizeof(sgDesc);
sgDesc.Flags = NDIS_SG_DMA_64_BIT_ADDRESS;
sgDesc.MaximumPhysicalMapping = 0x10000; // 64K
sgDesc.ProcessSGListHandler = ProcessSGListHandler;
sgDesc.SharedMemAllocateCompleteHandler = SharedMemAllocateCompleteHandler;
sgDesc.ScatterGatherListSize = 0; // OUT value
status = NdisMRegisterScatterGatherDma(pContext->MiniportHandle, &sgDesc, &pContext->DmaHandle);
if (status != NDIS_STATUS_SUCCESS)
{
DPrintf(0, "[%s] ERROR: NdisMRegisterScatterGatherDma failed (%X)!\n", __FUNCTION__, status);
}
else
{
DPrintf(0, "[%s] SG recommended size %d\n", __FUNCTION__, sgDesc.ScatterGatherListSize);
}
}
if (status == NDIS_STATUS_SUCCESS)
{
if (NDIS_CONNECT_MESSAGE_BASED == mic.InterruptType)
{
pContext->pMSIXInfoTable = mic.MessageInfoTable;
}
else if (pContext->bUsingMSIX)
{
DPrintf(0, "[%s] ERROR: Interrupt type %d, message table %p\n",
__FUNCTION__, mic.InterruptType, mic.MessageInfoTable);
status = NDIS_STATUS_RESOURCE_CONFLICT;
}
ParaNdis6_ApplyOffloadPersistentConfiguration(pContext);
DebugParseOffloadBits();
}
DEBUG_EXIT_STATUS(0, status);
return status;
}
/**********************************************************
NDIS6-related final initialization:
Uninstalling interrupt handler
Dellocate buffer list pool
Parameters:
context
***********************************************************/
VOID ParaNdis_FinalizeCleanup(PARANDIS_ADAPTER *pContext)
{
// we zero context members to be able examine them in the debugger/dump
if (pContext->InterruptHandle)
{
NdisMDeregisterInterruptEx(pContext->InterruptHandle);
pContext->InterruptHandle = NULL;
}
if (pContext->BufferListsPool)
{
NdisFreeNetBufferListPool(pContext->BufferListsPool);
pContext->BufferListsPool = NULL;
}
if (pContext->BufferListsPoolForArm)
{
NdisFreeNetBufferListPool(pContext->BufferListsPoolForArm);
pContext->BufferListsPoolForArm = NULL;
}
if (pContext->DmaHandle)
{
NdisMDeregisterScatterGatherDma(pContext->DmaHandle);
pContext->DmaHandle = NULL;
}
}
static
void ParaNdis_AdjustRxBufferHolderLength(
pRxNetDescriptor p,
ULONG ulDataOffset)
{
PMDL NextMdlLinkage = p->Holder;
ULONG ulBytesLeft = p->PacketInfo.dataLength + ulDataOffset;
ULONG ulPageDescIndex = PARANDIS_FIRST_RX_DATA_PAGE;
while(NextMdlLinkage != NULL)
{
ULONG ulThisMdlBytes = min(p->PhysicalPages[ulPageDescIndex].size, ulBytesLeft);
NdisAdjustMdlLength(NextMdlLinkage, ulThisMdlBytes);
ulBytesLeft -= ulThisMdlBytes;
NextMdlLinkage = NDIS_MDL_LINKAGE(NextMdlLinkage);
ulPageDescIndex++;
}
NETKVM_ASSERT(ulBytesLeft == 0);
}
static __inline
VOID NBLSetRSSInfo(PPARANDIS_ADAPTER pContext, PNET_BUFFER_LIST pNBL, PNET_PACKET_INFO PacketInfo)
{
#if PARANDIS_SUPPORT_RSS
CNdisRWLockState lockState;
pContext->RSSParameters.rwLock.acquireReadDpr(lockState);
if(pContext->RSSParameters.RSSMode != PARANDIS_RSS_DISABLED)
{
NET_BUFFER_LIST_SET_HASH_TYPE (pNBL, PacketInfo->RSSHash.Type);
NET_BUFFER_LIST_SET_HASH_FUNCTION(pNBL, PacketInfo->RSSHash.Function);
NET_BUFFER_LIST_SET_HASH_VALUE (pNBL, PacketInfo->RSSHash.Value);
}
pContext->RSSParameters.rwLock.releaseDpr(lockState);
#else
UNREFERENCED_PARAMETER(pContext);
UNREFERENCED_PARAMETER(pNBL);
UNREFERENCED_PARAMETER(PacketInfo);
#endif
}
static __inline
VOID NBLSet8021QInfo(PPARANDIS_ADAPTER pContext, PNET_BUFFER_LIST pNBL, PNET_PACKET_INFO pPacketInfo)
{
NDIS_NET_BUFFER_LIST_8021Q_INFO qInfo;
qInfo.Value = NULL;
if (IsPrioritySupported(pContext))
qInfo.TagHeader.UserPriority = pPacketInfo->Vlan.UserPriority;
if (IsVlanSupported(pContext))
qInfo.TagHeader.VlanId = pPacketInfo->Vlan.VlanId;
if(qInfo.Value != NULL)
pContext->extraStatistics.framesRxPriority++;
NET_BUFFER_LIST_INFO(pNBL, Ieee8021QNetBufferListInfo) = qInfo.Value;
}
#if PARANDIS_SUPPORT_RSC
static __inline
UINT PktGetTCPCoalescedSegmentsCount(PPARANDIS_ADAPTER pContext,
PNET_PACKET_INFO PacketInfo,
UINT mss)
{
// We have no exact data from the device, but can evaluate the number
// of coalesced segments according to mss (max segment size) value
// provided in the packet header. It includes only size of
// TCP payload and TCP payload of final packet was earlier
// splitted over TCP segments with payload of mss bytes each
ULONG TcpHeaderOffset = PacketInfo->L2HdrLen + PacketInfo->L3HdrLen;
auto TCPHdr = reinterpret_cast<TCPHeader *>RtlOffsetToPointer(PacketInfo->headersBuffer, TcpHeaderOffset);
ULONG IpAndTcpHeaderSize = PacketInfo->L3HdrLen + TCP_HEADER_LENGTH(TCPHdr);
ULONG payloadLen = PacketInfo->L2PayloadLen - IpAndTcpHeaderSize;
// if we have meaningless value from the device, use
// the best estimation - max possible TCP payload
// nMaxDataSize (typically 1500) starts after ethernet header
ULONG maxMss = pContext->MaxPacketSize.nMaxDataSize - IpAndTcpHeaderSize;
if (mss > maxMss || mss == 0)
{
mss = maxMss;
}
return payloadLen / mss + !!(payloadLen % mss);
}
static __inline
VOID NBLSetRSCInfo(PPARANDIS_ADAPTER pContext, PNET_BUFFER_LIST pNBL,
PNET_PACKET_INFO PacketInfo, UINT nCoalescedSegments, USHORT nDupAck)
{
NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO qCSInfo;
qCSInfo.Value = NULL;
qCSInfo.Receive.IpChecksumSucceeded = TRUE;
qCSInfo.Receive.IpChecksumValueInvalid = TRUE;
qCSInfo.Receive.TcpChecksumSucceeded = TRUE;
qCSInfo.Receive.TcpChecksumValueInvalid = TRUE;
NET_BUFFER_LIST_INFO(pNBL, TcpIpChecksumNetBufferListInfo) = qCSInfo.Value;
NET_BUFFER_LIST_COALESCED_SEG_COUNT(pNBL) = (USHORT) nCoalescedSegments;
NET_BUFFER_LIST_DUP_ACK_COUNT(pNBL) = nDupAck;
NdisInterlockedAddLargeStatistic(&pContext->RSC.Statistics.CoalescedOctets, PacketInfo->L2PayloadLen);
NdisInterlockedAddLargeStatistic(&pContext->RSC.Statistics.CoalesceEvents, 1);
NdisInterlockedAddLargeStatistic(&pContext->RSC.Statistics.CoalescedPkts, nCoalescedSegments);
}
#endif
#if defined(NETKVM_COPY_RX_DATA)
static PNET_BUFFER_LIST CloneNblFreeOriginalForArm(PARANDIS_ADAPTER *pContext, PNET_BUFFER_LIST original, PVOID bufferDesc)
{
PNET_BUFFER_LIST pNewNbl;
if (!original)
{
return NULL;
}
pNewNbl = NdisAllocateNetBufferList(pContext->BufferListsPoolForArm, 0, NULL);
if (pNewNbl)
{
PNET_BUFFER src = NET_BUFFER_LIST_FIRST_NB(original);
PNET_BUFFER dest = NET_BUFFER_LIST_FIRST_NB(pNewNbl);
ULONG done = 0;
NET_BUFFER_DATA_LENGTH(dest) = NET_BUFFER_DATA_LENGTH(src);
NET_BUFFER_DATA_OFFSET(dest) = 0;
NET_BUFFER_CURRENT_MDL_OFFSET(dest) = 0;
NdisCopyFromNetBufferToNetBuffer(dest, 0, NET_BUFFER_DATA_LENGTH(src), src, 0, &done);
if (done == NET_BUFFER_DATA_LENGTH(src))
{
NdisCopyReceiveNetBufferListInfo(pNewNbl, original);
pNewNbl->MiniportReserved[0] = bufferDesc;
}
else
{
DPrintf(0, "[%s] ERROR: Can't copy data to NBL (%d != %d)\n",
__FUNCTION__, done, NET_BUFFER_DATA_LENGTH(src));
NdisFreeNetBufferList(pNewNbl);
pNewNbl = NULL;
}
}
else
{
DPrintf(0, "[%s] ERROR: Can't allocate NBL\n", __FUNCTION__);
}
NdisFreeNetBufferList(original);
return pNewNbl;
}
#else
#define CloneNblFreeOriginalForArm(ctx, org, bufDesc) (org)
#endif
/**********************************************************
NDIS6 implementation of packet indication
Parameters:
context
PVOID pBuffersDescriptor - VirtIO buffer descriptor of data buffer
BOOLEAN bPrepareOnly - only return NBL for further indication in batch
Return value:
TRUE is packet indicated
FALSE if not (in this case, the descriptor should be freed now)
If priority header is in the packet. it will be removed and *pLength decreased
***********************************************************/
tPacketIndicationType ParaNdis_PrepareReceivedPacket(
PARANDIS_ADAPTER *pContext,
pRxNetDescriptor pBuffersDesc,
PUINT pnCoalescedSegmentsCount)
{
PMDL pMDL = pBuffersDesc->Holder;
PNET_BUFFER_LIST pNBL = NULL;
*pnCoalescedSegmentsCount = 1;
if (pMDL)
{
ULONG nBytesStripped = 0;
PNET_PACKET_INFO pPacketInfo = &pBuffersDesc->PacketInfo;
if (pContext->ulPriorityVlanSetting && pPacketInfo->hasVlanHeader)
{
nBytesStripped = ParaNdis_StripVlanHeaderMoveHead(pPacketInfo);
}
ParaNdis_PadPacketToMinimalLength(pPacketInfo);
ParaNdis_AdjustRxBufferHolderLength(pBuffersDesc, nBytesStripped);
pNBL = NdisAllocateNetBufferAndNetBufferList(pContext->BufferListsPool, 0, 0, pMDL, nBytesStripped, pPacketInfo->dataLength);
if (pNBL)
{
virtio_net_hdr_mrg_rxbuf *pHeader = (virtio_net_hdr_mrg_rxbuf *) pBuffersDesc->PhysicalPages[0].Virtual;
tChecksumCheckResult csRes;
NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO qCSInfo;
qCSInfo.Value = NULL;
pNBL->SourceHandle = pContext->MiniportHandle;
NBLSetRSSInfo(pContext, pNBL, pPacketInfo);
NBLSet8021QInfo(pContext, pNBL, pPacketInfo);
pNBL->MiniportReserved[0] = pBuffersDesc;
#if PARANDIS_SUPPORT_RSC
csRes.value = 0;
csRes.flags.IpOK = true;
csRes.flags.TcpOK = true;
if (pHeader->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE)
{
USHORT nDupAcks = 0;
if (pHeader->hdr.flags & VIRTIO_NET_HDR_F_RSC_INFO)
{
*pnCoalescedSegmentsCount = pHeader->hdr.rsc_ext_num_packets;
nDupAcks = pHeader->hdr.rsc_ext_num_dupacks;
pContext->extraStatistics.framesCoalescedWindows++;
}
else
{
*pnCoalescedSegmentsCount = PktGetTCPCoalescedSegmentsCount(pContext, pPacketInfo, pHeader->hdr.gso_size);
pContext->extraStatistics.framesCoalescedHost++;
}
NBLSetRSCInfo(pContext, pNBL, pPacketInfo, *pnCoalescedSegmentsCount, 0);
// according to the spec the device does not calculate TCP checksum
qCSInfo.Receive.IpChecksumValueInvalid = true;
qCSInfo.Receive.TcpChecksumValueInvalid = true;
}
else
#endif
{
csRes = ParaNdis_CheckRxChecksum(
pContext,
pHeader->hdr.flags,
&pBuffersDesc->PhysicalPages[PARANDIS_FIRST_RX_DATA_PAGE],
pPacketInfo,
nBytesStripped, TRUE);
}
if (csRes.value)
{
qCSInfo.Receive.IpChecksumFailed = csRes.flags.IpFailed;
qCSInfo.Receive.IpChecksumSucceeded = csRes.flags.IpOK;
qCSInfo.Receive.TcpChecksumFailed = csRes.flags.TcpFailed;
qCSInfo.Receive.TcpChecksumSucceeded = csRes.flags.TcpOK;
qCSInfo.Receive.UdpChecksumFailed = csRes.flags.UdpFailed;
qCSInfo.Receive.UdpChecksumSucceeded = csRes.flags.UdpOK;
}
NET_BUFFER_LIST_INFO(pNBL, TcpIpChecksumNetBufferListInfo) = qCSInfo.Value;
DPrintf(1, "datalen %d, GSO type/flags %d:%d, mss %d, %d segments, CS %X->%X\n",
pPacketInfo->dataLength, pHeader->hdr.gso_type, pHeader->hdr.flags, pHeader->hdr.gso_size,
*pnCoalescedSegmentsCount, csRes.value, (ULONG)(ULONG_PTR)qCSInfo.Value);
pNBL->Status = NDIS_STATUS_SUCCESS;
#if defined(ENABLE_HISTORY_LOG)
{
tTcpIpPacketParsingResult packetReview = ParaNdis_CheckSumVerify(
RtlOffsetToPointer(pPacketInfo->headersBuffer, ETH_HEADER_SIZE),
pPacketInfo->dataLength,
pcrIpChecksum | pcrTcpChecksum | pcrUdpChecksum,
__FUNCTION__
);
ParaNdis_DebugHistory(pContext, hopPacketReceived, pNBL, pPacketInfo->dataLength, (ULONG)(ULONG_PTR)qInfo.Value, packetReview.value);
}
#endif
}
}
return CloneNblFreeOriginalForArm(pContext, pNBL, pBuffersDesc);
}
/**********************************************************
NDIS procedure of returning us buffer of previously indicated packets
Parameters:
context
PNET_BUFFER_LIST pNBL - list of buffers to free
returnFlags - is dpc
The procedure frees:
received buffer descriptors back to list of RX buffers
all the allocated MDL structures
all the received NBLs back to our pool
***********************************************************/
VOID ParaNdis6_ReturnNetBufferLists(
NDIS_HANDLE miniportAdapterContext,
PNET_BUFFER_LIST pNBL,
ULONG returnFlags)
{
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)miniportAdapterContext;
auto NumNBLs = ParaNdis_CountNBLs(pNBL);
UNREFERENCED_PARAMETER(returnFlags);
DEBUG_ENTRY(5);
ParaNdis_ReuseRxNBLs(pNBL);
pContext->m_RxStateMachine.UnregisterOutstandingItems(NumNBLs);
}
NDIS_STATUS ParaNdis_ExactSendFailureStatus(PARANDIS_ADAPTER *pContext)
{
NDIS_STATUS status = NDIS_STATUS_FAILURE;
if (!pContext->bConnected) status = NDIS_STATUS_MEDIA_DISCONNECTED;
if (pContext->bSurprizeRemoved) status = NDIS_STATUS_NOT_ACCEPTED;
return status;
}
BOOLEAN ParaNdis_IsSendPossible(PARANDIS_ADAPTER *pContext)
{
BOOLEAN b;
b = !pContext->bSurprizeRemoved && pContext->bConnected;
return b;
}
/**********************************************************
NDIS required handler for run-time allocation of scatter-gather list
Parameters:
pSGL - scatter-hather list of elements (possible NULL when called directly)
Context - (tNetBufferEntry *) for specific NET_BUFFER in NBL
Called on DPC (DDK claims it)
***********************************************************/
VOID ProcessSGListHandler(
IN PDEVICE_OBJECT pDO,
IN PVOID Reserved,
IN PSCATTER_GATHER_LIST pSGL,
IN PVOID Context
)
{
UNREFERENCED_PARAMETER(Reserved);
UNREFERENCED_PARAMETER(pDO);
auto NBHolder = static_cast<CNB*>(Context);
NBHolder->MappingDone(pSGL);
}
/**********************************************************
Required procedure of NDIS
NDIS wants to cancel sending of each list which has specified CancelID
Can be tested only under NDIS Test
***********************************************************/
VOID ParaNdis6_CancelSendNetBufferLists(
NDIS_HANDLE miniportAdapterContext,
PVOID pCancelId)
{
PARANDIS_ADAPTER *pContext = (PARANDIS_ADAPTER *)miniportAdapterContext;
DEBUG_ENTRY(0);
for (UINT i = 0; i < pContext->nPathBundles; i++)
{
pContext->pPathBundles[i].txPath.CancelNBLs(pCancelId);
}
}
Reference in new issue View Git Blame Copy Permalink