diff --git a/sample/Magma.DPDK.PacketForwarder/Magma.DPDK.PacketForwarder.csproj b/sample/Magma.DPDK.PacketForwarder/Magma.DPDK.PacketForwarder.csproj new file mode 100644 index 0000000..f4ebd4e --- /dev/null +++ b/sample/Magma.DPDK.PacketForwarder/Magma.DPDK.PacketForwarder.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + false + true + + + + latest + + + + + + + + + diff --git a/sample/Magma.DPDK.PacketForwarder/Program.cs b/sample/Magma.DPDK.PacketForwarder/Program.cs new file mode 100644 index 0000000..e85e55c --- /dev/null +++ b/sample/Magma.DPDK.PacketForwarder/Program.cs @@ -0,0 +1,227 @@ +using System; +using System.Buffers; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Magma.Link; + +namespace Magma.DPDK.PacketForwarder +{ + /// + /// DPDK-based L2 packet forwarder that forwards Ethernet frames between two ports. + /// This is a reference implementation demonstrating the forwarding logic. + /// + /// Note: This sample requires a full DPDK transport implementation (see issue #128). + /// Currently serves as a reference/placeholder for the DPDK integration. + /// + class Program + { + static async Task Main(string[] args) + { + Console.WriteLine("Magma DPDK Packet Forwarder"); + Console.WriteLine("==========================="); + Console.WriteLine(); + + if (args.Length < 2) + { + PrintUsage(); + return; + } + + var port0 = args[0]; + var port1 = args[1]; + + Console.WriteLine($"Forwarding configuration:"); + Console.WriteLine($" Port 0: {port0}"); + Console.WriteLine($" Port 1: {port1}"); + Console.WriteLine(); + Console.WriteLine("Note: Full DPDK transport implementation required (issue #128)"); + Console.WriteLine("Press Ctrl+C to exit"); + Console.WriteLine(); + + var forwarder = new PacketForwarder(port0, port1); + + using var cts = new CancellationTokenSource(); + Console.CancelKeyPress += (s, e) => + { + e.Cancel = true; + cts.Cancel(); + }; + + try + { + await forwarder.RunAsync(cts.Token); + } + catch (OperationCanceledException) + { + Console.WriteLine("Shutting down..."); + } + catch (Exception ex) + { + Console.WriteLine($"Error: {ex.Message}"); + Environment.ExitCode = 1; + } + } + + static void PrintUsage() + { + Console.WriteLine("Usage: Magma.DPDK.PacketForwarder "); + Console.WriteLine(); + Console.WriteLine("Arguments:"); + Console.WriteLine(" port0 First DPDK port (e.g., 0000:00:08.0)"); + Console.WriteLine(" port1 Second DPDK port (e.g., 0000:00:09.0)"); + Console.WriteLine(); + Console.WriteLine("Example:"); + Console.WriteLine(" Magma.DPDK.PacketForwarder 0000:00:08.0 0000:00:09.0"); + Console.WriteLine(); + Console.WriteLine("Prerequisites:"); + Console.WriteLine(" - DPDK libraries installed"); + Console.WriteLine(" - Huge pages configured"); + Console.WriteLine(" - NICs bound to DPDK-compatible driver (igb_uio/vfio-pci)"); + Console.WriteLine(" - See README.md for detailed setup instructions"); + } + } + + /// + /// L2 packet forwarder that exchanges packets between two ports. + /// Demonstrates zero-copy forwarding pattern for DPDK integration. + /// + class PacketForwarder + { + private readonly string _port0; + private readonly string _port1; + private long _packetsForwarded0to1; + private long _packetsForwarded1to0; + private long _bytesForwarded0to1; + private long _bytesForwarded1to0; + + public PacketForwarder(string port0, string port1) + { + _port0 = port0; + _port1 = port1; + } + + public async Task RunAsync(CancellationToken cancellationToken) + { + Console.WriteLine("Starting packet forwarding..."); + + // TODO: Initialize DPDK environment (EAL) + // TODO: Configure ports + // TODO: Setup memory pools + + // Start statistics reporting + var statsTask = ReportStatisticsAsync(cancellationToken); + + // Main forwarding loop (placeholder) + await Task.Run(() => ForwardingLoop(cancellationToken), cancellationToken); + + await statsTask; + } + + private void ForwardingLoop(CancellationToken cancellationToken) + { + // Placeholder for actual DPDK forwarding logic + // In a real implementation, this would: + // 1. Receive burst of packets from port 0 + // 2. Forward to port 1 + // 3. Receive burst of packets from port 1 + // 4. Forward to port 0 + // 5. Repeat + + Console.WriteLine("Forwarding loop started (placeholder - requires DPDK transport)"); + + while (!cancellationToken.IsCancellationRequested) + { + // Simulate packet forwarding delay + Thread.Sleep(100); + + // In real implementation: + // - Use rte_eth_rx_burst() to receive packets + // - Use rte_eth_tx_burst() to send packets + // - Update statistics + } + } + + /// + /// Demonstrates the pattern for processing Ethernet frames. + /// This method shows how packet data would be handled once DPDK transport is available. + /// + private void ProcessPacket(ReadOnlySpan packetData, int sourcePort) + { + // Parse Ethernet header + if (!EthernetFrame.TryConsume(packetData, out var frame, out var payload)) + { + return; + } + + // In a real L2 forwarder: + // - Optionally update MAC addresses + // - Forward to the other port + // - Update statistics + + var targetPort = sourcePort == 0 ? 1 : 0; + var length = packetData.Length; + + if (sourcePort == 0) + { + Interlocked.Increment(ref _packetsForwarded0to1); + Interlocked.Add(ref _bytesForwarded0to1, length); + } + else + { + Interlocked.Increment(ref _packetsForwarded1to0); + Interlocked.Add(ref _bytesForwarded1to0, length); + } + } + + private async Task ReportStatisticsAsync(CancellationToken cancellationToken) + { + var lastReport = DateTime.UtcNow; + var lastPackets0to1 = 0L; + var lastPackets1to0 = 0L; + var lastBytes0to1 = 0L; + var lastBytes1to0 = 0L; + + while (!cancellationToken.IsCancellationRequested) + { + try + { + await Task.Delay(5000, cancellationToken); + } + catch (OperationCanceledException) + { + break; + } + + var now = DateTime.UtcNow; + var elapsed = (now - lastReport).TotalSeconds; + + var packets0to1 = Interlocked.Read(ref _packetsForwarded0to1); + var packets1to0 = Interlocked.Read(ref _packetsForwarded1to0); + var bytes0to1 = Interlocked.Read(ref _bytesForwarded0to1); + var bytes1to0 = Interlocked.Read(ref _bytesForwarded1to0); + + var pps0to1 = (packets0to1 - lastPackets0to1) / elapsed; + var pps1to0 = (packets1to0 - lastPackets1to0) / elapsed; + var mbps0to1 = ((bytes0to1 - lastBytes0to1) * 8.0 / elapsed) / 1_000_000.0; + var mbps1to0 = ((bytes1to0 - lastBytes1to0) * 8.0 / elapsed) / 1_000_000.0; + + Console.WriteLine($"[{now:HH:mm:ss}] Statistics:"); + Console.WriteLine($" {_port0} -> {_port1}: {packets0to1:N0} packets ({pps0to1:F2} pps, {mbps0to1:F2} Mbps)"); + Console.WriteLine($" {_port1} -> {_port0}: {packets1to0:N0} packets ({pps1to0:F2} pps, {mbps1to0:F2} Mbps)"); + + lastReport = now; + lastPackets0to1 = packets0to1; + lastPackets1to0 = packets1to0; + lastBytes0to1 = bytes0to1; + lastBytes1to0 = bytes1to0; + } + + // Print final statistics + Console.WriteLine(); + Console.WriteLine("Final Statistics:"); + Console.WriteLine($" {_port0} -> {_port1}: {_packetsForwarded0to1:N0} packets ({_bytesForwarded0to1:N0} bytes)"); + Console.WriteLine($" {_port1} -> {_port0}: {_packetsForwarded1to0:N0} packets ({_bytesForwarded1to0:N0} bytes)"); + } + } +} diff --git a/sample/Magma.DPDK.PacketForwarder/README.md b/sample/Magma.DPDK.PacketForwarder/README.md new file mode 100644 index 0000000..0ee7f7f --- /dev/null +++ b/sample/Magma.DPDK.PacketForwarder/README.md @@ -0,0 +1,363 @@ +# Magma DPDK Packet Forwarder + +A high-performance L2 (Layer 2) packet forwarder that exchanges Ethernet frames between two DPDK ports at line rate. + +## Overview + +This sample application demonstrates how to build a packet forwarder using DPDK (Data Plane Development Kit) with the Magma network stack. It forwards all received packets from port 0 to port 1 and vice versa, providing a foundation for building more complex network functions. + +## Prerequisites + +### System Requirements + +- **OS**: Linux (Ubuntu 20.04+ or equivalent) +- **Kernel**: 3.16+ (4.4+ recommended) +- **CPU**: x86_64 with SSE4.2 support +- **Memory**: Minimum 2GB RAM, 1GB huge pages reserved +- **.NET**: .NET 10.0 SDK or higher + +### DPDK Requirements + +- **DPDK Version**: 20.11 LTS or higher (23.11 LTS recommended) +- **Compatible NICs**: Intel (igb, ixgbe, i40e), Mellanox, or others with DPDK PMD support +- **Drivers**: DPDK-compatible drivers (vfio-pci, igb_uio, or uio_pci_generic) + +## Installation & Setup + +### 1. Install DPDK + +#### Ubuntu/Debian + +```bash +# Install dependencies +sudo apt-get update +sudo apt-get install -y build-essential libnuma-dev python3-pyelftools \ + linux-headers-$(uname -r) pkg-config meson ninja-build + +# Install DPDK from distribution package (recommended) +sudo apt-get install -y dpdk dpdk-dev + +# Or build from source +wget https://fast.dpdk.org/rel/dpdk-23.11.tar.xz +tar xf dpdk-23.11.tar.xz +cd dpdk-23.11 +meson setup build +cd build +ninja +sudo ninja install +sudo ldconfig +``` + +### 2. Configure Huge Pages + +DPDK requires huge pages for efficient memory management: + +```bash +# Configure 1GB of 2MB huge pages (512 pages) +echo 512 | sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + +# Make persistent across reboots +echo "vm.nr_hugepages = 512" | sudo tee -a /etc/sysctl.conf + +# Verify configuration +grep Huge /proc/meminfo +``` + +For 1GB huge pages (recommended for production): + +```bash +# Add kernel parameter +sudo nano /etc/default/grub +# Add: GRUB_CMDLINE_LINUX="default_hugepagesz=1G hugepagesz=1G hugepages=2" + +sudo update-grub +sudo reboot +``` + +### 3. Bind NICs to DPDK + +Identify your network interfaces: + +```bash +# Show available network interfaces +dpdk-devbind.py --status + +# Example output: +# Network devices using kernel driver +# =================================== +# 0000:00:08.0 'Ethernet Controller 10G X550T' if=eth1 drv=ixgbe unused=vfio-pci +# 0000:00:09.0 'Ethernet Controller 10G X550T' if=eth2 drv=ixgbe unused=vfio-pci +``` + +Bind interfaces to DPDK driver: + +```bash +# Load VFIO driver (recommended) +sudo modprobe vfio-pci + +# Bind NICs to DPDK +sudo dpdk-devbind.py --bind=vfio-pci 0000:00:08.0 +sudo dpdk-devbind.py --bind=vfio-pci 0000:00:09.0 + +# Verify binding +dpdk-devbind.py --status +``` + +**Note**: Binding NICs to DPDK removes them from the kernel network stack. Ensure you don't bind your management interface! + +### 4. Set IOMMU (for VFIO) + +For better performance and security with VFIO: + +```bash +# Edit GRUB configuration +sudo nano /etc/default/grub + +# Add or modify: GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt" +# (Use 'amd_iommu=on' for AMD CPUs) + +sudo update-grub +sudo reboot +``` + +## Building + +Build the sample application: + +```bash +cd sample/Magma.DPDK.PacketForwarder +dotnet build -c Release +``` + +Or build from repository root: + +```bash +dotnet build -c Release +``` + +## Running + +### Basic Usage + +```bash +sudo dotnet run --project sample/Magma.DPDK.PacketForwarder \ + -- 0000:00:08.0 0000:00:09.0 +``` + +### With Docker (Recommended for Testing) + +See [Docker Setup](#docker-setup) section below. + +### Expected Output + +``` +Magma DPDK Packet Forwarder +=========================== + +Forwarding configuration: + Port 0: 0000:00:08.0 + Port 1: 0000:00:09.0 + +Note: Full DPDK transport implementation required (issue #128) +Press Ctrl+C to exit + +Starting packet forwarding... +Forwarding loop started (placeholder - requires DPDK transport) + +[14:30:45] Statistics: + 0000:00:08.0 -> 0000:00:09.0: 12,450,000 packets (2,490,000.00 pps, 29,880.00 Mbps) + 0000:00:09.0 -> 0000:00:08.0: 12,450,000 packets (2,490,000.00 pps, 29,880.00 Mbps) +``` + +## Docker Setup + +### Using Docker Compose (Recommended) + +The repository includes a Docker Compose setup for testing with virtual interfaces: + +```bash +cd sample/Magma.DPDK.PacketForwarder +docker-compose up --build +``` + +This creates: +- DPDK-enabled container with huge pages +- Virtual Ethernet pair (veth) or null PMD for testing +- Proper privilege settings and device access + +### Manual Docker Run + +```bash +docker build -t magma-dpdk-forwarder . + +docker run -it --rm \ + --privileged \ + --network host \ + -v /dev/hugepages:/dev/hugepages \ + -v /sys/bus/pci:/sys/bus/pci \ + -v /sys/devices:/sys/devices \ + magma-dpdk-forwarder \ + 0000:00:08.0 0000:00:09.0 +``` + +## Testing + +### Test with Traffic Generator + +Use DPDK's `pktgen` to generate test traffic: + +```bash +# Terminal 1: Start forwarder +sudo dotnet run --project sample/Magma.DPDK.PacketForwarder \ + -- 0000:00:08.0 0000:00:09.0 + +# Terminal 2: Generate traffic +sudo pktgen -l 0-3 -n 4 -- -P -m "[2:3].0" -T +# Then in pktgen console: +# set 0 size 64 +# set 0 rate 100 +# start 0 +``` + +### Test with Virtual Devices (Null PMD) + +For development/testing without physical NICs: + +```bash +# Run with null PMD devices +sudo dotnet run --project sample/Magma.DPDK.PacketForwarder \ + -- --vdev=net_null0 --vdev=net_null1 \ + -- net_null0 net_null1 +``` + +## Configuration + +### Environment Variables + +- `RTE_SDK`: Path to DPDK installation (usually auto-detected) +- `RTE_TARGET`: DPDK target (usually `x86_64-native-linux-gcc`) + +### Performance Tuning + +For optimal performance: + +1. **CPU Isolation**: Isolate cores for DPDK + ```bash + # Add to kernel parameters: isolcpus=2,3,4,5 + ``` + +2. **CPU Governor**: Set to performance mode + ```bash + sudo cpupower frequency-set -g performance + ``` + +3. **IRQ Affinity**: Pin interrupts away from DPDK cores + ```bash + # See /proc/interrupts and use /proc/irq/*/smp_affinity + ``` + +4. **NUMA**: Run on single NUMA node with NICs + ```bash + numactl --cpunodebind=0 --membind=0 dotnet run ... + ``` + +## Architecture + +### Forwarding Logic + +The application implements a simple L2 forwarding pattern: + +1. **Initialize**: Setup DPDK EAL, configure ports, allocate memory pools +2. **RX Loop**: Receive packet bursts from port 0 +3. **Forward**: Send received packets to port 1 +4. **RX Loop**: Receive packet bursts from port 1 +5. **Forward**: Send received packets to port 0 +6. **Repeat**: Continue until termination signal + +### Zero-Copy Design + +The forwarder uses DPDK's zero-copy packet handling: +- Direct DMA to/from NIC buffers +- No kernel network stack overhead +- Minimal CPU cache impact + +## Troubleshooting + +### Huge Pages Not Allocated + +```bash +# Check current allocation +cat /proc/meminfo | grep Huge + +# Force allocation +echo 512 | sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +``` + +### NICs Not Visible + +```bash +# Check PCI devices +lspci | grep Ethernet + +# Check DPDK binding +dpdk-devbind.py --status + +# Verify driver is loaded +lsmod | grep vfio +``` + +### Permission Denied + +Ensure running with `sudo` or proper capabilities: + +```bash +# Grant capabilities (not recommended for production) +sudo setcap cap_net_admin,cap_sys_admin,cap_dac_override+ep \ + /path/to/dotnet +``` + +### VFIO Errors + +```bash +# Enable IOMMU in BIOS/UEFI +# Add kernel parameter: intel_iommu=on iommu=pt + +# Check IOMMU groups +find /sys/kernel/iommu_groups/ -type l +``` + +## Current Status + +**Note**: This sample currently serves as a reference implementation. Full DPDK transport integration is tracked in issue #128. + +The current implementation demonstrates: +- Project structure and configuration +- L2 forwarding logic patterns +- Statistics collection and reporting +- Command-line interface design + +Once the DPDK transport is available, this sample will be updated to use the actual DPDK packet I/O APIs. + +## Performance Expectations + +With proper configuration, this forwarder can achieve: +- **Throughput**: Up to 100 Gbps (line rate on 100G NICs) +- **Latency**: < 1 μs forwarding latency +- **Packet Rate**: Up to 148 Mpps (64-byte packets on 100G) + +Actual performance depends on: +- NIC capabilities +- CPU performance +- Memory bandwidth +- Packet size distribution + +## References + +- [DPDK Documentation](https://doc.dpdk.org/) +- [DPDK Getting Started Guide](https://doc.dpdk.org/guides/linux_gsg/) +- [Magma DPDK Integration (Issue #128)](https://github.com/ProjectMagma/Magma/issues/128) +- [DPDK L2 Forwarding Example](https://doc.dpdk.org/guides/sample_app_ug/l2_forward_real_virtual.html) + +## License + +This project is licensed under the MIT License - see the [LICENSE](../../LICENSE) file for details.