[meshnet] Manage system-level kernel and link tunables for packet performance

cloudbuild/external.pkr.hcl

@@ -123,6 +123,7 @@ build {
       "sudo modprobe br_netfilter",
       "echo \"1\" > sudo tee /proc/sys/net/bridge/bridge-nf-call-iptables",
       "echo \"1\" > sudo tee /proc/sys/net/ipv4/ip_forward",
+      "echo 'net.core.netdev_max_backlog = 10000\nnet.core.rmem_max = 16777216\nnet.core.wmem_max = 16777216\nnet.core.rmem_default = 16777216\nnet.core.wmem_default = 16777216' | sudo tee /etc/sysctl.d/99-kne.conf",
       "sudo sysctl --system",
       "sudo sysctl -p",
       "sudo mkdir -p /etc/containerd",

cloudbuild/internal.pkr.hcl

@@ -127,6 +127,7 @@ build {
       "sudo modprobe br_netfilter",
       "echo \"1\" > sudo tee /proc/sys/net/bridge/bridge-nf-call-iptables",
       "echo \"1\" > sudo tee /proc/sys/net/ipv4/ip_forward",
+      "echo 'net.core.netdev_max_backlog = 10000\nnet.core.rmem_max = 16777216\nnet.core.wmem_max = 16777216\nnet.core.rmem_default = 16777216\nnet.core.wmem_default = 16777216' | sudo tee /etc/sysctl.d/99-kne.conf",
       "sudo sysctl --system",
       "sudo sysctl -p",
       "sudo mkdir -p /etc/containerd",

docs/README.md

@@ -18,6 +18,8 @@ guide is broken up into multiple sections spanning multiple documents.
   a KNE topology after creation.
 - [Troubleshooting](troubleshoot.md): A troubleshooting guide if anything goes
   wrong along the way.
+- [Performance Tuning](performance_tuning.md): Recommended host kernel sysctls,
+  interface queue lengths, and gRPC flow control for high-density topologies.
 
 They are recommended to be done in order.
 

docs/performance_tuning.md

@@ -0,0 +1,106 @@
+# Performance and System Tuning in KNE
+
+This document details host kernel tunables, interface queue configurations, and
+gRPC overlay settings recommended for running high-density emulated topologies
+and bursty control-plane traffic (such as large BGP table exchanges and
+high-throughput IS-IS meshes) in KNE.
+
+---
+
+## 1. Host Kernel Sysctl Tunables
+
+When running large-scale or multi-vendor network topologies in KNE, default Linux
+networking buffers and device backlog limits can lead to silent packet dropouts
+under sudden traffic bursts.
+
+### Recommended Sysctl Configuration
+
+Create `/etc/sysctl.d/99-kne.conf` on K8s worker nodes:
+
+```ini
+# Increase maximum network device input queue backlog for bursty packet creation
+net.core.netdev_max_backlog = 10000
+
+# Increase maximum OS socket receive and send buffer sizes to 16 MB
+net.core.rmem_max = 16777216
+net.core.wmem_max = 16777216
+
+# Increase default OS socket receive and send buffer sizes to 16 MB
+net.core.rmem_default = 16777216
+net.core.wmem_default = 16777216
+```
+
+Apply the configuration immediately:
+
+```bash
+sudo sysctl --system
+```
+
+### Why These Tunables Matter
+
+- **`net.core.netdev_max_backlog = 10000`**: The default Linux backlog queue
+  (`1000`) can fill up rapidly when hundreds of virtual interfaces
+  simultaneously receive bursty control-plane frames (e.g., initial link-state
+  advertisements or topology convergence events). Increasing the backlog queue
+  prevents kernel-level packet drops before frames reach container sockets.
+- **`net.core.rmem_max` / `wmem_max` / `rmem_default` / `wmem_default = 16777216` (16 MB)**:
+  Default Linux socket buffers (typically ~212 KB) are insufficient for large
+  BGP updates, routing snapshots, or high-volume telemetry streams across
+  emulated nodes. Providing a 16 MB ceiling and default allows socket buffer
+  allocations (such as `SO_RCVBUF` / `SO_SNDBUF` of 4 MB used by high-performance
+  network bridges and routers) to allocate sufficient buffer memory without
+  kernel truncation or dropouts.
+
+> **Note**: These sysctl settings are pre-baked into KNE GCE VM images built via
+> CloudBuild/Packer (`cloudbuild/internal.pkr.hcl` and
+> `cloudbuild/external.pkr.hcl`). For custom Kubernetes clusters or bare-metal
+> setups, apply `/etc/sysctl.d/99-kne.conf` manually on each node.
+
+---
+
+## 2. Link Interface Queue Length (`txqueuelen`)
+
+Virtual Ethernet (`vEth`), TAP, and vxLAN links created by Meshnet carry full
+line-rate inter-container traffic. The default Linux `txqueuelen` for virtual
+interfaces is often small (`1000` or `0`), which can drop packets when container
+workloads burst faster than context switching can drain the virtual device.
+
+- Meshnet configures `txqueuelen = 10000` on created TAP, vEth, and vxLAN
+  interfaces.
+- You can override the default queue length via the `LINK_TXQUEUELEN` environment
+  variable on the `meshnet` daemon pod if desired:
+
+```yaml
+env:
+  - name: LINK_TXQUEUELEN
+    value: "10000"
+```
+
+---
+
+## 3. gRPC Overlay Stream & Connection Flow Control
+
+When using gRPC wire overlay (`INTER_NODE_LINK_TYPE: "GRPC"`) for cross-node mesh
+tunneling, Meshnet communicates via multiplexed bidirectional gRPC streams.
+
+To avoid HTTP/2 stream-level flow control stalls and handle bursty control-plane
+traffic:
+
+- **Stream Initial Window Size (`InitialWindowSize`)**: `4 MB` (overrides
+  standard HTTP/2 64 KB window to allow large packet bursts per stream).
+- **Connection Initial Window Size (`InitialConnWindowSize`)**: `16 MB`
+  (overrides default connection window to provide multiplexed stream headroom).
+- **Max Message Size (`MaxRecvMsgSize` / `MaxSendMsgSize`)**: `64 MB` (ensures
+  frames, Jumbo packets, and batch RPCs are never rejected).
+
+---
+
+## 4. MTU and Jumbo Frames
+
+KNE and Meshnet support standard MTU (`1500`) as well as Jumbo frames (`9216`
+bytes) for IS-IS LSPs, BGP Jumbo frames, and encapsulated overlay traffic:
+
+- Interface MTU can be specified per-link in the KNE topology definition (under
+  `interfaces.mtu`).
+- Meshnet internal packet buffers allocate up to `65535` bytes, fully preventing
+  frame truncation for Jumbo frames.

third_party/meshnet/daemon/vxlan/vxlan.go

@@ -13,6 +13,7 @@ import (
 	"github.com/vishvananda/netlink"
 
 	mpb "github.com/openconfig/kne/third_party/meshnet/daemon/proto/meshnet/v1beta1"
+	"github.com/openconfig/kne/third_party/meshnet/utils/wireutil"
 )
 
 var vxLanOvrlyLogger *log.Entry = nil
@@ -107,6 +108,20 @@ func CreateOrUpdate(v *mpb.RemotePod) error {
 		}
 	}
 
+	// Tune txqueuelen inside the container netns (configurable via LINK_TXQUEUELEN)
+	if podNs, err := ns.GetNS(veth.NsName); err == nil {
+		_ = podNs.Do(func(_ ns.NetNS) error {
+			if link, err := netlink.LinkByName(veth.LinkName); err == nil {
+				txqLen := wireutil.GetLinkTxQLen()
+				if err := netlink.LinkSetTxQLen(link, txqLen); err != nil {
+					vxLanOvrlyLogger.Warnf("failed to set txqueuelen %d on %s inside %s: %v", txqLen, veth.LinkName, veth.NsName, err)
+				}
+			}
+			return nil
+		})
+		podNs.Close()
+	}
+
 	return nil
 }
 

third_party/meshnet/utils/wireutil/sys_tune.go

@@ -14,3 +14,9 @@ func GetEnvInt(key string, defaultVal int) int {
 	}
 	return defaultVal
 }
+
+// GetLinkTxQLen returns the configured link txqueuelen (default 10000, configurable via LINK_TXQUEUELEN).
+func GetLinkTxQLen() int {
+	return GetEnvInt("LINK_TXQUEUELEN", 10000)
+}
+

third_party/meshnet/utils/wireutil/tap.go

@@ -62,6 +62,12 @@ func CreateOrAttachTAP(podNsPath string, ifName string, ipCIDR string) (*os.File
 			return fmt.Errorf("failed to find link %s inside netns %s: %w", ifName, podNsPath, err)
 		}
 
+		// Increase txqueuelen for high-throughput packet processing (configurable via LINK_TXQUEUELEN)
+		txqLen := GetLinkTxQLen()
+		if err := netlink.LinkSetTxQLen(link, txqLen); err != nil {
+			log.Warnf("CreateOrAttachTAP: failed to set txqueuelen %d on %s in netns %s: %v", txqLen, ifName, podNsPath, err)
+		}
+
 		if err := netlink.LinkSetUp(link); err != nil {
 			unix.Close(fd)
 			return fmt.Errorf("failed to set %s UP in netns %s: %w", ifName, podNsPath, err)

third_party/meshnet/utils/wireutil/veth.go

@@ -173,6 +173,12 @@ func ConfigurePodLinks(podNsPath string, links []PodLinkConfig) error {
 				}
 			}
 
+			// Increase txqueuelen for high-throughput packet processing (configurable via LINK_TXQUEUELEN)
+			txqLen := GetLinkTxQLen()
+			if err := netlink.LinkSetTxQLen(link, txqLen); err != nil {
+				log.Warnf("ConfigurePodLinks: failed to set txqueuelen %d on %s inside %s: %v", txqLen, cfg.LocalIntf, podNsPath, err)
+			}
+
 			if err := netlink.LinkSetUp(link); err != nil {
 				return fmt.Errorf("failed to set %s UP inside %s: %w", cfg.LocalIntf, podNsPath, err)
 			}