feat: add always-on performance instrumentation (Phase 1)

.github/workflows/integration-tests.yml

@@ -237,16 +237,32 @@ jobs:
           done
           echo "" >> "$REPORT"
 
-          # Application logs (most useful for debugging test failures)
-          echo "<details><summary>Application Logs</summary>" >> "$REPORT"
+          # Application logs — last 20 lines shown inline for quick context,
+          # full log (last 200 lines) in collapsible section.
+          echo "### Application Logs (last 20 lines)" >> "$REPORT"
           echo "" >> "$REPORT"
           for log in instance-logs/*-echo-server.log instance-logs/*-test-client.log instance-logs/*-test-client-iperf.log instance-logs/*-iperf3-server.log; do
             [ -f "$log" ] || continue
             LOGSIZE=$(wc -c < "$log")
             [ "$LOGSIZE" -gt 5 ] || continue  # skip near-empty files
+            LOGNAME=$(basename "$log")
+            echo "**${LOGNAME}**" >> "$REPORT"
+            echo '```' >> "$REPORT"
+            tail -20 "$log" >> "$REPORT"
+            echo '```' >> "$REPORT"
+          done
+          echo "" >> "$REPORT"
+
+          # Full application logs in collapsible section
+          echo "<details><summary>Full Application Logs (last 200 lines each)</summary>" >> "$REPORT"
+          echo "" >> "$REPORT"
+          for log in instance-logs/*-echo-server.log instance-logs/*-test-client.log instance-logs/*-test-client-iperf.log instance-logs/*-iperf3-server.log; do
+            [ -f "$log" ] || continue
+            LOGSIZE=$(wc -c < "$log")
+            [ "$LOGSIZE" -gt 5 ] || continue
             echo "#### $(basename "$log")" >> "$REPORT"
             echo '```' >> "$REPORT"
-            tail -40 "$log" >> "$REPORT"
+            tail -200 "$log" >> "$REPORT"
             echo '```' >> "$REPORT"
           done
           echo "</details>" >> "$REPORT"

.github/workflows/perf-tests.yml

@@ -193,16 +193,32 @@ jobs:
             echo "" >> "$REPORT"
           done
 
-          # Application logs (most useful for debugging — TRex server + DUT app logs)
-          echo "<details><summary>Application Logs</summary>" >> "$REPORT"
+          # Application logs — last 20 lines shown inline for quick context,
+          # full log (last 200 lines) in collapsible section.
+          echo "### Application Logs (last 20 lines)" >> "$REPORT"
+          echo "" >> "$REPORT"
+          for log in instance-logs/*-trex-server.log instance-logs/dut-*-app.log instance-logs/*-echo-*.log instance-logs/*-testpmd.log instance-logs/*-plain-echo.log; do
+            [ -f "$log" ] || continue
+            LOGSIZE=$(wc -c < "$log" 2>/dev/null || echo 0)
+            [ "$LOGSIZE" -gt 5 ] || continue
+            LOGNAME=$(basename "$log")
+            echo "**${LOGNAME}**" >> "$REPORT"
+            echo '```' >> "$REPORT"
+            tail -20 "$log" >> "$REPORT"
+            echo '```' >> "$REPORT"
+          done
+          echo "" >> "$REPORT"
+
+          # Full application logs in collapsible section
+          echo "<details><summary>Full Application Logs (last 200 lines each)</summary>" >> "$REPORT"
           echo "" >> "$REPORT"
           for log in instance-logs/*-trex-server.log instance-logs/dut-*-app.log instance-logs/*-echo-*.log instance-logs/*-testpmd.log instance-logs/*-plain-echo.log; do
             [ -f "$log" ] || continue
             LOGSIZE=$(wc -c < "$log" 2>/dev/null || echo 0)
             [ "$LOGSIZE" -gt 5 ] || continue
             echo "#### $(basename "$log")" >> "$REPORT"
             echo '```' >> "$REPORT"
-            tail -80 "$log" >> "$REPORT"
+            tail -200 "$log" >> "$REPORT"
             echo '```' >> "$REPORT"
           done
           echo "</details>" >> "$REPORT"

.kiro/specs/performance-optimization/tasks.md

@@ -2,14 +2,14 @@
 
 ## Phase 1: Instrumentation (Visibility First)
 
-- [ ] **P1.1**: Implement `PerfCounters` struct in `dpdk-udp/src/perf.rs` — all `AtomicU64` fields, cache-line aligned, `new()`, `snapshot()`, `reset_interval()` methods
-- [ ] **P1.2**: Implement `LatencySampler` in `dpdk-udp/src/perf.rs` — fixed-size ring buffer, configurable sample rate (default 1:1000), `record(duration_ns)`, `percentiles() -> (p50, p95, p99, p99.9, max)`
-- [ ] **P1.3**: Implement `PerfReporter` background thread — reads counters + sampler every N seconds, computes rates by diffing snapshots, emits structured key=value log line to stderr
-- [ ] **P1.4**: Wire counters into `UdpSocket` — add `Arc<PerfCounters>` field, increment on send/recv/drop/arp/icmp paths, add `perf_counters()`, `enable_perf_reporting()`, `perf_snapshot()` API methods
-- [ ] **P1.5**: Wire counters into multi-core topology — increment `rx_drops_ring_full`, `worker_idle_polls`, `worker_packets_processed`, ring enqueue failures in `rx_loop` and `worker_loop`
-- [ ] **P1.6**: Wire latency sampling — timestamp at `rx_burst` return, timestamp at `recv_from()` return, record delta on sampled packets
-- [ ] **P1.7**: Add `--perf-interval <seconds>` flag to echo app — enables `enable_perf_reporting()` at startup, default 10s
-- [ ] **P1.8**: Unit tests for `PerfCounters` (concurrent increment + snapshot), `LatencySampler` (percentile accuracy), `PerfReporter` (output format)
+- [x] **P1.1**: Implement `PerfCounters` struct in `dpdk-udp/src/perf.rs` — all `AtomicU64` fields, cache-line aligned, `new()`, `snapshot()`, `reset_interval()` methods
+- [x] **P1.2**: Implement `LatencySampler` in `dpdk-udp/src/perf.rs` — fixed-size ring buffer, configurable sample rate (default 1:1000), `record(duration_ns)`, `percentiles() -> (p50, p95, p99, p99.9, max)`
+- [x] **P1.3**: Implement `PerfReporter` background thread — reads counters + sampler every N seconds, computes rates by diffing snapshots, emits structured key=value log line to stderr
+- [x] **P1.4**: Wire counters into `UdpSocket` — add `Arc<PerfCounters>` field, increment on send/recv/drop/arp/icmp paths, add `perf_counters()`, `enable_perf_reporting()`, `perf_snapshot()` API methods
+- [x] **P1.5**: Wire counters into multi-core topology — increment `rx_drops_ring_full`, `worker_idle_polls`, `worker_packets_processed`, ring enqueue failures in `rx_loop` and `worker_loop`
+- [x] **P1.6**: Wire latency sampling — timestamp at `rx_burst` return, timestamp at `recv_from()` return, record delta on sampled packets
+- [x] **P1.7**: Add `--perf-interval <seconds>` flag to echo app — enables `enable_perf_reporting()` at startup, default 10s
+- [x] **P1.8**: Unit tests for `PerfCounters` (concurrent increment + snapshot), `LatencySampler` (percentile accuracy), `PerfReporter` (output format)
 - [ ] **P1.9**: Run perf benchmark with instrumentation enabled — verify < 1% throughput regression at 350K PPS vs uninstrumented baseline
 
 ## Phase 2: Quick Wins (Low-Risk, High-Impact)

CLAUDE.md

@@ -2,6 +2,61 @@
 
 @AGENTS.md
 
+## Development Loop
+
+**Every code change MUST follow this loop. Do not skip steps.**
+
+```
+  +------------------+
+  |  1. Write Code   |<-----------------------------------------+
+  +--------+---------+                                           |
+           |                                                     |
+           v                                                     |
+  +------------------+                                           |
+  |  2. Unit Tests   |   cargo build && cargo test               |
+  +--------+---------+                                           |
+           |                                                     |
+       pass|   fail --> fix code --------------------------------+
+           v                                                     |
+  +------------------+                                           |
+  |  3. Push / PR    |   git push or gh pr create                |
+  +--------+---------+                                           |
+           |                                                     |
+           v                                                     |
+  +------------------+                                           |
+  | 4. Integration   |   (auto-triggered on PR, or               |
+  |    Tests         |    ./scripts/ci-validate.sh)              |
+  +--------+---------+                                           |
+           |                                                     |
+       pass|   fail --> read logs, fix code ---------------------+
+           v                                                     |
+  +------------------+                                           |
+  | 5. Performance   |   gh workflow run perf-tests.yml          |
+  |    Tests         |   poll with gh run view --json            |
+  +--------+---------+                                           |
+           |                                                     |
+       pass|   fail --> read PR comments, fix code -------------+
+           v
+  +------------------+
+  | 6. Success!      |   Ask user to review PR
+  +------------------+
+```
+
+**Step details:**
+
+1. **Write code** — read files before modifying, follow patterns in AGENTS.md
+2. **Unit tests** — `cargo build && cargo test` locally. If they fail, fix and re-run. Do NOT proceed with failures.
+3. **Push / PR** — push to the feature branch. Create a PR if one doesn't exist yet, otherwise push a new commit.
+4. **Integration tests** — triggered automatically on PR, or manually via `./scripts/ci-validate.sh`. Poll with `gh run view --json status,conclusion`. If they fail, read the PR comments and instance logs to diagnose. Fix the code and go back to step 1.
+5. **Performance tests** — trigger with `gh workflow run perf-tests.yml`. Poll until complete. Read the PR comments for benchmark results and app logs. If they fail or regress, fix and go back to step 1.
+6. **Success** — all tests pass. Ask the user to review the PR.
+
+**Key rules:**
+- Never skip straight to PR without passing local tests
+- Never assume CI will catch what local tests missed
+- On failure, read the actual logs — do not guess
+- Loop back to step 1 on any failure, do not try to patch forward
+
 ## Claude Code (Hooks & Skills)
 
 ### Querying CI / GitHub Actions Results

apps/echo/src/main.rs

@@ -17,6 +17,7 @@ trait UdpSocketTrait {
     fn send_to(&self, buf: &[u8], addr: SocketAddr) -> io::Result<usize>;
     fn local_addr(&self) -> io::Result<SocketAddr>;
     fn set_read_timeout(&self, dur: Option<Duration>) -> io::Result<()>;
+    fn enable_perf_reporting(&self, _interval: Duration) -> io::Result<()> { Ok(()) }
 }
 
 // Implement trait for std::net::UdpSocket
@@ -56,6 +57,10 @@ impl UdpSocketTrait for dpdk_udp::UdpSocket {
     fn set_read_timeout(&self, dur: Option<Duration>) -> io::Result<()> {
         self.set_read_timeout(dur)
     }
+
+    fn enable_perf_reporting(&self, interval: Duration) -> io::Result<()> {
+        self.enable_perf_reporting(interval)
+    }
 }
 
 /// Try DPDK first, then fall back to standard networking.
@@ -130,6 +135,11 @@ struct Args {
     #[arg(long, default_value_t = 0)]
     rx_queues: u16,
 
+    /// Performance reporting interval in seconds (0 = disabled).
+    /// When set, emits structured [PERF] log lines to stderr every N seconds.
+    #[arg(long, default_value_t = 0)]
+    perf_interval: u64,
+
     /// Use synthetic packet mode (for protocol testing - developer option)
     #[arg(long, hide = true)]
     synthetic: bool,
@@ -156,6 +166,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         println!("Binding to {}", bind_addr);
 
         let socket = bind_socket(&bind_addr, args.workers, args.rx_queues)?;
+        if args.perf_interval > 0 {
+            socket.enable_perf_reporting(Duration::from_secs(args.perf_interval))?;
+            println!("Performance reporting enabled (interval: {}s)", args.perf_interval);
+        }
         run_echo_server(socket)?;
     }
 

dpdk-udp/Cargo.toml

@@ -4,6 +4,13 @@ version = "0.1.0"
 edition = "2021"
 description = "UDP protocol implementation with DPDK acceleration and raw socket fallback"
 
+[features]
+default = ["perf-counters"]
+## Enable hot-path performance counters (atomic increments on every packet).
+## Disable with `--no-default-features` to eliminate all instrumentation overhead
+## from the TX/RX fast paths — useful for latency-critical production deployments.
+perf-counters = []
+
 [dependencies]
 thiserror = { workspace = true }
 dpdk = { path = "../dpdk" }