rewrite sock_op program with libbpf 1.0+ (eunomia-bpf#84)

Author: SirHao

src/29-sockops/.gitignore

@@ -1,8 +1,2 @@
-.vscode
-package.json
-*.o
-*.skel.json
-*.skel.yaml
-package.yaml
-ecli
-ecc
+.output
+.vscode

src/29-sockops/Makefile

@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+OUTPUT := .output
+CLANG ?= clang
+LIBBPF_SRC := $(abspath ../third_party/libbpf/src)
+BPFTOOL_SRC := $(abspath ../third_party/bpftool/src)
+LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
+BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
+BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
+LIBBLAZESYM_SRC := $(abspath ../third_party/blazesym/)
+LIBBLAZESYM_OBJ := $(abspath $(OUTPUT)/libblazesym.a)
+LIBBLAZESYM_HEADER := $(abspath $(OUTPUT)/blazesym.h)
+ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
+			 | sed 's/arm.*/arm/' \
+			 | sed 's/aarch64/arm64/' \
+			 | sed 's/ppc64le/powerpc/' \
+			 | sed 's/mips.*/mips/' \
+			 | sed 's/riscv64/riscv/' \
+			 | sed 's/loongarch64/loongarch/')
+VMLINUX := ../third_party/vmlinux/$(ARCH)/vmlinux.h
+# Use our own libbpf API headers and Linux UAPI headers distributed with
+# libbpf to avoid dependency on system-wide headers, which could be missing or
+# outdated
+INCLUDES := -I$(OUTPUT) -I../third_party/libbpf/include/uapi -I$(dir $(VMLINUX))
+CFLAGS := -g -Wall
+ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
+
+APPS = bpf_contrack bpf_redirect# minimal minimal_legacy uprobe kprobe fentry usdt sockfilter tc ksyscall
+
+CARGO ?= $(shell which cargo)
+ifeq ($(strip $(CARGO)),)
+BZS_APPS :=
+else
+BZS_APPS := # profile
+APPS += $(BZS_APPS)
+# Required by libblazesym
+ALL_LDFLAGS += -lrt -ldl -lpthread -lm
+endif
+
+# Get Clang's default includes on this system. We'll explicitly add these dirs
+# to the includes list when compiling with `-target bpf` because otherwise some
+# architecture-specific dirs will be "missing" on some architectures/distros -
+# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h,
+# sys/cdefs.h etc. might be missing.
+#
+# Use '-idirafter': Don't interfere with include mechanics except where the
+# build would have failed anyways.
+CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - </dev/null 2>&1 \
+	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
+
+ifeq ($(V),1)
+	Q =
+	msg =
+else
+	Q = @
+	msg = @printf '  %-8s %s%s\n'					\
+		      "$(1)"						\
+		      "$(patsubst $(abspath $(OUTPUT))/%,%,$(2))"	\
+		      "$(if $(3), $(3))";
+	MAKEFLAGS += --no-print-directory
+endif
+
+define allow-override
+  $(if $(or $(findstring environment,$(origin $(1))),\
+            $(findstring command line,$(origin $(1)))),,\
+    $(eval $(1) = $(2)))
+endef
+
+$(call allow-override,CC,$(CROSS_COMPILE)cc)
+$(call allow-override,LD,$(CROSS_COMPILE)ld)
+
+.PHONY: all
+all: bpf_redirect.bpf.o bpf_contrack.bpf.o
+
+.PHONY: clean 
+
+
+clean:
+	$(call msg,CLEAN)
+	$(Q)rm -rf $(OUTPUT) *.bpf.o
+
+$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
+	$(call msg,MKDIR,$@)
+	$(Q)mkdir -p $@
+
+# Build libbpf
+$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
+	$(call msg,LIB,$@)
+	$(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1		      \
+		    OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@)		      \
+		    INCLUDEDIR= LIBDIR= UAPIDIR=			      \
+		    install
+
+# Build bpftool
+$(BPFTOOL): | $(BPFTOOL_OUTPUT)
+	$(call msg,BPFTOOL,$@)
+	$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap
+
+
+# Build BPF code
+%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT) $(BPFTOOL)
+	$(call msg,BPF,$@)
+	$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH)		      \
+		     $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES)		      \
+		     -c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+	$(Q)$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+
+
+

src/29-sockops/README.md

@@ -17,9 +17,8 @@ Merbridge 项目就是这样实现了用 eBPF 代替 iptables 为 Istio 进行
 ### 编译 eBPF 程序
 
 ```shell
-# Compile the bpf_sockops program
-clang -O2 -g  -Wall -target bpf  -c bpf_sockops.c -o bpf_sockops.o
-clang -O2 -g  -Wall -target bpf  -c bpf_redir.c -o bpf_redir.o
+# Compile the bpf program with libbpf
+make
 ```
 
 ### 加载 eBPF 程序
@@ -43,26 +42,45 @@ $ sudo bpftool prog show
 ### 运行 [iperf3](https://iperf.fr/) 服务器
 
 ```shell
-iperf3 -s -p 10000
+iperf3 -s -p 5001
 ```
 
 ### 运行 [iperf3](https://iperf.fr/) 客户端
 
 ```shell
-iperf3 -c 127.0.0.1 -t 10 -l 64k -p 10000
+iperf3 -c 127.0.0.1 -t 10 -l 64k -p 5001
 ```
 
 ### 收集追踪
 
+查看``sock_ops``追踪本地连接建立
 ```console
-$ ./trace.sh
+$ ./trace_bpf_output.sh
 iperf3-9516  [001] .... 22500.634108: 0: <<< ipv4 op = 4, port 18583 --> 4135
 iperf3-9516  [001] ..s1 22500.634137: 0: <<< ipv4 op = 5, port 4135 --> 18583
 iperf3-9516  [001] .... 22500.634523: 0: <<< ipv4 op = 4, port 19095 --> 4135
 iperf3-9516  [001] ..s1 22500.634536: 0: <<< ipv4 op = 5, port 4135 --> 19095
 ```
 
-你应该可以看到 4 个用于套接字建立的事件。如果你没有看到任何事件，那么 eBPF 程序可能没有正确地附加上。
+当iperf3 -c建立连接后，你应该可以看到上述用于套接字建立的事件。如果你没有看到任何事件，那么 eBPF 程序可能没有正确地附加上。
+
+此外，当``sk_msg``生效后，可以发现当使用tcpdump捕捉本地lo设备流量时，只能捕获三次握手和四次挥手流量，而iperf数据流量没有被捕获到。如果捕获到iperf数据流量，那么 eBPF 程序可能没有正确地附加上。
+
+
+```console
+$ ./trace_lo_traffic.sh
+# 三次握手
+13:24:07.181804 IP localhost.46506 > localhost.5001: Flags [S], seq 620239881, win 65495, options [mss 65495,sackOK,TS val 1982813394 ecr 0,nop,wscale 7], length 0
+13:24:07.181815 IP localhost.5001 > localhost.46506: Flags [S.], seq 1084484879, ack 620239882, win 65483, options [mss 65495,sackOK,TS val 1982813394 ecr 1982813394,nop,wscale 7], length 0
+13:24:07.181832 IP localhost.46506 > localhost.5001: Flags [.], ack 1, win 512, options [nop,nop,TS val 1982813394 ecr 1982813394], length 0
+
+# 四次挥手
+13:24:12.475649 IP localhost.46506 > localhost.5001: Flags [F.], seq 1, ack 1, win 512, options [nop,nop,TS val 1982818688 ecr 1982813394], length 0
+13:24:12.479621 IP localhost.5001 > localhost.46506: Flags [.], ack 2, win 512, options [nop,nop,TS val 1982818692 ecr 1982818688], length 0
+13:24:12.481265 IP localhost.5001 > localhost.46506: Flags [F.], seq 1, ack 2, win 512, options [nop,nop,TS val 1982818694 ecr 1982818688], length 0
+13:24:12.481270 IP localhost.46506 > localhost.5001: Flags [.], ack 2, win 512, options [nop,nop,TS val 1982818694 ecr 1982818694], length 0
+
+```
 
 ### 卸载 eBPF 程序
 

src/29-sockops/README_en.md

@@ -18,8 +18,7 @@ This example program redirects traffic from the sender's socket (outbound) to th
 
 ```shell
 # Compile the bpf_sockops program
-clang -O2 -g -Wall -target bpf -c bpf_sockops.c -o bpf_sockops.o
-clang -O2 -g -Wall -target bpf -c bpf_redir.c -o bpf_redir.o
+make
 ```
 
 ### Loading the eBPF Program
@@ -43,26 +42,44 @@ $ sudo bpftool prog show
 ### Running the [iperf3](https://iperf.fr/) Server
 
 ```shell
-iperf3 -s -p 10000
+iperf3 -s -p 5001
 ```
 
 ### Running the [iperf3](https://iperf.fr/) Client
 
 ```shell
-iperf3 -c 127.0.0.1 -t 10 -l 64k -p 10000
+iperf3 -c 127.0.0.1 -t 10 -l 64k -p 5001
 ```
 
 ### Collecting Traces
 
+Show connection setup tracing for localhost connection. 
 ```console
-$ ./trace.sh
+$ ./trace_bpf_output.sh
 iperf3-9516  [001] .... 22500.634108: 0: <<< ipv4 op = 4, port 18583 --> 4135
 iperf3-9516  [001] ..s1 22500.634137: 0: <<< ipv4 op = 5, port 4135 --> 18583
 iperf3-9516  [001] .... 22500.634523: 0: <<< ipv4 op = 4, port 19095 --> 4135
 iperf3-9516  [001] ..s1 22500.634536: 0: <<< ipv4 op = 5, port 4135 --> 19095
 ```
 
-You should be able to see 4 events for socket establishment. If you don't see any events, the eBPF program may not have been attached correctly.
+When ``iperf3 -c`` creates a connection, you should see the above events for socket setup. If you do not see any events, then the eBPF program may not have been properly attached.
+
+In addition, when ``sk_msg`` is enabled, it can be observed that when using tcpdump to capture the traffic on the local lo device, only the three-way handshake and the four-way handshake traffic are captured, but the iperf data traffic is not captured. If the iperf data traffic is captured, then the eBPF program may not have been properly attached.
+
+```console
+$ ./trace_lo_traffic.sh
+# three-way handshake
+13:24:07.181804 IP localhost.46506 > localhost.5001: Flags [S], seq 620239881, win 65495, options [mss 65495,sackOK,TS val 1982813394 ecr 0,nop,wscale 7], length 0
+13:24:07.181815 IP localhost.5001 > localhost.46506: Flags [S.], seq 1084484879, ack 620239882, win 65483, options [mss 65495,sackOK,TS val 1982813394 ecr 1982813394,nop,wscale 7], length 0
+13:24:07.181832 IP localhost.46506 > localhost.5001: Flags [.], ack 1, win 512, options [nop,nop,TS val 1982813394 ecr 1982813394], length 0
+
+# four-way handshake traffic
+13:24:12.475649 IP localhost.46506 > localhost.5001: Flags [F.], seq 1, ack 1, win 512, options [nop,nop,TS val 1982818688 ecr 1982813394], length 0
+13:24:12.479621 IP localhost.5001 > localhost.46506: Flags [.], ack 2, win 512, options [nop,nop,TS val 1982818692 ecr 1982818688], length 0
+13:24:12.481265 IP localhost.5001 > localhost.46506: Flags [F.], seq 1, ack 2, win 512, options [nop,nop,TS val 1982818694 ecr 1982818688], length 0
+13:24:12.481270 IP localhost.46506 > localhost.5001: Flags [.], ack 2, win 512, options [nop,nop,TS val 1982818694 ecr 1982818694], length 0
+```
+
 
 ### Unloading the eBPF Program
 

src/29-sockops/bpf_contrack.bpf.c

@@ -0,0 +1,32 @@
+#include "bpf_sockmap.h"
+
+char LICENSE[] SEC("license") = "Dual BSD/GPL";
+
+SEC("sockops")
+int bpf_sockops_handler(struct bpf_sock_ops *skops){
+    u32 family, op;
+
+	family = skops->family;
+	op = skops->op;
+	if (op != BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB
+        && op != BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB) {
+        return BPF_OK;
+    }
+
+    if(skops->remote_ip4 != LOCALHOST_IPV4 || skops->local_ip4!= LOCALHOST_IPV4) {
+        return BPF_OK;
+    }
+
+	struct sock_key key = {
+        .dip = skops->remote_ip4,
+        .sip = skops->local_ip4,
+        .sport = bpf_htonl(skops->local_port),  /* convert to network byte order */
+        .dport = skops->remote_port,
+        .family = skops->family,
+    };
+
+	bpf_printk(">>> new connection: OP:%d, PORT:%d --> %d\n", op, bpf_ntohl(key.sport), bpf_ntohl(key.dport));
+
+	bpf_sock_hash_update(skops, &sock_ops_map, &key, BPF_NOEXIST);
+    return BPF_OK;
+}

src/29-sockops/bpf_redir.c

@@ -0,0 +1,19 @@
+#include "bpf_sockmap.h"
+
+char LICENSE[] SEC("license") = "Dual BSD/GPL";
+
+SEC("sk_msg")
+int bpf_redir(struct sk_msg_md *msg)
+{
+    if(msg->remote_ip4 != LOCALHOST_IPV4 || msg->local_ip4!= LOCALHOST_IPV4) 
+        return SK_PASS;
+    
+    struct sock_key key = {
+        .sip = msg->remote_ip4,
+        .dip = msg->local_ip4,
+        .dport = bpf_htonl(msg->local_port), /* convert to network byte order */
+        .sport = msg->remote_port,
+        .family = msg->family,
+    };
+    return bpf_msg_redirect_hash(msg, &sock_ops_map, &key, BPF_F_INGRESS);
+}

src/29-sockops/bpf_redirect.bpf.c

@@ -0,0 +1,20 @@
+#include "vmlinux.h"
+#include <bpf/bpf_endian.h>
+#include <bpf/bpf_helpers.h>
+
+#define LOCALHOST_IPV4 16777343
+
+struct sock_key {
+    __u32 sip;
+    __u32 dip;
+    __u32 sport;
+    __u32 dport;
+    __u32 family;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_SOCKHASH);
+	__uint(max_entries, 65535);
+	__type(key, struct sock_key);
+	__type(value, int);
+} sock_ops_map SEC(".maps");