linux内核协议栈实现了一个虚拟机，允许用户程序向内核注入二进制字节码。注入的程序，就可以做一些有趣的事情，比如：负载均衡，数据包检测，加速容器网络转发，做个网损仪。
本篇，运行一个测试程序，丢弃网卡收到的icmp包。
xdp-drop.c，源码url：https://gitlab.com/mwiget/xdp-drop-test/-/tree/master
//https://gitlab.com/mwiget/xdp-drop-test/-/tree/master
//https://stackoverflow.com/questions/64861121/ebpf-program-load-fails-without-verifier-log
#include <linux/bpf.h>
#include <linux/in.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#define SEC(NAME) __attribute__((section(NAME), used))
/*
struct bpf_map_def {
unsigned int type;
unsigned int key_size;
unsigned int value_size;
unsigned int max_entries;
unsigned int map_flags;
unsigned int id;
unsigned int pinning;
unsigned int inner_id;
unsigned int inner_idx;
};
struct bpf_map_def SEC("maps") dummy_map = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int),
.max_entries = 1,
};*/
SEC("drop_icmp")
int drop_icmp_func(struct xdp_md *ctx) {
//int key = 0;
//int *v = (int*)bpf_map_lookup_elem(&dummy_map, &key);
int ipsize = 0;
void *data = (void *)(long)ctx->data;
void *data_end = (void *)(long)ctx->data_end;
struct ethhdr *eth = data;
ipsize = sizeof(*eth);
struct iphdr *ip = data + ipsize;
ipsize += sizeof(struct iphdr);
if (data + ipsize > data_end) {
// not an ip packet, too short. Pass it on
return XDP_PASS;
}
// technically, we should also check if it is an IP packet by
// checking the ethernet header proto field ...
if (ip->protocol == IPPROTO_ICMP) {
return XDP_DROP;
}
return XDP_PASS;
}
char __license[] SEC("license") = "GPL";
编译程序之前，也许需要升级下clang，参考博客[5]安装clang-13。
编译
clang -g -c -O2 -target bpf -c xdp-drop.c -o xdp-drop.o
加载程序到网卡中：
sudo ip link set dev you-nic-name xdp obj xdp-drop.o sec drop_icmp
查看网卡状态，我的usb网卡名字(you-nic-name)wlxe0e1a997ab49：
ip link show dev you-nic-name
测试，就是ping不通主机。
卸载之后，恢复。
sudo ip link set dev you-nic-name xdp off
昨天，因为clang的安装卡了三四个个小时。还是怀念在学校的日子，起码可以找个小弟，搞点测试，验证可行性。工作了，就像坐监，出卖时间换钱，没啥自己的时间。
本节在xdp-drop.c代码中增加丢包个数统计的功能。
xdp-drop.c，源码：
//https://gitlab.com/mwiget/xdp-drop-test/-/tree/master
//https://stackoverflow.com/questions/64861121/ebpf-program-load-fails-without-verifier-log
#include <linux/bpf.h>
#include <linux/in.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/types.h>
#include <stddef.h>
#include <memory.h>
#include <sys/types.h>
#define SEC(NAME) __attribute__((section(NAME), used))
#define __uint(name, val) int(*(name))[val]
#define __type(name, val) typeof(val) *(name)
#define __array(name, val) typeof(val) *(name)[]
/* struct bpf_map_def {
unsigned int type;
unsigned int key_size;
unsigned int value_size;
unsigned int max_entries;
};
struct bpf_map_def SEC("maps") drop_map = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(int),
.value_size = sizeof(int),
.max_entries = 1,
};*/
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(key_size, sizeof(int));
__uint(value_size, sizeof(int));
__uint(max_entries, 1);
}drop_map SEC(".maps");
static void *(*bpf_map_lookup_elem)(void *map, void *key) =
(void *) BPF_FUNC_map_lookup_elem;
SEC("drop_icmp")
int drop_icmp_func(struct xdp_md *ctx) {
int key = 0;
int *pkts_count=bpf_map_lookup_elem(&drop_map, &key);
int ipsize = 0;
void *data = (void *)(long)ctx->data;
void *data_end = (void *)(long)ctx->data_end;
struct ethhdr *eth = data;
ipsize = sizeof(*eth);
struct iphdr *ip = data + ipsize;
ipsize += sizeof(struct iphdr);
if (data + ipsize > data_end) {
// not an ip packet, too short. Pass it on
return XDP_PASS;
}
// technically, we should also check if it is an IP packet by
// checking the ethernet header proto field ...
if (ip->protocol == IPPROTO_ICMP) {
if(pkts_count){
(*pkts_count)++;
}
return XDP_DROP;
}
return XDP_PASS;
}
char __license[] SEC("license") = "GPL";
/* reference
https://www.tigera.io/learn/guides/ebpf/ebpf-xdp/
https://cloud.tencent.com/developer/article/1770282
https://feisky.xyz/posts/2021-01-29-ebpf-program/
*/
/*
fatal error: sys/cdefs.h: No such file or directory
sudo apt-get --reinstall install libc6 libc6-dev
export C_INCLUDE_PATH=$C_INCLUDE_PATH:/usr/include/x86_64-linux-gnu/
clang -g -c -O2 -target bpf -c xdp-drop.c -o xdp-drop.o
llvm-objdump -h xdp-drop.o
objdump -h xdp-drop.o
sudo ip link set dev wlxe0e1a997ab49 xdp obj xdp-drop.o sec drop_icmp
ip link show dev wlxe0e1a997ab49
sudo ip link set dev wlxe0e1a997ab49 xdp off
*/
我原来安装的是ubuntu 18.04。当加载到内核的程序中有自定义map，并且使用了bpf_map_lookup_elem函数时候，总是报“R1 type=inv expected=map_ptr”的错误。后来我就重装了ubuntu 20.04，遇见系统安装问题，卡了很久[6]。
用户程序drop-pkt-reader.c，读取丢包个数：
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <getopt.h>
#include <stdint.h>
#include <stdbool.h>
#include <signal.h>
#include <errno.h>
#include <unistd.h> //sleep
#include <libgen.h>// basename
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include <linux/if_link.h>//XDP_FLAGS_UPDATE_IF_NOEXIST
#include <net/if.h>
int print_if_nameindex(){
struct if_nameindex *if_ni, *i;
if_ni = if_nameindex();
if(if_ni){
for (i = if_ni; !(i->if_index == 0 && i->if_name == NULL); i++){
printf("%s,%d\n",i->if_name,i->if_index);
}
if_freenameindex(if_ni);
}
return 0;
}
/*
https://www.cnblogs.com/charlieroro/p/13403672.html
Native XDP(XDP_FLAGS_DRV_MODE)
Offloaded XDP(XDP_FLAGS_HW_MODE)
Generic XDP(XDP_FLAGS_SKB_MODE)
samples/bpf/xdp_sample_pkts_user.c
*/
static void usage(const char *prog)
{
fprintf(stderr,
"%s: %s -n <ifname> -o xxx.o\n\n"
"OPTS:\n"
" -F force loading prog\n"
" -S use skb-mode\n",
__func__, prog);
}
static int do_attach(int fd,uint32_t xdp_flags,int if_index,const char *if_name){
struct bpf_prog_info info = {};
uint32_t info_len = sizeof(info);
int prog_id=-1,err = 0;
err = bpf_xdp_attach(if_index, fd, xdp_flags, NULL);
if (err < 0) {
printf("ERROR: failed to attach program to %s\n", if_name);
return prog_id;
}
err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
if (err) {
printf("can't get prog info - %s\n", strerror(errno));
return prog_id;
}
prog_id = info.id;
return prog_id;
}
static int do_detach(int prog_id,uint32_t xdp_flags,int if_index, const char *if_name)
{
int curr_prog_id = 0;
int err = 0;
err = bpf_xdp_query_id(if_index, xdp_flags, &curr_prog_id);
if (err) {
printf("bpf_xdp_query_id failed\n");
return err;
}
if (prog_id == curr_prog_id) {
err = bpf_xdp_detach(if_index, xdp_flags, NULL);
if (err < 0)
printf("ERROR: failed to detach prog from %s\n", if_name);
} else if (!curr_prog_id) {
printf("couldn't find a prog id on a %s\n", if_name);
} else {
printf("program on interface changed, not removing\n");
}
return err;
}
void map_get_value_array(int fd, int key, int *value){
if ((bpf_map_lookup_elem(fd, &key, value)) != 0) {
fprintf(stderr,"ERR: bpf_map_lookup_elem failed key:0x%X\n", key);
*value=-1;
}
}
static volatile bool g_running=true;
static void sig_handler(int signo)
{
g_running=false;
}
// ./drop-pkt-reader -n wlxe0e1a997ab49 -o xdp-drop.o -S
int main(int argc, char **argv){
char if_name[32]={0};
char bpf_name[32]={0};
const char *optstr = "n:o:FS";
int prog_fd, map_fd=-1, opt;
int prog_id=-1;
struct bpf_program *prog;
struct bpf_object *obj;
struct bpf_map *map;
uint32_t xdp_flags=XDP_FLAGS_UPDATE_IF_NOEXIST;
const char *target_map="drop_map";
if (signal(SIGINT, sig_handler) ||
signal(SIGHUP, sig_handler) ||
signal(SIGTERM, sig_handler)||
signal(SIGTSTP,sig_handler)){
perror("signal");
return 1;
}
while ((opt = getopt(argc, argv, optstr)) != -1) {
switch (opt) {
case 'n':{
memcpy(if_name,optarg,strlen(optarg));
break;
}
case 'o':{
memcpy(bpf_name,optarg,strlen(optarg));
break;
}
case 'F':{
xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
break;
}
case 'S':{
xdp_flags |= XDP_FLAGS_SKB_MODE;
break;
}
default:{
usage(basename(argv[0]));
return 1;
}
}
}
uint32_t if_index=if_nametoindex(if_name);
printf("%s,%d,%d\n",if_name,if_index,optind);
if(if_index){
obj = bpf_object__open_file(bpf_name, NULL);
if (libbpf_get_error(obj)){
return 1;
}
//may try: bpf_object__find_program_by_title
prog = bpf_object__next_program(obj, NULL);
bpf_program__set_type(prog, BPF_PROG_TYPE_XDP);
if(prog){
const char *sec_name = bpf_program__section_name(prog);
printf("sec:%s\n",sec_name);
}
int err = bpf_object__load(obj);
if (err){
bpf_object__close(obj);
return 1;
}
prog_fd = bpf_program__fd(prog);
map =bpf_object__find_map_by_name(obj,target_map);
if(map){
printf("finding %s \n",target_map);
}else{
printf("finding %s failed\n",target_map);
bpf_object__close(obj);
return 1;
}
map_fd = bpf_map__fd(map);
prog_id=do_attach(prog_fd,xdp_flags,if_index,if_name);
}else{
printf("nic %s does not exist\n",if_name);
}
int value=0;
while(g_running){
sleep(5);
map_get_value_array(map_fd,0,&value);
if(value>0){
printf("drop pkts: %d\n",value);
}
}
do_detach(prog_id,xdp_flags,if_index,if_name);
if(obj){
bpf_object__close(obj);
}
return 0;
}
附带的CmakeLists.txt:
PROJECT(project)
cmake_minimum_required(VERSION 2.6)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -O2")
SET(CMAKE_CXX_FLAGS "-fPIC")
#for libbpf
set(LIBBPF_DIR /build/root/usr)
include_directories(${LIBBPF_DIR}/include)
LINK_DIRECTORIES(${LIBBPF_DIR}/lib64)
set(EXECUTABLE_NAME "drop-pkt-reader")
add_executable(${EXECUTABLE_NAME} ${CMAKE_SOURCE_DIR}/drop-pkt-reader.c)
target_link_libraries(${EXECUTABLE_NAME} bpf)
#cmake .. -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++
drop-pkt-reader.c依赖libbpf，需要源码编译安装[7]：
$ cd src
$ PKG_CONFIG_PATH=/build/root/lib64/pkgconfig DESTDIR=/build/root make install
安装路径为/build/root，CmakeLists.txt有对应的路径配置：set(LIBBPF_DIR /build/root/usr)。
把CmakeLists.txt和drop-pkt-reader.c复制到xxx文件夹中。编译程序：
cd xxx
mkdir build && cd build
cmake .. -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++
make
clang -g -c -O2 -target bpf -c xdp-drop.c -o xdp-drop.o
测试：
./drop-pkt-reader -n nic-name -o xdp-drop.o -S
[1]简要概述XDP
[2]Linux网络新技术基石 |eBPF and XDP
[3]eBPF 技术实践：加速容器网络转发
[4]一文看懂eBPF｜eBPF实现原理
[5]ubuntu 18.04 install clang 11
[6]ubuntu grub-install failed
[7]github libbpf