feat: enhance Makefile for E2E testing and update Docker CI workflow

.github/workflows/docker-image.yml

@@ -2,7 +2,7 @@ name: Docker Image CI
 
 on:
   push:
-    branches: ["main", "ghostbaby"]
+    branches: ["main"]
 
 jobs:
   build:

.github/workflows/push.yml

@@ -0,0 +1,72 @@
+name: Push request
+
+on:
+  pull_request:
+    branches:
+      - main
+      - ghostbaby
+
+jobs:
+  e2e-build:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@main
+      - name: Install dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install -y \
+          golang-1.22 \
+          git \
+          make \
+          gcc \
+          clang \
+          llvm \
+          && mkdir -p ./internal/binary &&  sudo rm -rf /var/lib/apt/lists/*
+      - name: Build binary
+        run: |
+          make build
+      - uses: actions/upload-artifact@master
+        with:
+          name: shepherd
+          path: |
+            ./cmd/shepherd
+      - uses: actions/upload-artifact@master
+        with:
+          name: config.yaml
+          path: |
+            ./cmd/config.yaml
+  e2e-6_10:
+    needs: e2e-build
+    uses: ./.github/workflows/reusable-workflow.yml
+    with:
+      kernel: "6.10"
+  e2e-6_6:
+    needs: e2e-build
+    uses: ./.github/workflows/reusable-workflow.yml
+    with:
+      kernel: "6.6"
+  e2e-6_1:
+    needs: e2e-build
+    uses: ./.github/workflows/reusable-workflow.yml
+    with:
+      kernel: "6.1"
+  e2e-5_15:
+    needs: e2e-build
+    uses: ./.github/workflows/reusable-workflow.yml
+    with:
+      kernel: "5.15"
+  e2e-5_10:
+    needs: e2e-build
+    uses: ./.github/workflows/reusable-workflow.yml
+    with:
+      kernel: "5.10"
+  e2e-5_8:
+    needs: e2e-build
+    uses: ./.github/workflows/reusable-workflow.yml
+    with:
+      kernel: "5.8"
+  e2e-5_4:
+    needs: e2e-build
+    uses: ./.github/workflows/reusable-workflow.yml
+    with:
+      kernel: "5.4"

.github/workflows/reusable-workflow.yml

@@ -0,0 +1,31 @@
+name: reusable jobs
+
+on:
+  workflow_call:
+    inputs:
+      kernel:
+        description: 'Kernel'
+        required: true
+        type: string
+
+jobs:
+  e2e-kernel:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@main
+      - name: Install qemu & friends
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y qemu-utils qemu-system-x86 sshpass
+      - uses: actions/download-artifact@master
+        with:
+          name: shepherd
+          path: ./cmd/
+      - uses: actions/download-artifact@master
+        with:
+          name: config.yaml
+          path: ./cmd/
+      - name: Run e2e tests on kernel ${{ inputs.kernel }}
+        run: |
+          KERNEL=${{ inputs.kernel }} make e2e

Makefile

@@ -28,3 +28,53 @@ image:
 	docker buildx create --use
 	docker buildx build --platform linux/amd64 -t ghostbaby/shepherd:v0.0.1-amd64 --push .
 	docker buildx build --platform linux/arm64 -t ghostbaby/shepherd:v0.0.1-arm64 --push .
+
+.ONESHELL:
+prepare_e2e_filesystem:
+	cd ./tests/e2e/vm/filesystem
+	# build filesystem image and store as tar archive
+	DOCKER_BUILDKIT=1 docker build --output "type=tar,dest=filesystem.tar" .
+	# convert tar to qcow2 image
+	sudo virt-make-fs --format=qcow2 --size=+100M filesystem.tar filesystem-large.qcow2
+	# reduce size of image
+	qemu-img convert filesystem-large.qcow2 -O qcow2 filesystem.qcow2
+	# reduce size by packing
+	zip filesystem.zip filesystem.qcow2
+	# remove unnecessary files
+	rm -f filesystem-large.qcow2 filesystem.qcow2 filesystem.tar
+
+.ONESHELL:
+start_qemu:
+	cd ./tests/e2e/vm/filesystem
+	rm -f filesystem.qcow2 filesystem-diff.qcow2
+	unzip ./filesystem.zip
+	sudo qemu-img create -f qcow2 -b filesystem.qcow2 -F qcow2 filesystem-diff.qcow2
+	PWD=$(pwd)
+	sudo qemu-system-x86_64 \
+	-cpu host \
+	-m 4G \
+	-smp 4 \
+	-kernel ${PWD}/tests/e2e/vm/kernels/${KERNEL}/bzImage \
+	-append "console=ttyS0 root=/dev/sda rw" \
+	-drive file="${PWD}/tests/e2e/vm/filesystem/filesystem-diff.qcow2,format=qcow2" \
+	-net nic -net user,hostfwd=tcp::10022-:22,hostfwd=tcp::16676-:6676,hostfwd=tcp::10443-:443 \
+	-enable-kvm \
+	-pidfile qemu.pid \
+	-nographic &
+
+.ONESHELL:
+prepare_e2e: start_qemu
+	while ! nc -z 127.0.0.1 10022 ; do echo "waiting for ssh"; sleep 1; done
+	sshpass -p root scp -o 'StrictHostKeyChecking no' -P 10022 ./cmd/shepherd root@127.0.0.1:/root/shepherd
+	sshpass -p root scp -o 'StrictHostKeyChecking no' -P 10022 ./cmd/config.yaml root@127.0.0.1:/root/config.yaml
+	sshpass -p root ssh -p 10022 root@127.0.0.1 'chmod 0655 /root/shepherd && systemctl start shepherd.service'
+	while ! sshpass -p root ssh -p 10022 root@127.0.0.1 'systemctl is-active shepherd.service' ; do echo "waiting for shepherd service"; sleep 1; done
+
+.ONESHELL:
+e2e: prepare_e2e
+	ifconfig
+	RC=$$?
+	pwd
+	sshpass -p root ssh -p 10022 root@127.0.0.1 'ls /root && ls /root/log/'
+	sudo cat ./tests/e2e/vm/filesystem/qemu.pid | sudo xargs kill
+	exit $$RC

internal/output/sched_delay.go

@@ -2,6 +2,7 @@ package output
 
 import (
 	"context"
+	"fmt"
 	"os"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
@@ -31,7 +32,7 @@ func ProcessSchedDelay(coll *ebpf.Collection, ctx context.Context, cfg config.Co
 		log.Fatalf("failed to connect to clickhouse: %v", err)
 	}
 
-	// defer conn.Close()
+	defer conn.Close()
 
 	// 准备批量插入语句
 	batch, err := conn.PrepareBatch(ctx, `
@@ -61,6 +62,8 @@ func ProcessSchedDelay(coll *ebpf.Collection, ctx context.Context, cfg config.Co
 				continue
 			}
 
+			fmt.Println(event)
+
 			batch, count, err = insertSchedMetrics(ctx, conn, batch, event, count)
 			if err != nil {
 				log.Errorf("failed to insert sched metrics: %v", err)

tests/e2e/README.md

@@ -0,0 +1,5 @@
+### e2e tests for various Linux kernels
+
+- e2e tests module uses `qemu` emulation tool to emulate various linux kernels, see target `start_qemu` in [Makefile](../../Makefile)
+- `qemu` uses precompiled linux kernels, see [kernel compilation README](./vm/kernels/README.md)
+- `qemu` uses prepared filesystem based on [Dockerfile](./vm/filesystem/Dockerfile)

tests/e2e/resources/fields_bytes.json

@@ -0,0 +1,60 @@
+{
+  "edges_fields": [
+    {
+      "field_name": "id",
+      "type": "string"
+    },
+    {
+      "field_name": "source",
+      "type": "string"
+    },
+    {
+      "field_name": "target",
+      "type": "string"
+    },
+    {
+      "field_name": "mainStat",
+      "type": "string"
+    },
+    {
+      "field_name": "secondaryStat",
+      "type": "string"
+    }
+  ],
+  "nodes_fields": [
+    {
+      "field_name": "id",
+      "type": "string"
+    },
+    {
+      "field_name": "title",
+      "type": "string"
+    },
+    {
+      "field_name": "subTitle",
+      "type": "string"
+    },
+    {
+      "field_name": "mainStat",
+      "displayName": "Bytes received ",
+      "type": "string"
+    },
+    {
+      "field_name": "secondaryStat",
+      "displayName": "Bytes responded ",
+      "type": "string"
+    },
+    {
+      "color": "blue",
+      "field_name": "arc__1",
+      "type": "number",
+      "displayName": "Bytes received"
+    },
+    {
+      "color": "yellow",
+      "field_name": "arc__2",
+      "type": "number",
+      "displayName": "Bytes responded"
+    }
+  ]
+}

tests/e2e/resources/fields_connection.json

@@ -0,0 +1,60 @@
+{
+  "edges_fields": [
+    {
+      "field_name": "id",
+      "type": "string"
+    },
+    {
+      "field_name": "source",
+      "type": "string"
+    },
+    {
+      "field_name": "target",
+      "type": "string"
+    },
+    {
+      "field_name": "mainStat",
+      "type": "string"
+    },
+    {
+      "field_name": "secondaryStat",
+      "type": "string"
+    }
+  ],
+  "nodes_fields": [
+    {
+      "field_name": "id",
+      "type": "string"
+    },
+    {
+      "field_name": "title",
+      "type": "string"
+    },
+    {
+      "field_name": "subTitle",
+      "type": "string"
+    },
+    {
+      "field_name": "mainStat",
+      "displayName": "All connections ",
+      "type": "string"
+    },
+    {
+      "field_name": "secondaryStat",
+      "displayName": "Persistent connections ",
+      "type": "string"
+    },
+    {
+      "color": "green",
+      "field_name": "arc__1",
+      "type": "number",
+      "displayName": "Persistent connections"
+    },
+    {
+      "color": "red",
+      "field_name": "arc__2",
+      "type": "number",
+      "displayName": "Short-lived connections"
+    }
+  ]
+}

tests/e2e/resources/fields_duration.json

@@ -0,0 +1,60 @@
+{
+  "edges_fields": [
+    {
+      "field_name": "id",
+      "type": "string"
+    },
+    {
+      "field_name": "source",
+      "type": "string"
+    },
+    {
+      "field_name": "target",
+      "type": "string"
+    },
+    {
+      "field_name": "mainStat",
+      "type": "string"
+    },
+    {
+      "field_name": "secondaryStat",
+      "type": "string"
+    }
+  ],
+  "nodes_fields": [
+    {
+      "field_name": "id",
+      "type": "string"
+    },
+    {
+      "field_name": "title",
+      "type": "string"
+    },
+    {
+      "field_name": "subTitle",
+      "type": "string"
+    },
+    {
+      "field_name": "mainStat",
+      "displayName": "Average duration ",
+      "type": "string"
+    },
+    {
+      "field_name": "secondaryStat",
+      "displayName": "Max duration ",
+      "type": "string"
+    },
+    {
+      "color": "purple",
+      "field_name": "arc__1",
+      "type": "number",
+      "displayName": "Average duration"
+    },
+    {
+      "color": "white",
+      "field_name": "arc__2",
+      "type": "number",
+      "displayName": "Max duration"
+    }
+  ]
+}