From 592be8ec12e13002459ea79b6bb5016b1515b9ac Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 00:01:59 +0000
Subject: [PATCH 01/42] example

---
 examples/basic_p4/l2switch/spec.yaml | 60 ++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 examples/basic_p4/l2switch/spec.yaml

diff --git a/examples/basic_p4/l2switch/spec.yaml b/examples/basic_p4/l2switch/spec.yaml
new file mode 100644
index 0000000..8b81fb5
--- /dev/null
+++ b/examples/basic_p4/l2switch/spec.yaml
@@ -0,0 +1,60 @@
+nodes:
+- name: P4
+  image: tinynetwork/p4bmv2:develop
+  docker_run_extra_args: -v `realpath $PWD/../P4`:/behavioral-model/src
+  interfaces:
+  - { name: vm1, type: direct, args: VM1#net0, addr: 10:00:00:00:00:10 }
+  - { name: vm2, type: direct, args: VM2#net0, addr: 20:00:00:00:00:20 }
+  - { name: vm3, type: direct, args: VM3#net0, addr: 30:00:00:00:00:30 }
+  sysctls:
+  - sysctl: net.ipv4.ip_forward=0
+  - sysctl: net.ipv6.conf.all.forwarding=0
+
+- name: VM1
+  image: nicolaka/netshoot
+  docker_run_extra_args: --entrypoint bash
+  interfaces:
+  - { name: net0, type: direct, args: P4#vm1, addr: 10:10:10:10:10:10 }
+
+- name: VM2
+  image: nicolaka/netshoot
+  docker_run_extra_args: --entrypoint bash
+  interfaces:
+  - { name: net0, type: direct, args: P4#vm2, addr: 20:20:20:20:20:20 }
+
+- name: VM3
+  image: nicolaka/netshoot
+  docker_run_extra_args: --entrypoint bash
+  interfaces:
+  - { name: net0, type: direct, args: P4#vm3, addr: 30:30:30:30:30:30 }
+
+node_configs:
+- name: P4
+  cmds:
+  - cmd: ip link set vm1 address 10:00:00:00:00:10
+  - cmd: ip link set vm2 address 20:00:00:00:00:20
+  - cmd: ip link set vm3 address 30:00:00:00:00:30
+  - cmd: ip addr add 192.168.10.1/24 dev vm1
+  - cmd: ip addr add 192.168.20.1/24 dev vm2
+  - cmd: ip addr add 192.168.30.1/24 dev vm3
+
+- name: VM1
+  cmds:
+  - cmd: ip link set net0 address 10:10:10:10:10:10
+  - cmd: ip addr add 192.168.10.10/24 dev net0
+  - cmd: ip neigh replace 192.168.10.1 lladdr 10:00:00:00:00:10 dev net0
+  - cmd: ip route add default via 192.168.10.1
+
+- name: VM2
+  cmds:
+  - cmd: ip link set net0 address 20:20:20:20:20:20
+  - cmd: ip addr add 192.168.20.20/24 dev net0
+  - cmd: ip neigh replace 192.168.20.1 lladdr 20:00:00:00:00:20 dev net0
+  - cmd: ip route add default via 192.168.20.1
+
+- name: VM3
+  cmds:
+  - cmd: ip link set net0 address 30:30:30:30:30:30
+  - cmd: ip addr add 192.168.30.30/24 dev net0
+  - cmd: ip neigh replace 192.168.30.1 lladdr 30:00:00:00:00:30 dev net0
+  - cmd: ip route add default via 192.168.30.1

From a12b7d7f7b91a05afbcb3a8994a08398ac294276 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 00:02:09 +0000
Subject: [PATCH 02/42] dockerfile

---
 Dockerfiles/p4bmv2/Dockerfile | 3 +++
 Dockerfiles/p4bmv2/Makefile   | 8 ++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 Dockerfiles/p4bmv2/Dockerfile
 create mode 100644 Dockerfiles/p4bmv2/Makefile

diff --git a/Dockerfiles/p4bmv2/Dockerfile b/Dockerfiles/p4bmv2/Dockerfile
new file mode 100644
index 0000000..870406c
--- /dev/null
+++ b/Dockerfiles/p4bmv2/Dockerfile
@@ -0,0 +1,3 @@
+FROM p4lang/behavioral-model
+RUN apt-get update -y && apt-get upgrade -y
+RUN apt-get install -y iproute2
diff --git a/Dockerfiles/p4bmv2/Makefile b/Dockerfiles/p4bmv2/Makefile
new file mode 100644
index 0000000..399d485
--- /dev/null
+++ b/Dockerfiles/p4bmv2/Makefile
@@ -0,0 +1,8 @@
+IMG=tinynetwork/p4bmv2:develop
+build:
+	docker build -t $(IMG) .
+push:
+	docker push $(IMG)
+all: build push
+run:
+	docker run --rm -it --privileged $(IMG) bash

From 0f3c6e7721385c82768f2a5a19b465edbfab0011 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 00:09:39 +0000
Subject: [PATCH 03/42] ex

---
 examples/basic_p4/l2switch/runtime.txt | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 examples/basic_p4/l2switch/runtime.txt

diff --git a/examples/basic_p4/l2switch/runtime.txt b/examples/basic_p4/l2switch/runtime.txt
new file mode 100644
index 0000000..ec3ec9f
--- /dev/null
+++ b/examples/basic_p4/l2switch/runtime.txt
@@ -0,0 +1,4 @@
+table_set_default MyIngress.ipv4_lpm MyIngress.drop
+table_add MyIngress.ipv4_lpm MyIngress.ipv4_forward 192.168.10.10/32 => 10:10:10:10:10:10 1
+table_add MyIngress.ipv4_lpm MyIngress.ipv4_forward 192.168.20.20/32 => 20:20:20:20:20:20 2
+table_add MyIngress.ipv4_lpm MyIngress.ipv4_forward 192.168.30.30/32 => 30:30:30:30:30:30 3

From cb773de1a3d9aeb05601775205c70402b54d20e4 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 00:28:07 +0000
Subject: [PATCH 04/42] img

---
 Dockerfiles/p4bmv2/Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfiles/p4bmv2/Dockerfile b/Dockerfiles/p4bmv2/Dockerfile
index 870406c..4ca336e 100644
--- a/Dockerfiles/p4bmv2/Dockerfile
+++ b/Dockerfiles/p4bmv2/Dockerfile
@@ -1,3 +1,3 @@
-FROM p4lang/behavioral-model
-RUN apt-get update -y && apt-get upgrade -y
-RUN apt-get install -y iproute2
+# p4lang/behavioral-model is included by p4lang/p4c
+FROM p4lang/p4c
+RUN apt-get update -y && apt-get install -y iproute2

From 8001054c2b9b351c8873d6df6b5c867af3e086b9 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 01:05:23 +0000
Subject: [PATCH 05/42] img

---
 Dockerfiles/p4bmv2/Dockerfile |  4 +++-
 Dockerfiles/p4bmv2/wait.sh    | 11 +++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)
 create mode 100644 Dockerfiles/p4bmv2/wait.sh

diff --git a/Dockerfiles/p4bmv2/Dockerfile b/Dockerfiles/p4bmv2/Dockerfile
index 4ca336e..05a88f2 100644
--- a/Dockerfiles/p4bmv2/Dockerfile
+++ b/Dockerfiles/p4bmv2/Dockerfile
@@ -1,3 +1,5 @@
 # p4lang/behavioral-model is included by p4lang/p4c
 FROM p4lang/p4c
-RUN apt-get update -y && apt-get install -y iproute2
+RUN apt-get update -y && apt-get install -y iproute2 vim
+COPY ./wait.sh /usr/bin/wait.sh
+RUN chmod +x /usr/bin/wait.sh
diff --git a/Dockerfiles/p4bmv2/wait.sh b/Dockerfiles/p4bmv2/wait.sh
new file mode 100644
index 0000000..aff0916
--- /dev/null
+++ b/Dockerfiles/p4bmv2/wait.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+for i in `seq 10`; do
+  echo EOF | bm_CLI
+  if [ "$?" == "0" ]; then
+    echo "bm_CLI is ready!"
+    exit 0
+  fi
+  sleep 1
+done
+echo "bm_CLI is not ready"
+exit 1

From f186ca0fccfb288c7b56194cbfc07a5dc4fb3703 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 01:05:43 +0000
Subject: [PATCH 06/42] ex

---
 examples/basic_p4/l2switch/spec.yaml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/examples/basic_p4/l2switch/spec.yaml b/examples/basic_p4/l2switch/spec.yaml
index 8b81fb5..e4cca06 100644
--- a/examples/basic_p4/l2switch/spec.yaml
+++ b/examples/basic_p4/l2switch/spec.yaml
@@ -1,7 +1,7 @@
 nodes:
 - name: P4
   image: tinynetwork/p4bmv2:develop
-  docker_run_extra_args: -v `realpath $PWD/../P4`:/behavioral-model/src
+  docker_run_extra_args: --entrypoint bash
   interfaces:
   - { name: vm1, type: direct, args: VM1#net0, addr: 10:00:00:00:00:10 }
   - { name: vm2, type: direct, args: VM2#net0, addr: 20:00:00:00:00:20 }
@@ -38,6 +38,14 @@ node_configs:
   - cmd: ip addr add 192.168.20.1/24 dev vm2
   - cmd: ip addr add 192.168.30.1/24 dev vm3
 
+  - cmd: p4c --target bmv2 --arch v1model /main.p4 --p4runtime-files p4info.txt -o /
+  - cmd: >-
+      nohup simple_switch /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
+      --nanolog ipc:///tmp/bm-0-log.ipc --log-console -L debug
+      --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc &
+  - cmd: wait.sh
+  - cmd: sh -c "cat /runtime.txt | bm_CLI"
+
 - name: VM1
   cmds:
   - cmd: ip link set net0 address 10:10:10:10:10:10
@@ -58,3 +66,8 @@ node_configs:
   - cmd: ip addr add 192.168.30.30/24 dev net0
   - cmd: ip neigh replace 192.168.30.1 lladdr 30:00:00:00:00:30 dev net0
   - cmd: ip route add default via 192.168.30.1
+
+postinit:
+  cmds:
+  - cmd: docker cp main.p4 P4:/main.p4
+  - cmd: docker cp runtime.txt P4:/runtime.txt

From 27cf9ec5e9f432490683b78e5ff5a864431d09d1 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 01:05:49 +0000
Subject: [PATCH 07/42] ex

---
 examples/basic_p4/l2switch/main.p4 | 176 +++++++++++++++++++++++++++++
 1 file changed, 176 insertions(+)
 create mode 100644 examples/basic_p4/l2switch/main.p4

diff --git a/examples/basic_p4/l2switch/main.p4 b/examples/basic_p4/l2switch/main.p4
new file mode 100644
index 0000000..0e221a3
--- /dev/null
+++ b/examples/basic_p4/l2switch/main.p4
@@ -0,0 +1,176 @@
+/* -*- P4_16 -*- */
+#include <core.p4>
+#include <v1model.p4>
+
+const bit<16> TYPE_IPV4 = 0x800;
+
+/*************************************************************************
+*********************** H E A D E R S  ***********************************
+*************************************************************************/
+
+typedef bit<9>  egressSpec_t;
+typedef bit<48> macAddr_t;
+typedef bit<32> ip4Addr_t;
+
+header ethernet_t {
+    macAddr_t dstAddr;
+    macAddr_t srcAddr;
+    bit<16>   etherType;
+}
+
+header ipv4_t {
+    bit<4>    version;
+    bit<4>    ihl;
+    bit<8>    diffserv;
+    bit<16>   totalLen;
+    bit<16>   identification;
+    bit<3>    flags;
+    bit<13>   fragOffset;
+    bit<8>    ttl;
+    bit<8>    protocol;
+    bit<16>   hdrChecksum;
+    ip4Addr_t srcAddr;
+    ip4Addr_t dstAddr;
+}
+
+struct metadata {
+    /* empty */
+}
+
+struct headers {
+    ethernet_t   ethernet;
+    ipv4_t       ipv4;
+}
+
+/*************************************************************************
+*********************** P A R S E R  ***********************************
+*************************************************************************/
+
+parser MyParser(packet_in packet,
+                out headers hdr,
+                inout metadata meta,
+                inout standard_metadata_t standard_metadata) {
+
+    state start {
+        transition parse_ethernet;
+    }
+
+    state parse_ethernet {
+        packet.extract(hdr.ethernet);
+        transition select(hdr.ethernet.etherType) {
+            TYPE_IPV4: parse_ipv4;
+            default: accept;
+        }
+    }
+
+    state parse_ipv4 {
+        packet.extract(hdr.ipv4);
+        transition accept;
+    }
+
+}
+
+/*************************************************************************
+************   C H E C K S U M    V E R I F I C A T I O N   *************
+*************************************************************************/
+
+control MyVerifyChecksum(inout headers hdr, inout metadata meta) {
+    apply {  }
+}
+
+
+/*************************************************************************
+**************  I N G R E S S   P R O C E S S I N G   *******************
+*************************************************************************/
+
+control MyIngress(inout headers hdr,
+                  inout metadata meta,
+                  inout standard_metadata_t standard_metadata) {
+    action drop() {
+        mark_to_drop(standard_metadata);
+    }
+
+    action ipv4_forward(macAddr_t dstAddr, egressSpec_t port) {
+        standard_metadata.egress_spec = port;
+        hdr.ethernet.srcAddr = hdr.ethernet.dstAddr;
+        hdr.ethernet.dstAddr = dstAddr;
+        hdr.ipv4.ttl = hdr.ipv4.ttl - 1;
+    }
+
+    table ipv4_lpm {
+        key = {
+            hdr.ipv4.dstAddr: lpm;
+        }
+        actions = {
+            ipv4_forward;
+            drop;
+            NoAction;
+        }
+        size = 1024;
+        default_action = drop();
+    }
+
+    apply {
+        if (hdr.ipv4.isValid()) {
+            ipv4_lpm.apply();
+        }
+    }
+}
+
+/*************************************************************************
+****************  E G R E S S   P R O C E S S I N G   *******************
+*************************************************************************/
+
+control MyEgress(inout headers hdr,
+                 inout metadata meta,
+                 inout standard_metadata_t standard_metadata) {
+    apply {  }
+}
+
+/*************************************************************************
+*************   C H E C K S U M    C O M P U T A T I O N   **************
+*************************************************************************/
+
+control MyComputeChecksum(inout headers  hdr, inout metadata meta) {
+    apply {
+        update_checksum(
+        hdr.ipv4.isValid(),
+            { hdr.ipv4.version,
+              hdr.ipv4.ihl,
+              hdr.ipv4.diffserv,
+              hdr.ipv4.totalLen,
+              hdr.ipv4.identification,
+              hdr.ipv4.flags,
+              hdr.ipv4.fragOffset,
+              hdr.ipv4.ttl,
+              hdr.ipv4.protocol,
+              hdr.ipv4.srcAddr,
+              hdr.ipv4.dstAddr },
+            hdr.ipv4.hdrChecksum,
+            HashAlgorithm.csum16);
+    }
+}
+
+/*************************************************************************
+***********************  D E P A R S E R  *******************************
+*************************************************************************/
+
+control MyDeparser(packet_out packet, in headers hdr) {
+    apply {
+        packet.emit(hdr.ethernet);
+        packet.emit(hdr.ipv4);
+    }
+}
+
+/*************************************************************************
+***********************  S W I T C H  *******************************
+*************************************************************************/
+
+V1Switch(
+    MyParser(),
+    MyVerifyChecksum(),
+    MyIngress(),
+    MyEgress(),
+    MyComputeChecksum(),
+    MyDeparser()
+) main;
\ No newline at end of file

From 3180ef238ec8867e47b1225263e493f62a1ffba6 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 01:12:10 +0000
Subject: [PATCH 08/42] ex

---
 examples/basic_p4/l2switch/main.p4 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/basic_p4/l2switch/main.p4 b/examples/basic_p4/l2switch/main.p4
index 0e221a3..4a3898c 100644
--- a/examples/basic_p4/l2switch/main.p4
+++ b/examples/basic_p4/l2switch/main.p4
@@ -173,4 +173,4 @@ V1Switch(
     MyEgress(),
     MyComputeChecksum(),
     MyDeparser()
-) main;
\ No newline at end of file
+) main;

From 90359dae2f66b372fc222df59578a964172f034b Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 01:14:02 +0000
Subject: [PATCH 09/42] ex

---
 examples/basic_p4/{l2switch => l3switch}/main.p4     | 0
 examples/basic_p4/{l2switch => l3switch}/runtime.txt | 0
 examples/basic_p4/{l2switch => l3switch}/spec.yaml   | 7 -------
 3 files changed, 7 deletions(-)
 rename examples/basic_p4/{l2switch => l3switch}/main.p4 (100%)
 rename examples/basic_p4/{l2switch => l3switch}/runtime.txt (100%)
 rename examples/basic_p4/{l2switch => l3switch}/spec.yaml (87%)

diff --git a/examples/basic_p4/l2switch/main.p4 b/examples/basic_p4/l3switch/main.p4
similarity index 100%
rename from examples/basic_p4/l2switch/main.p4
rename to examples/basic_p4/l3switch/main.p4
diff --git a/examples/basic_p4/l2switch/runtime.txt b/examples/basic_p4/l3switch/runtime.txt
similarity index 100%
rename from examples/basic_p4/l2switch/runtime.txt
rename to examples/basic_p4/l3switch/runtime.txt
diff --git a/examples/basic_p4/l2switch/spec.yaml b/examples/basic_p4/l3switch/spec.yaml
similarity index 87%
rename from examples/basic_p4/l2switch/spec.yaml
rename to examples/basic_p4/l3switch/spec.yaml
index e4cca06..017545c 100644
--- a/examples/basic_p4/l2switch/spec.yaml
+++ b/examples/basic_p4/l3switch/spec.yaml
@@ -31,13 +31,9 @@ nodes:
 node_configs:
 - name: P4
   cmds:
-  - cmd: ip link set vm1 address 10:00:00:00:00:10
-  - cmd: ip link set vm2 address 20:00:00:00:00:20
-  - cmd: ip link set vm3 address 30:00:00:00:00:30
   - cmd: ip addr add 192.168.10.1/24 dev vm1
   - cmd: ip addr add 192.168.20.1/24 dev vm2
   - cmd: ip addr add 192.168.30.1/24 dev vm3
-
   - cmd: p4c --target bmv2 --arch v1model /main.p4 --p4runtime-files p4info.txt -o /
   - cmd: >-
       nohup simple_switch /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
@@ -48,21 +44,18 @@ node_configs:
 
 - name: VM1
   cmds:
-  - cmd: ip link set net0 address 10:10:10:10:10:10
   - cmd: ip addr add 192.168.10.10/24 dev net0
   - cmd: ip neigh replace 192.168.10.1 lladdr 10:00:00:00:00:10 dev net0
   - cmd: ip route add default via 192.168.10.1
 
 - name: VM2
   cmds:
-  - cmd: ip link set net0 address 20:20:20:20:20:20
   - cmd: ip addr add 192.168.20.20/24 dev net0
   - cmd: ip neigh replace 192.168.20.1 lladdr 20:00:00:00:00:20 dev net0
   - cmd: ip route add default via 192.168.20.1
 
 - name: VM3
   cmds:
-  - cmd: ip link set net0 address 30:30:30:30:30:30
   - cmd: ip addr add 192.168.30.30/24 dev net0
   - cmd: ip neigh replace 192.168.30.1 lladdr 30:00:00:00:00:30 dev net0
   - cmd: ip route add default via 192.168.30.1

From b23cd5ee8ceb3140f64dc215c5a068cf0fb4d61a Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Mon, 5 Jun 2023 01:16:48 +0000
Subject: [PATCH 10/42] 2023.06.05-01:16

---
 examples/basic_p4/l3switch/spec.yaml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/examples/basic_p4/l3switch/spec.yaml b/examples/basic_p4/l3switch/spec.yaml
index 017545c..236e61f 100644
--- a/examples/basic_p4/l3switch/spec.yaml
+++ b/examples/basic_p4/l3switch/spec.yaml
@@ -9,19 +9,16 @@ nodes:
   sysctls:
   - sysctl: net.ipv4.ip_forward=0
   - sysctl: net.ipv6.conf.all.forwarding=0
-
 - name: VM1
   image: nicolaka/netshoot
   docker_run_extra_args: --entrypoint bash
   interfaces:
   - { name: net0, type: direct, args: P4#vm1, addr: 10:10:10:10:10:10 }
-
 - name: VM2
   image: nicolaka/netshoot
   docker_run_extra_args: --entrypoint bash
   interfaces:
   - { name: net0, type: direct, args: P4#vm2, addr: 20:20:20:20:20:20 }
-
 - name: VM3
   image: nicolaka/netshoot
   docker_run_extra_args: --entrypoint bash
@@ -47,13 +44,11 @@ node_configs:
   - cmd: ip addr add 192.168.10.10/24 dev net0
   - cmd: ip neigh replace 192.168.10.1 lladdr 10:00:00:00:00:10 dev net0
   - cmd: ip route add default via 192.168.10.1
-
 - name: VM2
   cmds:
   - cmd: ip addr add 192.168.20.20/24 dev net0
   - cmd: ip neigh replace 192.168.20.1 lladdr 20:00:00:00:00:20 dev net0
   - cmd: ip route add default via 192.168.20.1
-
 - name: VM3
   cmds:
   - cmd: ip addr add 192.168.30.30/24 dev net0

From 17103bf95e3b3bc4306acaa262484891cd57a108 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 01:35:02 +0000
Subject: [PATCH 11/42] 2023.06.06-01:35

---
 examples/basic_p4/l3switch/README.md | 52 ++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 examples/basic_p4/l3switch/README.md

diff --git a/examples/basic_p4/l3switch/README.md b/examples/basic_p4/l3switch/README.md
new file mode 100644
index 0000000..8aee56f
--- /dev/null
+++ b/examples/basic_p4/l3switch/README.md
@@ -0,0 +1,52 @@
+# P4Runtime and BM-CLI mapping
+
+## Misc
+```
+tables
+actions
+```
+
+## Modifying Default Table Entry
+
+bm cli of `simple_switch`
+```
+table_set_default MyIngress.ipv4_lpm MyIngress.drop
+```
+
+p4runtime cli of `simple_switch_grpc`
+```
+te = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.drop", is_default=True)
+te.modify
+```
+
+## Inserting Table Entry
+
+bm cli of `simple_switch`
+```
+table_add MyIngress.ipv4_lpm MyIngress.ipv4_forward 192.168.10.10/32 => 10:10:10:10:10:10 1
+table_add MyIngress.ipv4_lpm MyIngress.ipv4_forward 192.168.20.20/32 => 20:20:20:20:20:20 2
+table_add MyIngress.ipv4_lpm MyIngress.ipv4_forward 192.168.30.30/32 => 30:30:30:30:30:30 3
+```
+
+p4runtime cli of `simple_switch_grpc`
+```
+te = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward")
+te.match["hdr.ipv4.dstAddr"] = "192.168.20.20/32"
+te.action["dstAddr"] = '20:20:20:20:20:20'
+te.action["port"] = "2"
+te.insert
+
+te = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward")
+te.match["hdr.ipv4.dstAddr"] = "192.168.10.10/32"
+te.action["dstAddr"] = '10:10:10:10:10:10'
+te.action["port"] = "1"
+te.insert
+
+te = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward");
+te.match["hdr.ipv4.dstAddr"] = "192.168.30.30/32";
+te.action["dstAddr"] = '30:30:30:30:30:30';
+te.action["port"] = "3";
+te.insert;
+
+for te in table_entry["MyIngress.ipv4_lpm"].read(): print(te)
+```

From f3e667e13bc7498054e40402bff54a4800b61f1d Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 03:28:57 +0000
Subject: [PATCH 12/42] 2023.06.06-03:28

---
 examples/basic_p4/l3switch/README.md | 40 ++++++++++++++++------------
 1 file changed, 23 insertions(+), 17 deletions(-)

diff --git a/examples/basic_p4/l3switch/README.md b/examples/basic_p4/l3switch/README.md
index 8aee56f..bb46d37 100644
--- a/examples/basic_p4/l3switch/README.md
+++ b/examples/basic_p4/l3switch/README.md
@@ -30,23 +30,29 @@ table_add MyIngress.ipv4_lpm MyIngress.ipv4_forward 192.168.30.30/32 => 30:30:30
 
 p4runtime cli of `simple_switch_grpc`
 ```
-te = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward")
-te.match["hdr.ipv4.dstAddr"] = "192.168.20.20/32"
-te.action["dstAddr"] = '20:20:20:20:20:20'
-te.action["port"] = "2"
-te.insert
-
-te = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward")
-te.match["hdr.ipv4.dstAddr"] = "192.168.10.10/32"
-te.action["dstAddr"] = '10:10:10:10:10:10'
-te.action["port"] = "1"
-te.insert
-
-te = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward");
-te.match["hdr.ipv4.dstAddr"] = "192.168.30.30/32";
-te.action["dstAddr"] = '30:30:30:30:30:30';
-te.action["port"] = "3";
-te.insert;
+te1 = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward")
+te1.match["hdr.ipv4.dstAddr"] = "192.168.20.20/32"
+te1.action["dstAddr"] = '20:20:20:20:20:20'
+te1.action["port"] = "2"
+te1.insert()
+
+te2 = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward")
+te2.match["hdr.ipv4.dstAddr"] = "192.168.10.10/32"
+te2.action["dstAddr"] = '10:10:10:10:10:10'
+te2.action["port"] = "1"
+te2.insert()
+
+te3 = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward")
+te3.match["hdr.ipv4.dstAddr"] = "192.168.30.30/32"
+te3.action["dstAddr"] = '30:30:30:30:30:30'
+te3.action["port"] = "3"
+te3.insert()
+
+te4 = table_entry["MyIngress.ipv4_lpm"](action="MyIngress.ipv4_forward")
+te4.match["hdr.ipv4.dstAddr"] = "192.168.40.40/32"
+te4.action["dstAddr"] = 'aa:aa:aa:aa:aa:aa'
+te4.action["port"] = "255"
+te4.insert()
 
 for te in table_entry["MyIngress.ipv4_lpm"].read(): print(te)
 ```

From b84a3634b0811c4e25750f5f09a296b8a6823100 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 04:19:40 +0000
Subject: [PATCH 13/42] ex

---
 examples/basic_p4/l3switch/README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/examples/basic_p4/l3switch/README.md b/examples/basic_p4/l3switch/README.md
index bb46d37..192e16a 100644
--- a/examples/basic_p4/l3switch/README.md
+++ b/examples/basic_p4/l3switch/README.md
@@ -1,5 +1,14 @@
 # P4Runtime and BM-CLI mapping
 
+## Running p4runtime-shell
+
+```
+// copy p4info.txt and main.json
+docker run -it --net container:P4 --name p4r -v /tmp/P4runtime-nanoswitch/:/tmp/ myproj/p4rt-sh-dev /bin/bash
+source $VENV/bin/activate
+/p4runtime-sh/p4runtime-sh --grpc-addr 0.0.0.0:9559 --device-id 0 --election-id 0,1 --config /p4info.txt,/main.json
+```
+
 ## Misc
 ```
 tables

From 7503c8e79ba5a1b0571906849df57b8445983c45 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 04:29:18 +0000
Subject: [PATCH 14/42] 2023.06.06-04:29

---
 examples/basic_p4/l3switch/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/basic_p4/l3switch/README.md b/examples/basic_p4/l3switch/README.md
index 192e16a..c5b8d5b 100644
--- a/examples/basic_p4/l3switch/README.md
+++ b/examples/basic_p4/l3switch/README.md
@@ -3,6 +3,7 @@
 ## Running p4runtime-shell
 
 ```
+docker exec P4 simple_switch_grpc /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3 --nanolog ipc:///tmp/bm-0-log.ipc --log-console -L debug --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc -- --cpu-port 255
 // copy p4info.txt and main.json
 docker run -it --net container:P4 --name p4r -v /tmp/P4runtime-nanoswitch/:/tmp/ myproj/p4rt-sh-dev /bin/bash
 source $VENV/bin/activate

From d3f52cd798399906ea21b8fb02e0d99c462692cf Mon Sep 17 00:00:00 2001
From: Hiroki SHIROKURA <slank.dev@gmail.com>
Date: Tue, 6 Jun 2023 13:51:05 +0900
Subject: [PATCH 15/42] Update README.md

---
 examples/basic_p4/l3switch/README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/basic_p4/l3switch/README.md b/examples/basic_p4/l3switch/README.md
index c5b8d5b..dfee0c6 100644
--- a/examples/basic_p4/l3switch/README.md
+++ b/examples/basic_p4/l3switch/README.md
@@ -8,6 +8,8 @@ docker exec P4 simple_switch_grpc /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3 --nanolo
 docker run -it --net container:P4 --name p4r -v /tmp/P4runtime-nanoswitch/:/tmp/ myproj/p4rt-sh-dev /bin/bash
 source $VENV/bin/activate
 /p4runtime-sh/p4runtime-sh --grpc-addr 0.0.0.0:9559 --device-id 0 --election-id 0,1 --config /p4info.txt,/main.json
+/p4runtime-sh/p4runtime-sh --grpc-addr 0.0.0.0:9559 --device-id 0 --election-id 0,1 --config /p4c/p4info.txt,/main.json
+
 ```
 
 ## Misc

From 58e5d01c990c9413b9252c57651ce98ad6079453 Mon Sep 17 00:00:00 2001
From: Hiroki SHIROKURA <slank.dev@gmail.com>
Date: Tue, 6 Jun 2023 14:18:57 +0900
Subject: [PATCH 16/42] Update README.md

---
 examples/basic_p4/l3switch/README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/examples/basic_p4/l3switch/README.md b/examples/basic_p4/l3switch/README.md
index dfee0c6..b696af5 100644
--- a/examples/basic_p4/l3switch/README.md
+++ b/examples/basic_p4/l3switch/README.md
@@ -13,9 +13,13 @@ source $VENV/bin/activate
 ```
 
 ## Misc
+
+https://github.com/p4lang/p4runtime-shell#available-commands
+https://github.com/p4lang/p4runtime-shell/blob/main/usage/packet_io.md
 ```
 tables
 actions
+packet_in.sniff(lambda m: print(m), timeout=1)
 ```
 
 ## Modifying Default Table Entry

From d802bbfc80d34c2bc7e741db85feed7d796f40ba Mon Sep 17 00:00:00 2001
From: Hiroki SHIROKURA <slank.dev@gmail.com>
Date: Tue, 6 Jun 2023 14:19:15 +0900
Subject: [PATCH 17/42] Update README.md

---
 examples/basic_p4/l3switch/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/examples/basic_p4/l3switch/README.md b/examples/basic_p4/l3switch/README.md
index b696af5..7993dc9 100644
--- a/examples/basic_p4/l3switch/README.md
+++ b/examples/basic_p4/l3switch/README.md
@@ -14,8 +14,9 @@ source $VENV/bin/activate
 
 ## Misc
 
-https://github.com/p4lang/p4runtime-shell#available-commands
-https://github.com/p4lang/p4runtime-shell/blob/main/usage/packet_io.md
+- https://github.com/p4lang/p4runtime-shell#available-commands
+- https://github.com/p4lang/p4runtime-shell/blob/main/usage/packet_io.md
+
 ```
 tables
 actions

From f50434e7f7ca5047cb81bc47b07066030be68b94 Mon Sep 17 00:00:00 2001
From: Hiroki SHIROKURA <slank.dev@gmail.com>
Date: Tue, 6 Jun 2023 14:29:52 +0900
Subject: [PATCH 18/42] Update README.md

---
 examples/basic_p4/l3switch/README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/basic_p4/l3switch/README.md b/examples/basic_p4/l3switch/README.md
index 7993dc9..919f445 100644
--- a/examples/basic_p4/l3switch/README.md
+++ b/examples/basic_p4/l3switch/README.md
@@ -8,8 +8,11 @@ docker exec P4 simple_switch_grpc /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3 --nanolo
 docker run -it --net container:P4 --name p4r -v /tmp/P4runtime-nanoswitch/:/tmp/ myproj/p4rt-sh-dev /bin/bash
 source $VENV/bin/activate
 /p4runtime-sh/p4runtime-sh --grpc-addr 0.0.0.0:9559 --device-id 0 --election-id 0,1 --config /p4info.txt,/main.json
-/p4runtime-sh/p4runtime-sh --grpc-addr 0.0.0.0:9559 --device-id 0 --election-id 0,1 --config /p4c/p4info.txt,/main.json
+```
 
+for all in one image
+```
+/p4runtime-sh/p4runtime-sh --grpc-addr 0.0.0.0:9559 --device-id 0 --election-id 0,1 --config /p4c/p4info.txt,/main.json
 ```
 
 ## Misc

From dca1cb2ac641a817674fc832d469baa07f35e67d Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 05:09:13 +0000
Subject: [PATCH 19/42] 2023.06.06-05:09

---
 examples/basic_p4/l3switch/spec.yaml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/examples/basic_p4/l3switch/spec.yaml b/examples/basic_p4/l3switch/spec.yaml
index 236e61f..19daebe 100644
--- a/examples/basic_p4/l3switch/spec.yaml
+++ b/examples/basic_p4/l3switch/spec.yaml
@@ -32,12 +32,20 @@ node_configs:
   - cmd: ip addr add 192.168.20.1/24 dev vm2
   - cmd: ip addr add 192.168.30.1/24 dev vm3
   - cmd: p4c --target bmv2 --arch v1model /main.p4 --p4runtime-files p4info.txt -o /
+
+  ## BM-CLI
+  #- cmd: >-
+  #    nohup simple_switch /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
+  #    --nanolog ipc:///tmp/bm-0-log.ipc --log-console -L debug
+  #    --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc &
+  #- cmd: wait.sh
+  #- cmd: sh -c "cat /runtime.txt | bm_CLI"
+
+  ## P4Runtime
   - cmd: >-
-      nohup simple_switch /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
+      nohup simple_switch_grpc /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
       --nanolog ipc:///tmp/bm-0-log.ipc --log-console -L debug
-      --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc &
-  - cmd: wait.sh
-  - cmd: sh -c "cat /runtime.txt | bm_CLI"
+      --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc -- --cpu-port 255 &
 
 - name: VM1
   cmds:

From 99e178e4b21ba0028228791ed9069ca370432ec2 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 11:09:01 +0000
Subject: [PATCH 20/42] img

---
 Dockerfiles/p4bmv2/Dockerfile   | 11 +++++++++++
 Dockerfiles/p4bmv2/p4runtime-sh | 19 +++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 Dockerfiles/p4bmv2/p4runtime-sh

diff --git a/Dockerfiles/p4bmv2/Dockerfile b/Dockerfiles/p4bmv2/Dockerfile
index 05a88f2..82fce98 100644
--- a/Dockerfiles/p4bmv2/Dockerfile
+++ b/Dockerfiles/p4bmv2/Dockerfile
@@ -1,5 +1,16 @@
+FROM p4lang/p4runtime-sh:latest as deps
+
 # p4lang/behavioral-model is included by p4lang/p4c
 FROM p4lang/p4c
 RUN apt-get update -y && apt-get install -y iproute2 vim
 COPY ./wait.sh /usr/bin/wait.sh
 RUN chmod +x /usr/bin/wait.sh
+
+ENV VENV /p4runtime-sh/venv
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl git python3 python3-venv && rm -rf /var/cache/apt/* /var/lib/apt/lists/*
+COPY --from=deps /p4runtime-sh /p4runtime-sh
+COPY ./p4runtime-sh /p4runtime-sh/p4runtime-sh
+RUN pip3 install p4runtime-shell && chmod +x /p4runtime-sh/p4runtime-sh
+
+RUN apt-get update -y && apt-get install -y tcpdump
diff --git a/Dockerfiles/p4bmv2/p4runtime-sh b/Dockerfiles/p4bmv2/p4runtime-sh
new file mode 100644
index 0000000..cfe5f85
--- /dev/null
+++ b/Dockerfiles/p4bmv2/p4runtime-sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+# Copyright 2019 Barefoot Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import p4runtime_sh.shell as sh
+sh.main()

From 7858afac4556b85198b9ee11e3d47956ef74a31d Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 11:13:13 +0000
Subject: [PATCH 21/42] 2023.06.06-11:13

---
 examples/basic_p4/l3switch/Makefile | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 examples/basic_p4/l3switch/Makefile

diff --git a/examples/basic_p4/l3switch/Makefile b/examples/basic_p4/l3switch/Makefile
new file mode 100644
index 0000000..a41208f
--- /dev/null
+++ b/examples/basic_p4/l3switch/Makefile
@@ -0,0 +1,3 @@
+r:
+	docker cp main.py P4:/
+	docker exec -it P4 python3 /main.py

From 6cbef8604cd453d0ee1cae9d8d5227ed5a0f9793 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 11:13:16 +0000
Subject: [PATCH 22/42] 2023.06.06-11:13

---
 examples/basic_p4/l3switch/main.py | 62 ++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 examples/basic_p4/l3switch/main.py

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
new file mode 100644
index 0000000..3cf1472
--- /dev/null
+++ b/examples/basic_p4/l3switch/main.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+import os
+import time
+import struct
+import pprint
+import subprocess
+from fcntl import ioctl
+import p4runtime_sh.shell as sh
+from p4runtime_sh.p4runtime import P4RuntimeClient
+
+def openTun(tunName):
+    tun = open("/dev/net/tun", "r+b", buffering=0)
+    LINUX_IFF_TAP = 0x0002
+    LINUX_IFF_NO_PI = 0x1000
+    LINUX_TUNSETIFF = 0x400454CA
+    flags = LINUX_IFF_TAP | LINUX_IFF_NO_PI
+    ifs = struct.pack("16sH22s", tunName.encode("utf-8"), flags, b"")
+    ioctl(tun, LINUX_TUNSETIFF, ifs)
+    subprocess.check_call(f'ip link set {tunName} up', shell=True)
+    return tun
+
+sh.setup(
+  device_id=0,
+  grpc_addr='localhost:9559',
+  election_id=(0, 1),
+  config=sh.FwdPipeConfig('/p4c/p4info.txt', '/main.json'))
+te1 = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te1.match["hdr.ipv4.dstAddr"] = "192.168.10.10/32"
+te1.action["dstAddr"] = '10:10:10:10:10:10'
+te1.action["port"] = "1"
+te1.insert()
+te2 = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te2.match["hdr.ipv4.dstAddr"] = "192.168.20.20/32"
+te2.action["dstAddr"] = '20:20:20:20:20:20'
+te2.action["port"] = "2"
+te2.insert()
+te3 = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te3.match["hdr.ipv4.dstAddr"] = "192.168.30.30/32"
+te3.action["dstAddr"] = '30:30:30:30:30:30'
+te3.action["port"] = "3"
+te3.insert()
+te4 = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te4.match["hdr.ipv4.dstAddr"] = "192.168.40.40/32"
+te4.action["dstAddr"] = 'aa:aa:aa:aa:aa:aa'
+te4.action["port"] = "255"
+te4.insert()
+sh.teardown()
+
+## PREPARE TAP
+tap = openTun("hoge")
+
+## MAIN ROUTING
+client = P4RuntimeClient(
+  device_id=0,
+  grpc_addr='localhost:9559',
+  election_id=(0, 1))
+while True:
+    rep = client.get_stream_packet("packet", timeout=1)
+    if rep is not None:
+        #print("PacketIN")
+        #pprint.pprint(rep.packet.payload)
+        tap.write(rep.packet.payload)

From f0b4524fad159a9eaf98f20bca9fb7ee1da2e317 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 11:21:32 +0000
Subject: [PATCH 23/42] 2023.06.06-11:21

---
 examples/basic_p4/l3switch/spec.yaml | 29 ++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/examples/basic_p4/l3switch/spec.yaml b/examples/basic_p4/l3switch/spec.yaml
index 19daebe..b7c64ac 100644
--- a/examples/basic_p4/l3switch/spec.yaml
+++ b/examples/basic_p4/l3switch/spec.yaml
@@ -28,10 +28,9 @@ nodes:
 node_configs:
 - name: P4
   cmds:
-  - cmd: ip addr add 192.168.10.1/24 dev vm1
-  - cmd: ip addr add 192.168.20.1/24 dev vm2
-  - cmd: ip addr add 192.168.30.1/24 dev vm3
-  - cmd: p4c --target bmv2 --arch v1model /main.p4 --p4runtime-files p4info.txt -o /
+  - cmd: >-
+      p4c --target bmv2 --arch v1model /main.p4
+      --p4runtime-files p4info.txt -o /
 
   ## BM-CLI
   #- cmd: >-
@@ -45,25 +44,27 @@ node_configs:
   - cmd: >-
       nohup simple_switch_grpc /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
       --nanolog ipc:///tmp/bm-0-log.ipc --log-console -L debug
-      --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc -- --cpu-port 255 &
+      --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc
+      -- --cpu-port 255 &
 
 - name: VM1
   cmds:
-  - cmd: ip addr add 192.168.10.10/24 dev net0
-  - cmd: ip neigh replace 192.168.10.1 lladdr 10:00:00:00:00:10 dev net0
-  - cmd: ip route add default via 192.168.10.1
+  - cmd: ip addr add 10.0.1.2/24 dev net0
+  - cmd: ip neigh replace 10.0.1.1 lladdr 10:00:00:00:00:10 dev net0
+  - cmd: ip route add default via 10.0.1.1
 - name: VM2
   cmds:
-  - cmd: ip addr add 192.168.20.20/24 dev net0
-  - cmd: ip neigh replace 192.168.20.1 lladdr 20:00:00:00:00:20 dev net0
-  - cmd: ip route add default via 192.168.20.1
+  - cmd: ip addr add 10.0.2.2/24 dev net0
+  - cmd: ip neigh replace 10.0.2.1 lladdr 20:00:00:00:00:20 dev net0
+  - cmd: ip route add default via 10.0.2.1
 - name: VM3
   cmds:
-  - cmd: ip addr add 192.168.30.30/24 dev net0
-  - cmd: ip neigh replace 192.168.30.1 lladdr 30:00:00:00:00:30 dev net0
-  - cmd: ip route add default via 192.168.30.1
+  - cmd: ip addr add 10.0.3.2/24 dev net0
+  - cmd: ip neigh replace 10.0.3.1 lladdr 30:00:00:00:00:30 dev net0
+  - cmd: ip route add default via 10.0.3.1
 
 postinit:
   cmds:
+  - cmd: docker cp main.py P4:/main.py
   - cmd: docker cp main.p4 P4:/main.p4
   - cmd: docker cp runtime.txt P4:/runtime.txt

From 5c2547646f732e30c8b58170027a4d06123f8203 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 11:21:35 +0000
Subject: [PATCH 24/42] 2023.06.06-11:21

---
 examples/basic_p4/l3switch/main.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index 3cf1472..2b35c94 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -25,23 +25,24 @@ def openTun(tunName):
   election_id=(0, 1),
   config=sh.FwdPipeConfig('/p4c/p4info.txt', '/main.json'))
 te1 = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
-te1.match["hdr.ipv4.dstAddr"] = "192.168.10.10/32"
+te1.match["hdr.ipv4.dstAddr"] = "10.0.1.2/32"
 te1.action["dstAddr"] = '10:10:10:10:10:10'
 te1.action["port"] = "1"
 te1.insert()
 te2 = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
-te2.match["hdr.ipv4.dstAddr"] = "192.168.20.20/32"
+te2.match["hdr.ipv4.dstAddr"] = "10.0.2.2/32"
 te2.action["dstAddr"] = '20:20:20:20:20:20'
 te2.action["port"] = "2"
 te2.insert()
 te3 = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
-te3.match["hdr.ipv4.dstAddr"] = "192.168.30.30/32"
+te3.match["hdr.ipv4.dstAddr"] = "10.0.3.2/32"
 te3.action["dstAddr"] = '30:30:30:30:30:30'
 te3.action["port"] = "3"
 te3.insert()
 te4 = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
-te4.match["hdr.ipv4.dstAddr"] = "192.168.40.40/32"
-te4.action["dstAddr"] = 'aa:aa:aa:aa:aa:aa'
+te4.match["hdr.ipv4.dstAddr"] = "10.0.0.0/8"
+te4.action["dstAddr"] = 'ff:ff:ff:ff:ff:ff'
+#te4.action["dstAddr"] = 'aa:aa:aa:aa:aa:aa'
 te4.action["port"] = "255"
 te4.insert()
 sh.teardown()

From 3fe59cc9a520499e12a0c9812ca92a8497778e52 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 11:27:01 +0000
Subject: [PATCH 25/42] 2023.06.06-11:27

---
 examples/basic_p4/l3switch/main.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index 2b35c94..d810f2e 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -39,16 +39,29 @@ def openTun(tunName):
 te3.action["dstAddr"] = '30:30:30:30:30:30'
 te3.action["port"] = "3"
 te3.insert()
-te4 = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
-te4.match["hdr.ipv4.dstAddr"] = "10.0.0.0/8"
-te4.action["dstAddr"] = 'ff:ff:ff:ff:ff:ff'
-#te4.action["dstAddr"] = 'aa:aa:aa:aa:aa:aa'
-te4.action["port"] = "255"
-te4.insert()
+te = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te.match["hdr.ipv4.dstAddr"] = "10.0.1.1/32"
+te.action["dstAddr"] = 'ff:ff:ff:ff:ff:ff'
+te.action["port"] = "255"
+te.insert()
+te = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te.match["hdr.ipv4.dstAddr"] = "10.0.2.1/32"
+te.action["dstAddr"] = 'ff:ff:ff:ff:ff:ff'
+te.action["port"] = "255"
+te.insert()
+te = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te.match["hdr.ipv4.dstAddr"] = "10.0.3.1/32"
+te.action["dstAddr"] = 'ff:ff:ff:ff:ff:ff'
+te.action["port"] = "255"
+te.insert()
 sh.teardown()
 
 ## PREPARE TAP
-tap = openTun("hoge")
+swp = [
+  openTun("swp1"),
+  openTun("swp2"),
+  openTun("swp3"),
+]
 
 ## MAIN ROUTING
 client = P4RuntimeClient(
@@ -60,4 +73,4 @@ def openTun(tunName):
     if rep is not None:
         #print("PacketIN")
         #pprint.pprint(rep.packet.payload)
-        tap.write(rep.packet.payload)
+        swp[0].write(rep.packet.payload)

From 2be642cbed92a23d3abe47cc11cf08f12a8f8fc2 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 11:40:26 +0000
Subject: [PATCH 26/42] 2023.06.06-11:40

---
 examples/basic_p4/l3switch/spec.yaml | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/examples/basic_p4/l3switch/spec.yaml b/examples/basic_p4/l3switch/spec.yaml
index b7c64ac..f22bcd6 100644
--- a/examples/basic_p4/l3switch/spec.yaml
+++ b/examples/basic_p4/l3switch/spec.yaml
@@ -31,22 +31,13 @@ node_configs:
   - cmd: >-
       p4c --target bmv2 --arch v1model /main.p4
       --p4runtime-files p4info.txt -o /
-
-  ## BM-CLI
-  #- cmd: >-
-  #    nohup simple_switch /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
-  #    --nanolog ipc:///tmp/bm-0-log.ipc --log-console -L debug
-  #    --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc &
-  #- cmd: wait.sh
-  #- cmd: sh -c "cat /runtime.txt | bm_CLI"
-
-  ## P4Runtime
   - cmd: >-
       nohup simple_switch_grpc /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
       --nanolog ipc:///tmp/bm-0-log.ipc --log-console -L debug
       --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc
       -- --cpu-port 255 &
-
+  - cmd: >-
+      nohup python3 /main.py &
 - name: VM1
   cmds:
   - cmd: ip addr add 10.0.1.2/24 dev net0

From 47764159790936757c03d6e9d8621fb25b5c87c7 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 11:44:29 +0000
Subject: [PATCH 27/42] 2023.06.06-11:44

---
 examples/basic_p4/l3switch/Makefile | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/examples/basic_p4/l3switch/Makefile b/examples/basic_p4/l3switch/Makefile
index a41208f..b86e14b 100644
--- a/examples/basic_p4/l3switch/Makefile
+++ b/examples/basic_p4/l3switch/Makefile
@@ -1,3 +1,13 @@
-r:
+c:
 	docker cp main.py P4:/
 	docker exec -it P4 python3 /main.py
+
+d:
+	docker cp main.p4 P4:/
+	docker exec P4 p4c --target bmv2 --arch v1model /main.p4 --p4runtime-files p4info.txt -o /
+	docker exec P4 simple_switch_grpc /main.json \
+		-i 1@vm1 -i 2@vm2 -i 3@vm3 \
+		--nanolog ipc:///tmp/bm-0-log.ipc \
+		--log-console -L debug \
+		--notifications-addr ipc:///tmp/bmv2-0-notifications.ipc \
+		-- --cpu-port 255

From 11161703e1e448323a655190568fdbf0a037b760 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:24:10 +0000
Subject: [PATCH 28/42] img

---
 Dockerfiles/p4bmv2/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfiles/p4bmv2/Dockerfile b/Dockerfiles/p4bmv2/Dockerfile
index 82fce98..e449d44 100644
--- a/Dockerfiles/p4bmv2/Dockerfile
+++ b/Dockerfiles/p4bmv2/Dockerfile
@@ -13,4 +13,4 @@ COPY --from=deps /p4runtime-sh /p4runtime-sh
 COPY ./p4runtime-sh /p4runtime-sh/p4runtime-sh
 RUN pip3 install p4runtime-shell && chmod +x /p4runtime-sh/p4runtime-sh
 
-RUN apt-get update -y && apt-get install -y tcpdump
+RUN apt-get update -y && apt-get install -y tcpdump psmisc

From 6302a8622e600460c1bb64048ae87012ff02dfb9 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:25:10 +0000
Subject: [PATCH 29/42] img

---
 Dockerfiles/p4bmv2/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfiles/p4bmv2/Dockerfile b/Dockerfiles/p4bmv2/Dockerfile
index e449d44..b3333bc 100644
--- a/Dockerfiles/p4bmv2/Dockerfile
+++ b/Dockerfiles/p4bmv2/Dockerfile
@@ -13,4 +13,4 @@ COPY --from=deps /p4runtime-sh /p4runtime-sh
 COPY ./p4runtime-sh /p4runtime-sh/p4runtime-sh
 RUN pip3 install p4runtime-shell && chmod +x /p4runtime-sh/p4runtime-sh
 
-RUN apt-get update -y && apt-get install -y tcpdump psmisc
+RUN apt-get update -y && apt-get install -y tcpdump psmisc iputils-ping net-tools

From ed3b6be3cd789cfc00a72ccc608d5df6d224d471 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:25:18 +0000
Subject: [PATCH 30/42] 2023.06.06-12:25

---
 examples/basic_p4/l3switch/Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/examples/basic_p4/l3switch/Makefile b/examples/basic_p4/l3switch/Makefile
index b86e14b..7e8918d 100644
--- a/examples/basic_p4/l3switch/Makefile
+++ b/examples/basic_p4/l3switch/Makefile
@@ -1,11 +1,13 @@
 c:
 	docker cp main.py P4:/
+	docker exec P4 killall -9 python3 || true
 	docker exec -it P4 python3 /main.py
 
 d:
 	docker cp main.p4 P4:/
+	docker exec P4 killall -9 simple_switch_grpc || true
 	docker exec P4 p4c --target bmv2 --arch v1model /main.p4 --p4runtime-files p4info.txt -o /
-	docker exec P4 simple_switch_grpc /main.json \
+	docker exec -it P4 simple_switch_grpc /main.json \
 		-i 1@vm1 -i 2@vm2 -i 3@vm3 \
 		--nanolog ipc:///tmp/bm-0-log.ipc \
 		--log-console -L debug \

From 93d1e6b88087945fbaf2885c418528c66e4ecb00 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:25:26 +0000
Subject: [PATCH 31/42] 2023.06.06-12:25

---
 examples/basic_p4/l3switch/spec.yaml | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/examples/basic_p4/l3switch/spec.yaml b/examples/basic_p4/l3switch/spec.yaml
index f22bcd6..3b313ae 100644
--- a/examples/basic_p4/l3switch/spec.yaml
+++ b/examples/basic_p4/l3switch/spec.yaml
@@ -28,16 +28,17 @@ nodes:
 node_configs:
 - name: P4
   cmds:
-  - cmd: >-
-      p4c --target bmv2 --arch v1model /main.p4
-      --p4runtime-files p4info.txt -o /
-  - cmd: >-
-      nohup simple_switch_grpc /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
-      --nanolog ipc:///tmp/bm-0-log.ipc --log-console -L debug
-      --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc
-      -- --cpu-port 255 &
-  - cmd: >-
-      nohup python3 /main.py &
+  - cmd: echo hello
+  # - cmd: >-
+  #     p4c --target bmv2 --arch v1model /main.p4
+  #     --p4runtime-files p4info.txt -o /
+  # - cmd: >-
+  #     nohup simple_switch_grpc /main.json -i 1@vm1 -i 2@vm2 -i 3@vm3
+  #     --nanolog ipc:///tmp/bm-0-log.ipc --log-console -L debug
+  #     --notifications-addr ipc:///tmp/bmv2-0-notifications.ipc
+  #     -- --cpu-port 255 &
+  # - cmd: >-
+  #     nohup python3 /main.py &
 - name: VM1
   cmds:
   - cmd: ip addr add 10.0.1.2/24 dev net0

From c49f8cd5a6d899d273baad552a39aa5eef7c7eba Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:26:36 +0000
Subject: [PATCH 32/42] 2023.06.06-12:26

---
 examples/basic_p4/l3switch/main.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index d810f2e..d147dff 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -39,20 +39,14 @@ def openTun(tunName):
 te3.action["dstAddr"] = '30:30:30:30:30:30'
 te3.action["port"] = "3"
 te3.insert()
-te = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.to_controller")
 te.match["hdr.ipv4.dstAddr"] = "10.0.1.1/32"
-te.action["dstAddr"] = 'ff:ff:ff:ff:ff:ff'
-te.action["port"] = "255"
 te.insert()
-te = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.to_controller")
 te.match["hdr.ipv4.dstAddr"] = "10.0.2.1/32"
-te.action["dstAddr"] = 'ff:ff:ff:ff:ff:ff'
-te.action["port"] = "255"
 te.insert()
-te = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.ipv4_forward")
+te = sh.TableEntry("MyIngress.ipv4_lpm")(action="MyIngress.to_controller")
 te.match["hdr.ipv4.dstAddr"] = "10.0.3.1/32"
-te.action["dstAddr"] = 'ff:ff:ff:ff:ff:ff'
-te.action["port"] = "255"
 te.insert()
 sh.teardown()
 
@@ -71,6 +65,11 @@ def openTun(tunName):
 while True:
     rep = client.get_stream_packet("packet", timeout=1)
     if rep is not None:
-        #print("PacketIN")
-        #pprint.pprint(rep.packet.payload)
-        swp[0].write(rep.packet.payload)
+        #pprint.pprint(rep)
+        #print(type(rep.packet.metadata))
+        #pprint.pprint(rep.packet.metadata[0].value)
+        #pprint.pprint(rep.packet.metadata[1].value)
+        v = struct.unpack("@c", rep.packet.metadata[0].value)
+        v = int.from_bytes(v[0], "little")
+        print(f"PacketIN({v})")
+        swp[v-1].write(rep.packet.payload)

From f6ff6838207c93fff29b5e5393b9f5cd5d38809f Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:27:01 +0000
Subject: [PATCH 33/42] 2023.06.06-12:27

---
 examples/basic_p4/l3switch/main.p4 | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/examples/basic_p4/l3switch/main.p4 b/examples/basic_p4/l3switch/main.p4
index 4a3898c..7e7d66f 100644
--- a/examples/basic_p4/l3switch/main.p4
+++ b/examples/basic_p4/l3switch/main.p4
@@ -1,7 +1,7 @@
 /* -*- P4_16 -*- */
 #include <core.p4>
 #include <v1model.p4>
-
+#define CPU_PORT 255
 const bit<16> TYPE_IPV4 = 0x800;
 
 /*************************************************************************
@@ -12,6 +12,12 @@ typedef bit<9>  egressSpec_t;
 typedef bit<48> macAddr_t;
 typedef bit<32> ip4Addr_t;
 
+@controller_header("packet_in")
+header packet_in_header_t {
+    bit<9> ingress_port;
+    bit<7> _pad;
+}
+
 header ethernet_t {
     macAddr_t dstAddr;
     macAddr_t srcAddr;
@@ -34,12 +40,14 @@ header ipv4_t {
 }
 
 struct metadata {
-    /* empty */
+    bit<9> ingress_port;
+    bit<7> _pad;
 }
 
 struct headers {
-    ethernet_t   ethernet;
-    ipv4_t       ipv4;
+    ethernet_t         ethernet;
+    ipv4_t             ipv4;
+    packet_in_header_t packet_in;
 }
 
 /*************************************************************************
@@ -97,6 +105,12 @@ control MyIngress(inout headers hdr,
         hdr.ipv4.ttl = hdr.ipv4.ttl - 1;
     }
 
+    action to_controller() {
+        standard_metadata.egress_spec = CPU_PORT;
+        hdr.packet_in.setValid();
+        hdr.packet_in.ingress_port = standard_metadata.ingress_port;
+    }
+
     table ipv4_lpm {
         key = {
             hdr.ipv4.dstAddr: lpm;
@@ -105,6 +119,7 @@ control MyIngress(inout headers hdr,
             ipv4_forward;
             drop;
             NoAction;
+            to_controller;
         }
         size = 1024;
         default_action = drop();
@@ -124,7 +139,8 @@ control MyIngress(inout headers hdr,
 control MyEgress(inout headers hdr,
                  inout metadata meta,
                  inout standard_metadata_t standard_metadata) {
-    apply {  }
+    apply {
+    }
 }
 
 /*************************************************************************
@@ -157,6 +173,7 @@ control MyComputeChecksum(inout headers  hdr, inout metadata meta) {
 
 control MyDeparser(packet_out packet, in headers hdr) {
     apply {
+        packet.emit(hdr.packet_in);
         packet.emit(hdr.ethernet);
         packet.emit(hdr.ipv4);
     }

From d90dc1a17ebd6570915e874abd74dfe175731d82 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:34:49 +0000
Subject: [PATCH 34/42] 2023.06.06-12:34

---
 examples/basic_p4/l3switch/spec.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/examples/basic_p4/l3switch/spec.yaml b/examples/basic_p4/l3switch/spec.yaml
index 3b313ae..acbb6e9 100644
--- a/examples/basic_p4/l3switch/spec.yaml
+++ b/examples/basic_p4/l3switch/spec.yaml
@@ -3,9 +3,9 @@ nodes:
   image: tinynetwork/p4bmv2:develop
   docker_run_extra_args: --entrypoint bash
   interfaces:
-  - { name: vm1, type: direct, args: VM1#net0, addr: 10:00:00:00:00:10 }
-  - { name: vm2, type: direct, args: VM2#net0, addr: 20:00:00:00:00:20 }
-  - { name: vm3, type: direct, args: VM3#net0, addr: 30:00:00:00:00:30 }
+  - { name: vm1, type: direct, args: VM1#net0, addr: 52:54:00:00:00:01 }
+  - { name: vm2, type: direct, args: VM2#net0, addr: 52:54:00:00:00:02 }
+  - { name: vm3, type: direct, args: VM3#net0, addr: 52:54:00:00:00:03 }
   sysctls:
   - sysctl: net.ipv4.ip_forward=0
   - sysctl: net.ipv6.conf.all.forwarding=0
@@ -42,17 +42,17 @@ node_configs:
 - name: VM1
   cmds:
   - cmd: ip addr add 10.0.1.2/24 dev net0
-  - cmd: ip neigh replace 10.0.1.1 lladdr 10:00:00:00:00:10 dev net0
+  - cmd: ip neigh replace 10.0.1.1 lladdr 52:54:00:00:00:01 dev net0
   - cmd: ip route add default via 10.0.1.1
 - name: VM2
   cmds:
   - cmd: ip addr add 10.0.2.2/24 dev net0
-  - cmd: ip neigh replace 10.0.2.1 lladdr 20:00:00:00:00:20 dev net0
+  - cmd: ip neigh replace 10.0.2.1 lladdr 52:54:00:00:00:02 dev net0
   - cmd: ip route add default via 10.0.2.1
 - name: VM3
   cmds:
   - cmd: ip addr add 10.0.3.2/24 dev net0
-  - cmd: ip neigh replace 10.0.3.1 lladdr 30:00:00:00:00:30 dev net0
+  - cmd: ip neigh replace 10.0.3.1 lladdr 52:54:00:00:00:03 dev net0
   - cmd: ip route add default via 10.0.3.1
 
 postinit:

From 01b303f11ebaf5298c3c7fe6f31fdf6b93108e65 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:34:54 +0000
Subject: [PATCH 35/42] 2023.06.06-12:34

---
 examples/basic_p4/l3switch/main.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index d147dff..9ef0ef3 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -8,7 +8,7 @@
 import p4runtime_sh.shell as sh
 from p4runtime_sh.p4runtime import P4RuntimeClient
 
-def openTun(tunName):
+def openTun(tunName, macaddr, ipaddr):
     tun = open("/dev/net/tun", "r+b", buffering=0)
     LINUX_IFF_TAP = 0x0002
     LINUX_IFF_NO_PI = 0x1000
@@ -16,7 +16,9 @@ def openTun(tunName):
     flags = LINUX_IFF_TAP | LINUX_IFF_NO_PI
     ifs = struct.pack("16sH22s", tunName.encode("utf-8"), flags, b"")
     ioctl(tun, LINUX_TUNSETIFF, ifs)
+    subprocess.check_call(f'ip link set {tunName} address {macaddr}', shell=True)
     subprocess.check_call(f'ip link set {tunName} up', shell=True)
+    subprocess.check_call(f'ip addr add {ipaddr} dev {tunName}', shell=True)
     return tun
 
 sh.setup(
@@ -52,9 +54,9 @@ def openTun(tunName):
 
 ## PREPARE TAP
 swp = [
-  openTun("swp1"),
-  openTun("swp2"),
-  openTun("swp3"),
+  openTun("swp1", "52:54:00:00:00:01", "10.0.1.1/24"),
+  openTun("swp2", "52:54:00:00:00:02", "10.0.2.1/24"),
+  openTun("swp3", "52:54:00:00:00:03", "10.0.3.1/24"),
 ]
 
 ## MAIN ROUTING

From 9ab7bd5e145f8e9ef696346080dbc67fa87fc883 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:39:16 +0000
Subject: [PATCH 36/42] 2023.06.06-12:39

---
 examples/basic_p4/l3switch/main.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index 9ef0ef3..7662024 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -8,6 +8,9 @@
 import p4runtime_sh.shell as sh
 from p4runtime_sh.p4runtime import P4RuntimeClient
 
+def exec(cmd):
+    subprocess.check_call(cmd, shell=True)
+
 def openTun(tunName, macaddr, ipaddr):
     tun = open("/dev/net/tun", "r+b", buffering=0)
     LINUX_IFF_TAP = 0x0002
@@ -58,6 +61,9 @@ def openTun(tunName, macaddr, ipaddr):
   openTun("swp2", "52:54:00:00:00:02", "10.0.2.1/24"),
   openTun("swp3", "52:54:00:00:00:03", "10.0.3.1/24"),
 ]
+exec('ip nei replace 10.0.1.2 lladdr 10:10:10:10:10:10 dev swp1')
+exec('ip nei replace 10.0.2.2 lladdr 20:20:20:20:20:20 dev swp2')
+exec('ip nei replace 10.0.3.2 lladdr 30:30:30:30:30:30 dev swp3')
 
 ## MAIN ROUTING
 client = P4RuntimeClient(

From a8a4737472c15a7150493a0937b540d2e43c8706 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 12:59:08 +0000
Subject: [PATCH 37/42] 2023.06.06-12:59

---
 examples/basic_p4/l3switch/main.py | 44 ++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 11 deletions(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index 7662024..a79a581 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -4,6 +4,7 @@
 import struct
 import pprint
 import subprocess
+import threading
 from fcntl import ioctl
 import p4runtime_sh.shell as sh
 from p4runtime_sh.p4runtime import P4RuntimeClient
@@ -70,14 +71,35 @@ def openTun(tunName, macaddr, ipaddr):
   device_id=0,
   grpc_addr='localhost:9559',
   election_id=(0, 1))
-while True:
-    rep = client.get_stream_packet("packet", timeout=1)
-    if rep is not None:
-        #pprint.pprint(rep)
-        #print(type(rep.packet.metadata))
-        #pprint.pprint(rep.packet.metadata[0].value)
-        #pprint.pprint(rep.packet.metadata[1].value)
-        v = struct.unpack("@c", rep.packet.metadata[0].value)
-        v = int.from_bytes(v[0], "little")
-        print(f"PacketIN({v})")
-        swp[v-1].write(rep.packet.payload)
+
+def loop_packet_in():
+    while True:
+        rep = client.get_stream_packet("packet", timeout=1)
+        if rep is not None:
+            #pprint.pprint(rep)
+            #print(type(rep.packet.metadata))
+            #pprint.pprint(rep.packet.metadata[0].value)
+            #pprint.pprint(rep.packet.metadata[1].value)
+            v = struct.unpack("@c", rep.packet.metadata[0].value)
+            v = int.from_bytes(v[0], "little")
+            print(f"PacketIN({v})")
+            swp[v-1].write(rep.packet.payload)
+
+def loop_packet_out():
+    while True:
+        for port in swp:
+            #print(port)
+            print("out blocking start")
+            data = os.read(port.fileno(), 1500)
+            print(data)
+            print("out blocking fin")
+        time.sleep(1)
+
+thread1 = threading.Thread(target=loop_packet_in)
+thread1.start()
+
+thread2 = threading.Thread(target=loop_packet_out)
+thread2.start()
+
+thread1.join()
+thread2.join()

From fdea37bc154e842bd1a592023b7b8eeacc78fece Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 13:14:44 +0000
Subject: [PATCH 38/42] 2023.06.06-13:14

---
 examples/basic_p4/l3switch/main.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index a79a581..5c987c9 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -85,21 +85,17 @@ def loop_packet_in():
             print(f"PacketIN({v})")
             swp[v-1].write(rep.packet.payload)
 
-def loop_packet_out():
+def loop_packet_out(portIdx):
+    time.sleep(1)
+    port = swp[portIdx-1]
     while True:
-        for port in swp:
-            #print(port)
-            print("out blocking start")
-            data = os.read(port.fileno(), 1500)
-            print(data)
-            print("out blocking fin")
-        time.sleep(1)
+        data = os.read(port.fileno(), 1500)
+        print(f"PacketOut({portIdx})")
+        print(data)
 
 thread1 = threading.Thread(target=loop_packet_in)
 thread1.start()
-
-thread2 = threading.Thread(target=loop_packet_out)
-thread2.start()
-
+threading.Thread(target=loop_packet_out, args=(1,)).start()
+threading.Thread(target=loop_packet_out, args=(2,)).start()
+threading.Thread(target=loop_packet_out, args=(3,)).start()
 thread1.join()
-thread2.join()

From e85f97cb215a1c13243fba58354c304356f94ab0 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 13:15:42 +0000
Subject: [PATCH 39/42] 2023.06.06-13:15

---
 examples/basic_p4/l3switch/main.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index 5c987c9..b8eedec 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -86,7 +86,6 @@ def loop_packet_in():
             swp[v-1].write(rep.packet.payload)
 
 def loop_packet_out(portIdx):
-    time.sleep(1)
     port = swp[portIdx-1]
     while True:
         data = os.read(port.fileno(), 1500)

From 5c59570bb8758b331e4f6405421548af256e08af Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 13:15:51 +0000
Subject: [PATCH 40/42] 2023.06.06-13:15

---
 examples/basic_p4/l3switch/main.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index b8eedec..28d81ae 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -76,10 +76,6 @@ def loop_packet_in():
     while True:
         rep = client.get_stream_packet("packet", timeout=1)
         if rep is not None:
-            #pprint.pprint(rep)
-            #print(type(rep.packet.metadata))
-            #pprint.pprint(rep.packet.metadata[0].value)
-            #pprint.pprint(rep.packet.metadata[1].value)
             v = struct.unpack("@c", rep.packet.metadata[0].value)
             v = int.from_bytes(v[0], "little")
             print(f"PacketIN({v})")

From 93b1f23ea2ce57199686764280fb5ef0e5782568 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 13:24:37 +0000
Subject: [PATCH 41/42] 2023.06.06-13:24

---
 examples/basic_p4/l3switch/main.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index 28d81ae..85182d2 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -8,6 +8,9 @@
 from fcntl import ioctl
 import p4runtime_sh.shell as sh
 from p4runtime_sh.p4runtime import P4RuntimeClient
+from p4.v1 import p4runtime_pb2
+from p4.config.v1 import p4info_pb2
+
 
 def exec(cmd):
     subprocess.check_call(cmd, shell=True)
@@ -25,6 +28,7 @@ def openTun(tunName, macaddr, ipaddr):
     subprocess.check_call(f'ip addr add {ipaddr} dev {tunName}', shell=True)
     return tun
 
+
 sh.setup(
   device_id=0,
   grpc_addr='localhost:9559',
@@ -86,7 +90,16 @@ def loop_packet_out(portIdx):
     while True:
         data = os.read(port.fileno(), 1500)
         print(f"PacketOut({portIdx})")
-        print(data)
+        req = p4runtime_pb2.StreamMessageRequest()
+        req.packet.payload = data
+        #metadata = p4runtime_pb2.PacketMetadata()
+        # metadata.metadata_id = 1
+        # metadata.value = portIdx
+        # req.packet.metadata.append(metadata)
+        # metadata.metadata_id = 3
+        # metadata.value = mcast_grp
+        # req.packet.metadata.append(metadata)
+        client.stream_out_q.put(req)
 
 thread1 = threading.Thread(target=loop_packet_in)
 thread1.start()

From 5539b2daecdea7e6952789a03e93ced818b21343 Mon Sep 17 00:00:00 2001
From: Hiroki Shirokura <hiroki.shirokura@linecorp.com>
Date: Tue, 6 Jun 2023 13:24:49 +0000
Subject: [PATCH 42/42] 2023.06.06-13:24

---
 examples/basic_p4/l3switch/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/basic_p4/l3switch/main.py b/examples/basic_p4/l3switch/main.py
index 85182d2..31f35d4 100644
--- a/examples/basic_p4/l3switch/main.py
+++ b/examples/basic_p4/l3switch/main.py
@@ -92,7 +92,8 @@ def loop_packet_out(portIdx):
         print(f"PacketOut({portIdx})")
         req = p4runtime_pb2.StreamMessageRequest()
         req.packet.payload = data
-        #metadata = p4runtime_pb2.PacketMetadata()
+        # XXX(slankdev)
+        # metadata = p4runtime_pb2.PacketMetadata()
         # metadata.metadata_id = 1
         # metadata.value = portIdx
         # req.packet.metadata.append(metadata)