Add easy way to test with K8s

Signed-off-by: utam0k <k0ma@utam0k.jp>

.gitignore

@@ -19,4 +19,4 @@ tags.temp
 *~
 
 /bundle.tar.gz
-/test.log
+/test.log

Makefile

@@ -60,7 +60,32 @@ test-oci: oci-tests rust-oci-tests
 .PHONY: test-all
 test-all: unittest featuretest oci-tests containerd-test # currently not doing rust-oci here
 
+.PHONY: test/k3s
+test/k3s: bin/k3s
+	sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo 'default_runtime_name = "youki"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo '[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo '  runtime_type = "io.containerd.runc.v2"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo '  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki.options]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo '    BinaryName = "$(PWD)/youki"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo "CONTAINERD_NAMESPACE='default'" | sudo tee /etc/systemd/system/k3s-runwasi.service.env && \
+	echo "NO_PROXY=192.168.0.0/16" | sudo tee -a /etc/systemd/system/k3s-runwasi.service.env && \
+	sudo systemctl daemon-reload && \
+	sudo systemctl restart k3s-youki && \
+	sudo bin/k3s kubectl apply -f tests/k8s/deploy.yaml
+	sudo bin/k3s kubectl wait deployment nginx-deployment --for condition=Available=True --timeout=90s && \
+	sudo bin/k3s kubectl get pods -o wide
+
+.PHONY: test/k3s/clean
+test/k3s/clean:
+	sudo bin/k3s-youki-uninstall.sh
+
 # Misc
+#
+.PHONY: bin/k3s
+bin/k3s:
+	mkdir -p bin && \
+	curl -sfL https://get.k3s.io | INSTALL_K3S_BIN_DIR=$(PWD)/bin INSTALL_K3S_SYMLINK=skip INSTALL_K3S_NAME=youki sh -
 
 .PHONY: lint
 lint:
@@ -69,7 +94,7 @@ lint:
 
 .PHONY: hack/bpftrace
 hack/bpftrace:
-	BPFTRACE_STRLEN=120 ./hack/debug.bt
+	BPFTRACE_STRLEN=125 ./hack/debug.bt
 
 .PHONY: clean
 clean:

crates/libcontainer/src/process/container_init_process.rs

@@ -204,7 +204,7 @@ pub fn container_init_process(
         // before pivot_root is called. This runs in the container namespaces.
         if let Some(hooks) = hooks {
             hooks::run_hooks(hooks.create_container().as_ref(), container)
-                .context("Failed to run create container hooks")?;
+                .context("failed to run create container hooks")?;
         }
 
         let bind_service = namespaces.get(LinuxNamespaceType::User).is_some();
@@ -216,7 +216,7 @@ pub fn container_init_process(
                 bind_service,
                 namespaces.get(LinuxNamespaceType::Cgroup).is_some(),
             )
-            .with_context(|| "Failed to prepare rootfs")?;
+            .with_context(|| "failed to prepare rootfs")?;
 
         // Entering into the rootfs jail. If mount namespace is specified, then
         // we use pivot_root, but if we are on the host mount namespace, we will

crates/libcontainer/src/process/container_intermediate_process.rs

@@ -119,7 +119,8 @@ pub fn container_intermediate_process(
                     write(exec_notify_fd, buf.as_bytes())?;
                     close(exec_notify_fd)?;
                 }
-                Err(ProcessError::InitProcessFailed)
+                log::error!("failed to initialize container process: {e}");
+                Err(ProcessError::InitProcessFailed { msg: e.to_string() })
             }
         }
     })?;

crates/libcontainer/src/process/mod.rs

@@ -24,7 +24,7 @@ pub enum ProcessError {
         child_name: String,
     },
     #[error("failed init process")]
-    InitProcessFailed,
+    InitProcessFailed { msg: String },
     #[error("failed intermediate process")]
     IntermediateProcessFailed,
     #[error("io error: {0}")]

crates/libcontainer/src/rootfs/mount.rs

@@ -48,7 +48,7 @@ impl Mount {
     }
 
     pub fn setup_mount(&self, mount: &SpecMount, options: &MountOptions) -> Result<()> {
-        log::debug!("Mounting {:?}", mount);
+        log::debug!("mounting {:?}", mount);
         let mut mount_option_config = parse_mount(mount);
 
         match mount.typ().as_deref() {

crates/libcontainer/src/seccomp/mod.rs

@@ -136,7 +136,7 @@ pub fn initialize_seccomp(seccomp: &LinuxSeccomp) -> Result<Option<io::RawFd>> {
                 // When the action is the same as the default action, the rule is redundant. We can
                 // skip this here to avoid failing when we add the rules.
                 log::warn!(
-                    "Detect a seccomp action that is the same as the default action: {:?}",
+                    "detect a seccomp action that is the same as the default action: {:?}",
                     syscall
                 );
                 continue;

hack/debug.bt

@@ -41,9 +41,45 @@ tracepoint:syscalls:sys_exit_openat
 tracepoint:syscalls:sys_enter_clone3
 /comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
 {
-    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "clone3");
+    printf("%-12ld %15s %-8d %-9s\n", elapsed , comm, pid, "clone3");
 }
 
+tracepoint:syscalls:sys_enter_setns
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "setns");
+	printf("fd=%d, flag=%d\n", args->fd, args->flags);
+}
+
+tracepoint:syscalls:sys_enter_capset
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s\n", elapsed , comm, pid, "capset");
+}
+
+tracepoint:syscalls:sys_enter_pivot_root
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "pivt_root");
+	printf("new_root=%s, put_old=%s\n", str(args->new_root), str(args->put_old));
+}
+
+tracepoint:syscalls:sys_enter_mount
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "mount");
+    printf("dev_name=%s, dir_name=%s\n", str(args->dev_name), str(args->dir_name));
+}
+
+tracepoint:syscalls:sys_enter_setresuid
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "setresuid");
+    printf("ruid=%d, euid=%d, suid=%d\n", args->ruid, args->euid, args->suid);
+}
+
+
+
 END
 {
     clear(@filename);

tests/k8s/deploy.yaml

@@ -0,0 +1,27 @@
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: youki
+handler: youki
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      runtimeClassName: youki
+      containers:
+      - name: nginx
+        image: nginx:1.16.1
+        ports:
+        - containerPort: 80
+      automountServiceAccountToken: false