From ec2f58d4b268667cb42346cb3630262a18ded108 Mon Sep 17 00:00:00 2001 From: utam0k Date: Sat, 6 May 2023 12:02:18 +0000 Subject: [PATCH] Add easy way to test with K8s Signed-off-by: utam0k --- .gitignore | 2 +- Makefile | 27 ++++++++++++- .../src/process/container_init_process.rs | 4 +- .../process/container_intermediate_process.rs | 3 +- crates/libcontainer/src/process/mod.rs | 2 +- crates/libcontainer/src/rootfs/mount.rs | 2 +- crates/libcontainer/src/seccomp/mod.rs | 2 +- hack/debug.bt | 38 ++++++++++++++++++- tests/k8s/deploy.yaml | 27 +++++++++++++ 9 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 tests/k8s/deploy.yaml diff --git a/.gitignore b/.gitignore index 1836f879..0dd74a3f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ tags.temp *~ /bundle.tar.gz -/test.log \ No newline at end of file +/test.log diff --git a/Makefile b/Makefile index bd0095d0..012c7420 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,32 @@ test-oci: oci-tests rust-oci-tests .PHONY: test-all test-all: unittest featuretest oci-tests containerd-test # currently not doing rust-oci here +.PHONY: test/k3s +test/k3s: bin/k3s + sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo 'default_runtime_name = "youki"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo '[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo ' runtime_type = "io.containerd.runc.v2"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo ' [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki.options]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo ' BinaryName = "$(PWD)/youki"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo "CONTAINERD_NAMESPACE='default'" | sudo tee /etc/systemd/system/k3s-runwasi.service.env && \ + echo "NO_PROXY=192.168.0.0/16" | sudo tee -a /etc/systemd/system/k3s-runwasi.service.env && \ + sudo systemctl daemon-reload && \ + sudo systemctl restart k3s-youki && \ + sudo bin/k3s kubectl apply -f tests/k8s/deploy.yaml + sudo bin/k3s kubectl wait deployment nginx-deployment --for condition=Available=True --timeout=90s && \ + sudo bin/k3s kubectl get pods -o wide + +.PHONY: test/k3s/clean +test/k3s/clean: + sudo bin/k3s-youki-uninstall.sh + # Misc +# +.PHONY: bin/k3s +bin/k3s: + mkdir -p bin && \ + curl -sfL https://get.k3s.io | INSTALL_K3S_BIN_DIR=$(PWD)/bin INSTALL_K3S_SYMLINK=skip INSTALL_K3S_NAME=youki sh - .PHONY: lint lint: @@ -69,7 +94,7 @@ lint: .PHONY: hack/bpftrace hack/bpftrace: - BPFTRACE_STRLEN=120 ./hack/debug.bt + BPFTRACE_STRLEN=125 ./hack/debug.bt .PHONY: clean clean: diff --git a/crates/libcontainer/src/process/container_init_process.rs b/crates/libcontainer/src/process/container_init_process.rs index dd6b210f..7024e46e 100644 --- a/crates/libcontainer/src/process/container_init_process.rs +++ b/crates/libcontainer/src/process/container_init_process.rs @@ -204,7 +204,7 @@ pub fn container_init_process( // before pivot_root is called. This runs in the container namespaces. if let Some(hooks) = hooks { hooks::run_hooks(hooks.create_container().as_ref(), container) - .context("Failed to run create container hooks")?; + .context("failed to run create container hooks")?; } let bind_service = namespaces.get(LinuxNamespaceType::User).is_some(); @@ -216,7 +216,7 @@ pub fn container_init_process( bind_service, namespaces.get(LinuxNamespaceType::Cgroup).is_some(), ) - .with_context(|| "Failed to prepare rootfs")?; + .with_context(|| "failed to prepare rootfs")?; // Entering into the rootfs jail. If mount namespace is specified, then // we use pivot_root, but if we are on the host mount namespace, we will diff --git a/crates/libcontainer/src/process/container_intermediate_process.rs b/crates/libcontainer/src/process/container_intermediate_process.rs index aabdba44..7d606c43 100644 --- a/crates/libcontainer/src/process/container_intermediate_process.rs +++ b/crates/libcontainer/src/process/container_intermediate_process.rs @@ -119,7 +119,8 @@ pub fn container_intermediate_process( write(exec_notify_fd, buf.as_bytes())?; close(exec_notify_fd)?; } - Err(ProcessError::InitProcessFailed) + log::error!("failed to initialize container process: {e}"); + Err(ProcessError::InitProcessFailed { msg: e.to_string() }) } } })?; diff --git a/crates/libcontainer/src/process/mod.rs b/crates/libcontainer/src/process/mod.rs index 5962b6c0..94bf6bf2 100644 --- a/crates/libcontainer/src/process/mod.rs +++ b/crates/libcontainer/src/process/mod.rs @@ -24,7 +24,7 @@ pub enum ProcessError { child_name: String, }, #[error("failed init process")] - InitProcessFailed, + InitProcessFailed { msg: String }, #[error("failed intermediate process")] IntermediateProcessFailed, #[error("io error: {0}")] diff --git a/crates/libcontainer/src/rootfs/mount.rs b/crates/libcontainer/src/rootfs/mount.rs index 418baee8..07fa80fc 100644 --- a/crates/libcontainer/src/rootfs/mount.rs +++ b/crates/libcontainer/src/rootfs/mount.rs @@ -48,7 +48,7 @@ impl Mount { } pub fn setup_mount(&self, mount: &SpecMount, options: &MountOptions) -> Result<()> { - log::debug!("Mounting {:?}", mount); + log::debug!("mounting {:?}", mount); let mut mount_option_config = parse_mount(mount); match mount.typ().as_deref() { diff --git a/crates/libcontainer/src/seccomp/mod.rs b/crates/libcontainer/src/seccomp/mod.rs index d6bb4e16..c06071d5 100644 --- a/crates/libcontainer/src/seccomp/mod.rs +++ b/crates/libcontainer/src/seccomp/mod.rs @@ -136,7 +136,7 @@ pub fn initialize_seccomp(seccomp: &LinuxSeccomp) -> Result> { // When the action is the same as the default action, the rule is redundant. We can // skip this here to avoid failing when we add the rules. log::warn!( - "Detect a seccomp action that is the same as the default action: {:?}", + "detect a seccomp action that is the same as the default action: {:?}", syscall ); continue; diff --git a/hack/debug.bt b/hack/debug.bt index b8028832..6b104330 100755 --- a/hack/debug.bt +++ b/hack/debug.bt @@ -41,9 +41,45 @@ tracepoint:syscalls:sys_exit_openat tracepoint:syscalls:sys_enter_clone3 /comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ { - printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "clone3"); + printf("%-12ld %15s %-8d %-9s\n", elapsed , comm, pid, "clone3"); } +tracepoint:syscalls:sys_enter_setns +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "setns"); + printf("fd=%d, flag=%d\n", args->fd, args->flags); +} + +tracepoint:syscalls:sys_enter_capset +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s\n", elapsed , comm, pid, "capset"); +} + +tracepoint:syscalls:sys_enter_pivot_root +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "pivt_root"); + printf("new_root=%s, put_old=%s\n", str(args->new_root), str(args->put_old)); +} + +tracepoint:syscalls:sys_enter_mount +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "mount"); + printf("dev_name=%s, dir_name=%s\n", str(args->dev_name), str(args->dir_name)); +} + +tracepoint:syscalls:sys_enter_setresuid +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "setresuid"); + printf("ruid=%d, euid=%d, suid=%d\n", args->ruid, args->euid, args->suid); +} + + + END { clear(@filename); diff --git a/tests/k8s/deploy.yaml b/tests/k8s/deploy.yaml new file mode 100644 index 00000000..6fe094e5 --- /dev/null +++ b/tests/k8s/deploy.yaml @@ -0,0 +1,27 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: youki +handler: youki +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: youki + containers: + - name: nginx + image: nginx:1.16.1 + ports: + - containerPort: 80 + automountServiceAccountToken: false