Merge pull request #1884 from utam0k/test-k3s

Add easy way to test with K8s

.github/scripts/dependency.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-sudo apt-get -y update
-sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev libseccomp-dev
+apt-get -y update
+apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev libseccomp-dev

.github/workflows/benchmark_execution_time.yml

@@ -15,7 +15,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
 
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/setup-rust-toolchain@v1.3.7
@@ -41,7 +41,7 @@ jobs:
           ref: main
 
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
 
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/setup-rust-toolchain@v1.3.7

.github/workflows/e2e.yaml

@@ -1,4 +1,4 @@
-name: 🧪 Containerd integration test
+name: 🧪 e2e test
 
 on:
   push:
@@ -17,7 +17,7 @@ jobs:
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/setup-rust-toolchain@v1.3.7
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
       - name: Build youki
         run: make youki-release
       - name: Upload youki binary
@@ -57,3 +57,21 @@ jobs:
           runc --version
       - name: Integration Test
         run: sudo make TEST_RUNTIME=io.containerd.runc.v2 TESTFLAGS="-timeout 40m" integration
+  #
+  # k8s-tests:
+  #   runs-on: ubuntu-22.04
+  #   needs: [youki-build]
+  #   timeout-minutes: 40
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - name: Download youki binary
+  #       uses: actions/download-artifact@v3
+  #       with:
+  #         name: youki
+  #     - name: Add the permission to run
+  #       run: chmod +x ./youki
+  #     - name: test/k8s/deploy
+  #       run: make test/k8s/deploy
+  #     # - name: Debug
+  #     #   if: ${{ always() }}
+  #     #   uses: mxschmitt/action-tmate@v3

.github/workflows/integration_tests_validation.yaml

@@ -31,7 +31,7 @@ jobs:
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/setup-rust-toolchain@v1.3.7
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
       - name: Install runc 1.1.0
         run: |
           wget -q https://github.com/opencontainers/runc/releases/download/v1.1.0/runc.amd64

.github/workflows/main.yml

@@ -46,7 +46,7 @@ jobs:
         uses: actions-rust-lang/setup-rust-toolchain@v1.3.7
       - run: rustup component add rustfmt clippy
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
       - run: sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev libseccomp-dev
       - name: Check formatting
         run: cargo fmt --all -- --check
@@ -84,7 +84,7 @@ jobs:
         with:
           tool: cargo-llvm-cov@0.4.0
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
       - name: Run Test Coverage for youki
         run: |
           cargo llvm-cov clean --workspace
@@ -105,7 +105,7 @@ jobs:
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/setup-rust-toolchain@v1.3.7
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
       - uses: actions/setup-go@v3
         with:
           go-version: "1.17.6"

.github/workflows/podman_tests.yaml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
       - run: make youki-dev
       - run: sudo cp youki /usr/local/bin
       - name: Install requirements for Podman

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
         with:
           submodules: recursive
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/setup-rust-toolchain@v1.3.7
       - name: Build
@@ -30,7 +30,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install requirements
-        run: ./.github/scripts/dependency.sh
+        run: sudo ./.github/scripts/dependency.sh
       - name: Setup Rust toolchain and cache
         uses: actions-rust-lang/setup-rust-toolchain@v1.3.7
       - name: Release build

.gitignore

@@ -19,4 +19,6 @@ tags.temp
 *~
 
 /bundle.tar.gz
-/test.log
+/test.log
+
+/tests/k8s/_out/

Makefile

@@ -1,5 +1,9 @@
 ROOT = $(shell git rev-parse --show-toplevel)
 
+DOCKER_BUILD ?= docker buildx build
+
+KIND_CLUSTER_NAME ?= youki
+
 # builds
 
 .PHONY:build
@@ -60,6 +64,55 @@ test-oci: oci-tests rust-oci-tests
 .PHONY: test-all
 test-all: unittest featuretest oci-tests containerd-test # currently not doing rust-oci here
 
+.PHONY: test/k3s
+test/k3s: bin/k3s
+	sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo 'default_runtime_name = "youki"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo '[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo '  runtime_type = "io.containerd.runc.v2"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo '  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki.options]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo '    BinaryName = "$(PWD)/youki"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
+	echo "CONTAINERD_NAMESPACE='default'" | sudo tee /etc/systemd/system/k3s-runwasi.service.env && \
+	echo "NO_PROXY=192.168.0.0/16" | sudo tee -a /etc/systemd/system/k3s-runwasi.service.env && \
+	sudo systemctl daemon-reload && \
+	sudo systemctl restart k3s-youki && \
+	sudo bin/k3s kubectl apply -f tests/k8s/deploy.yaml
+	sudo bin/k3s kubectl wait deployment nginx-deployment --for condition=Available=True --timeout=90s && \
+	sudo bin/k3s kubectl get pods -o wide
+
+.PHONY: test/k3s/clean
+test/k3s/clean:
+	sudo bin/k3s-youki-uninstall.sh
+
+.PHONY: test/k8s/cluster
+test/k8s/cluster: bin/kind tests/k8s/_out/img bin/kind
+	bin/kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat tests/k8s/_out/img)"
+
+.PHONY: test/k8s/deploy
+test/k8s/deploy: test/k8s/cluster
+	kubectl --context=kind-$(KIND_CLUSTER_NAME) apply -f tests/k8s/deploy.yaml
+	kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment nginx-deployment --for condition=Available=True --timeout=90s
+	kubectl --context=kind-$(KIND_CLUSTER_NAME) get pods -o wide
+
+# Bin
+
+.PHONY: bin/k3s
+bin/k3s:
+	mkdir -p bin && \
+	curl -sfL https://get.k3s.io | INSTALL_K3S_BIN_DIR=$(PWD)/bin INSTALL_K3S_SYMLINK=skip INSTALL_K3S_NAME=youki sh -
+
+.PHONY: bin/kind
+bin/kind: tests/k8s/Dockerfile
+	$(DOCKER_BUILD) --output=bin/ -f tests/k8s/Dockerfile --target kind-bin .
+
+.PHONY: test/k8s/clean
+test/k8s/clean:
+	kind delete cluster --name $(KIND_CLUSTER_NAME)
+	rm -r tests/k8s/_out
+
+tests/k8s/_out/img: tests/k8s/Dockerfile Cargo.toml Cargo.lock $(shell find . -type f -name '*.rs')
+	mkdir -p $(@D) && $(DOCKER_BUILD) -f tests/k8s/Dockerfile --iidfile=$(@) --load .
+
 # Misc
 
 .PHONY: lint
@@ -69,7 +122,7 @@ lint:
 
 .PHONY: hack/bpftrace
 hack/bpftrace:
-	BPFTRACE_STRLEN=120 ./hack/debug.bt
+	BPFTRACE_STRLEN=125 ./hack/debug.bt
 
 .PHONY: clean
 clean:

crates/libcontainer/src/process/container_init_process.rs

@@ -204,7 +204,7 @@ pub fn container_init_process(
         // before pivot_root is called. This runs in the container namespaces.
         if let Some(hooks) = hooks {
             hooks::run_hooks(hooks.create_container().as_ref(), container)
-                .context("Failed to run create container hooks")?;
+                .context("failed to run create container hooks")?;
         }
 
         let bind_service = namespaces.get(LinuxNamespaceType::User).is_some();
@@ -216,7 +216,7 @@ pub fn container_init_process(
                 bind_service,
                 namespaces.get(LinuxNamespaceType::Cgroup).is_some(),
             )
-            .with_context(|| "Failed to prepare rootfs")?;
+            .with_context(|| "failed to prepare rootfs")?;
 
         // Entering into the rootfs jail. If mount namespace is specified, then
         // we use pivot_root, but if we are on the host mount namespace, we will
@@ -376,9 +376,9 @@ pub fn container_init_process(
         warn!("seccomp not available, unable to enforce no_new_privileges!")
     }
 
-    capabilities::reset_effective(syscall).context("Failed to reset effective capabilities")?;
+    capabilities::reset_effective(syscall).context("failed to reset effective capabilities")?;
     if let Some(caps) = proc.capabilities() {
-        capabilities::drop_privileges(caps, syscall).context("Failed to drop capabilities")?;
+        capabilities::drop_privileges(caps, syscall).context("failed to drop capabilities")?;
     }
 
     // Change directory to process.cwd if process.cwd is not empty

crates/libcontainer/src/process/container_intermediate_process.rs

@@ -119,7 +119,8 @@ pub fn container_intermediate_process(
                     write(exec_notify_fd, buf.as_bytes())?;
                     close(exec_notify_fd)?;
                 }
-                Err(ProcessError::InitProcessFailed)
+                log::error!("failed to initialize container process: {e}");
+                Err(ProcessError::InitProcessFailed { msg: e.to_string() })
             }
         }
     })?;

crates/libcontainer/src/process/mod.rs

@@ -24,7 +24,7 @@ pub enum ProcessError {
         child_name: String,
     },
     #[error("failed init process")]
-    InitProcessFailed,
+    InitProcessFailed { msg: String },
     #[error("failed intermediate process")]
     IntermediateProcessFailed,
     #[error("io error: {0}")]

crates/libcontainer/src/rootfs/mount.rs

@@ -48,7 +48,7 @@ impl Mount {
     }
 
     pub fn setup_mount(&self, mount: &SpecMount, options: &MountOptions) -> Result<()> {
-        log::debug!("Mounting {:?}", mount);
+        log::debug!("mounting {:?}", mount);
         let mut mount_option_config = parse_mount(mount);
 
         match mount.typ().as_deref() {

crates/libcontainer/src/seccomp/mod.rs

@@ -136,7 +136,7 @@ pub fn initialize_seccomp(seccomp: &LinuxSeccomp) -> Result<Option<io::RawFd>> {
                 // When the action is the same as the default action, the rule is redundant. We can
                 // skip this here to avoid failing when we add the rules.
                 log::warn!(
-                    "Detect a seccomp action that is the same as the default action: {:?}",
+                    "detect a seccomp action that is the same as the default action: {:?}",
                     syscall
                 );
                 continue;

hack/debug.bt

@@ -41,9 +41,45 @@ tracepoint:syscalls:sys_exit_openat
 tracepoint:syscalls:sys_enter_clone3
 /comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
 {
-    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "clone3");
+    printf("%-12ld %15s %-8d %-9s\n", elapsed , comm, pid, "clone3");
 }
 
+tracepoint:syscalls:sys_enter_setns
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "setns");
+	printf("fd=%d, flag=%d\n", args->fd, args->flags);
+}
+
+tracepoint:syscalls:sys_enter_capset
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s\n", elapsed , comm, pid, "capset");
+}
+
+tracepoint:syscalls:sys_enter_pivot_root
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "pivt_root");
+	printf("new_root=%s, put_old=%s\n", str(args->new_root), str(args->put_old));
+}
+
+tracepoint:syscalls:sys_enter_mount
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "mount");
+    printf("dev_name=%s, dir_name=%s\n", str(args->dev_name), str(args->dir_name));
+}
+
+tracepoint:syscalls:sys_enter_setresuid
+/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/
+{
+    printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "setresuid");
+    printf("ruid=%d, euid=%d, suid=%d\n", args->ruid, args->euid, args->suid);
+}
+
+
+
 END
 {
     clear(@filename);

tests/k8s/Dockerfile

@@ -0,0 +1,36 @@
+# syntax=docker/dockerfile:1.4
+
+ARG KIND_NODE_VERSION=v1.23.13
+
+FROM kindest/node:${KIND_NODE_VERSION} AS kind-base
+
+FROM kind-base AS shim-build
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /tmp/rustup.sh && sh /tmp/rustup.sh -y --profile=minimal
+ENV PATH="/root/.cargo/bin:${PATH}"
+WORKDIR /shim
+COPY ./youki /shim/youki
+
+FROM scratch AS shim
+COPY --from=shim-build /shim/youki /
+
+FROM kind-base AS kind-fetch
+ARG TARGETARCH
+ARG KIND_VERSION=v0.17.0
+RUN curl -sSLf https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-${TARGETARCH} > /root/kind && chmod +x /root/kind
+
+FROM scratch AS kind-bin
+COPY --from=kind-fetch /root/kind /kind
+
+FROM kind-base
+RUN <<EOF
+set -e
+echo '[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki]' >> /etc/containerd/config.toml
+echo '  runtime_type = "io.containerd.runc.v2"' >> /etc/containerd/config.toml
+echo '   [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki.options]' >> /etc/containerd/config.toml
+echo '     BinaryName = "/usr/local/bin/youki"' >> /etc/containerd/config.toml
+sed -i 's,SystemdCgroup = true,,' /etc/containerd/config.toml
+EOF
+COPY .github/scripts/dependency.sh .github/scripts/dependency.sh
+RUN apt-get update && .github/scripts/dependency.sh
+COPY --link --from=shim /* /usr/local/bin/
+

tests/k8s/deploy.yaml

@@ -0,0 +1,27 @@
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: youki
+handler: youki
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      runtimeClassName: youki
+      containers:
+      - name: nginx
+        image: nginx:1.16.1
+        ports:
+        - containerPort: 80
+      automountServiceAccountToken: false