Fix cross cotainers for tests running

Signed-off-by: Yashodhan Joshi <yjdoc2@gmail.com>

Cross.toml

@@ -1,5 +1,6 @@
 [build]
 default-target = "x86_64-unknown-linux-gnu"
+env.passthrough = ["XDG_RUNTIME_DIR"]
 
 [target.aarch64-unknown-linux-gnu]
 dockerfile = "cross/Dockerfile.gnu"

crates/libcgroups/src/systemd/dbus_native/client.rs

@@ -26,4 +26,11 @@ pub trait SystemdClient {
     fn systemd_version(&self) -> Result<u32, SystemdClientError>;
 
     fn control_cgroup_root(&self) -> Result<PathBuf, SystemdClientError>;
+
+    fn add_process_to_unit(
+        &self,
+        unit_name: &str,
+        subcgroup: &str,
+        pid: u32,
+    ) -> Result<(), SystemdClientError>;
 }

crates/libcgroups/src/systemd/dbus_native/dbus.rs

@@ -453,6 +453,10 @@ impl SystemdClient for DbusConnection {
         let cgroup_root = proxy.control_group()?;
         Ok(PathBuf::from(&cgroup_root))
     }
+    fn add_process_to_unit(&self, unit_name: &str, subcgroup: &str, pid: u32) -> Result<()> {
+        let proxy = self.create_proxy();
+        proxy.attach_process(unit_name, subcgroup, pid)
+    }
 }
 
 #[cfg(test)]

crates/libcgroups/src/systemd/dbus_native/proxy.rs

@@ -239,4 +239,11 @@ impl<'conn> Proxy<'conn> {
             v => panic!("control group expected string variant, got {:?} instead", v),
         }
     }
+    pub fn attach_process(&self, name: &str, cgroup: &str, pid: u32) -> Result<()> {
+        self.method_call::<_, ()>(
+            "org.freedesktop.systemd1.Manager",
+            "AttachProcessesToUnit",
+            Some((name, cgroup, vec![pid])),
+        )
+    }
 }

crates/libcgroups/src/systemd/manager.rs

@@ -21,7 +21,7 @@ use super::{
 use crate::{
     common::{
         self, AnyCgroupManager, CgroupManager, ControllerOpt, FreezerState, JoinSafelyError,
-        PathBufExt, WrapIoResult, WrappedIoError, CGROUP_PROCS,
+        PathBufExt, WrapIoResult, WrappedIoError,
     },
     systemd::{dbus_native::serialize::Variant, unified::Unified},
     v2::manager::V2ManagerError,
@@ -355,7 +355,8 @@ impl CgroupManager for Manager {
         }
         if self.client.transient_unit_exists(&self.unit_name) {
             tracing::debug!("Transient unit {:?} already exists", self.unit_name);
-            common::write_cgroup_file(self.full_path.join(CGROUP_PROCS), pid)?;
+            self.client
+                .add_process_to_unit(&self.unit_name, "", pid.as_raw() as u32)?;
             return Ok(());
         }
 
@@ -435,8 +436,11 @@ mod tests {
     use anyhow::{Context, Result};
 
     use super::*;
-    use crate::systemd::dbus_native::{
-        client::SystemdClient, serialize::Variant, utils::SystemdClientError,
+    use crate::{
+        common::DEFAULT_CGROUP_ROOT,
+        systemd::dbus_native::{
+            client::SystemdClient, serialize::Variant, utils::SystemdClientError,
+        },
     };
 
     struct TestSystemdClient {}
@@ -479,6 +483,15 @@ mod tests {
         fn control_cgroup_root(&self) -> Result<PathBuf, SystemdClientError> {
             Ok(PathBuf::from("/"))
         }
+
+        fn add_process_to_unit(
+            &self,
+            _unit_name: &str,
+            _subcgroup: &str,
+            _pid: u32,
+        ) -> Result<(), SystemdClientError> {
+            Ok(())
+        }
     }
 
     #[test]
@@ -533,4 +546,36 @@ mod tests {
 
         Ok(())
     }
+    #[test]
+    fn test_task_addition() {
+        let manager = Manager::new(
+            DEFAULT_CGROUP_ROOT.into(),
+            ":youki:test".into(),
+            "youki_test_container".into(),
+            false,
+        )
+        .unwrap();
+        let mut p1 = std::process::Command::new("sleep")
+            .arg("1s")
+            .spawn()
+            .unwrap();
+        let p1_id = nix::unistd::Pid::from_raw(p1.id() as i32);
+        let mut p2 = std::process::Command::new("sleep")
+            .arg("1s")
+            .spawn()
+            .unwrap();
+        let p2_id = nix::unistd::Pid::from_raw(p2.id() as i32);
+        manager.add_task(p1_id).unwrap();
+        manager.add_task(p2_id).unwrap();
+        let all_pids = manager.get_all_pids().unwrap();
+        assert!(all_pids.contains(&p1_id));
+        assert!(all_pids.contains(&p2_id));
+        // wait till both processes are finished so we can cleanup the cgroup
+        let _ = p1.wait();
+        let _ = p2.wait();
+        manager.remove().unwrap();
+        // the remove call above should remove the dir, we just do this again
+        // for contingency, and thus ignore the result
+        let _ = fs::remove_dir(&manager.full_path);
+    }
 }

crates/libcontainer/src/container/tenant_builder.rs

@@ -332,7 +332,7 @@ impl TenantContainerBuilder {
         let spec_linux = spec.linux().as_ref().unwrap();
         let mut linux_builder = LinuxBuilder::default().namespaces(ns);
 
-        if let Some(ref cgroup_path) = spec_linux.cgroups_path(){
+        if let Some(ref cgroup_path) = spec_linux.cgroups_path() {
             linux_builder = linux_builder.cgroups_path(cgroup_path.clone());
         }
         let linux = linux_builder.build()?;

cross/Dockerfile.gnu

@@ -13,3 +13,6 @@ RUN dpkg --add-architecture ${CROSS_DEB_ARCH} && \
     zlib1g-dev:${CROSS_DEB_ARCH} \
     # dependencies to build wasmedge-sys
     libzstd-dev:${CROSS_DEB_ARCH}
+
+COPY hack/busctl.sh /bin/busctl
+RUN chmod +x /bin/busctl

cross/Dockerfile.musl

@@ -22,6 +22,9 @@ ENV LIBSECCOMP_LIB_PATH="${CROSS_SYSROOT}/lib"
 ENV WASMEDGE_DEP_STDCXX_LINK_TYPE="static"
 ENV WASMEDGE_DEP_STDCXX_LIB_PATH="${CROSS_SYSROOT}/lib"
 
+COPY hack/busctl.sh /bin/busctl
+RUN chmod +x /bin/busctl
+
 # wasmedge-sys (through llvm) needs some symbols defined in libgcc
 RUN mkdir /.cargo && cat <<'EOF' > /.cargo/config.toml
 [target.'cfg(target_env = "musl")']

hack/busctl.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# This hack script is the dummy busctl command used when running tests with cross containers.
+
+# The issue is that we cannot run systemd or dbus inside the test container without a lot 
+# of hacks. For one specific test - test_task_addition, we need to check that the task
+# addition via systemd manager works. We mount the host dbus socket in the test container, so 
+# dbus calls work, but for the initial authentication, we use busctl which needs dbus and systemd
+# to be present and running. So instead of doing all that, we simply run the container with the 
+# actual test running user's uid/gid and here we echo the only relevant line from busctl's 
+# output, using id to get the uid. This is a hack, but less complex than actually setting up
+# and running the systemd+dbus inside the container.
+
+echo "OwnerUID=$(id -u)"

scripts/cargo.sh

@@ -56,7 +56,11 @@ if [ "$CARGO" == "cross" ]; then
 
     # mount run to have access to dbus socket.
     # mount /tmp so as shared for test_make_parent_mount_private
-    export CROSS_CONTAINER_OPTS="--privileged -v/run:/run --mount=type=bind,source=/tmp,destination=/tmp,bind-propagation=shared"
+    # Then there are few "hacks" specificallt for test_task_addition
+    # run with user same as the invoking user, so that the dbus is connected with correct user
+    # we want pid ns of host, because we will be connecting to the host dbus, and it needs task pid from host
+    # finally we need to mount the cgroup as read-only, as we need that to check if the tasks are correctly added
+    export CROSS_CONTAINER_OPTS="--privileged --user `id -u`:`id -g` --pid=host -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v/run:/run --mount=type=bind,source=/tmp,destination=/tmp,bind-propagation=shared"
 fi
 
 if [ "$1" == "--print-target-dir" ]; then

tests/rootless-tests/run.sh

@@ -17,6 +17,7 @@ rand=$(head -c 10 /dev/random | base64)
 log=$(podman run --runtime $runtime fedora echo "$rand")
 echo $log | grep $rand
 
+podman kill exec-test || true # ignore failure for killing
 podman rm --force --ignore exec-test
 podman run -d --runtime $runtime --name exec-test busybox sleep 10m
 
@@ -25,5 +26,19 @@ rand=$(head -c 10 /dev/random | base64)
 log=$(podman exec --runtime $runtime exec-test echo "$rand")
 echo $log | grep $rand
 
+CGROUP_SUB_PATH=$(podman inspect exec-test | jq .[0].State.CgroupPath | tr -d "\"")
+CGROUP_PATH="/sys/fs/cgroup$CGROUP_SUB_PATH/cgroup.procs"
+
+# assert we have exactly one process in the cgroup
+test $(cat $CGROUP_PATH | wc -l) -eq 1
+# assert pid match
+test $(cat $CGROUP_PATH) -eq $(podman inspect exec-test | jq .[0].State.Pid)
+
+podman exec -d --runtime $runtime exec-test sleep 5m
+
+# we cannot exactly check the pid of tenant here, instead just check that there are
+# two processes in the same cgroup now
+test $(cat $CGROUP_PATH | wc -l) -eq 2
+
 podman kill exec-test
 podman rm --force --ignore exec-test