Merge remote-tracking branch 'upstream/main' into upgrade-oci-spec-rs

Signed-off-by: Takashi IIGUNI <iiguni.tks@gmail.com>
2024-05-10 01:26:14 +02:00 · 2021-09-07 02:32:41 +00:00 · 2021-09-07 02:32:41 +00:00 · c83ac6a22b
parent 8d3ff9b5f2 ef9a92a334
commit c83ac6a22b
54 changed files with 1615 additions and 617 deletions
--- a/.codecov.yml
+++ b/.codecov.yml
@ -0,0 +1,15 @@
+---
+codecov:
+  notify:
+    after_n_builds: 1
+    require_ci_to_pass: false
+
+coverage:
+  precision: 2
+  round: down
+  range: 50..75
+
+comment:
+  layout: "header, diff"
+  behavior: default
+  require_changes: false
--- a/.github/grcov.yml
+++ b/.github/grcov.yml
@ -0,0 +1,7 @@
+branch: true
+ignore-not-existing: true
+llvm: true
+filter: covered
+output-type: lcov
+output-path: ./lcov.info
+prefix-dir: /home/user/build/
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -7,11 +7,27 @@ on:
      - main

 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      dirs: ${{ steps.filter.outputs.changes }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            .: 'src/*'
+            ./test_framework: test_framework/*
+            ./youki_integration_test: youki_integration_test/*
+            ./cgroups: cgroups/*
  check:
+    needs: [changes]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        rust: [stable]
+        dirs: ${{ fromJSON(needs.changes.outputs.dirs) }}
    steps:
      - uses: actions/checkout@v2
      - uses: actions/cache@v2
@ -29,13 +45,13 @@ jobs:
          override: true
      - run: rustup component add rustfmt clippy
      - run: sudo apt-get -y update
-      - run: sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev
+      - run: sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev
      - name: Check formatting
        run: cargo fmt --all -- --check
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-targets --all-features -- -D warnings
+        working-directory: ${{matrix.dirs}}
+      - name: Check clippy lints
+        working-directory: ${{matrix.dirs}}
+        run: cargo clippy --all-targets --all-features -- -D warnings
  tests:
    runs-on: ubuntu-latest
    strategy:
@ -55,17 +71,66 @@ jobs:
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: ${{ matrix.rust }}
+          override: true
      - run: sudo apt-get -y update
      - run: sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev
-      - name: Build
-        run: ./build.sh --release
      - name: Run tests
-        run: cargo test
+        run: cargo test --no-fail-fast
      - name: Run doc tests
        run: cargo test --doc
      - name: Run cgroup tests
        working-directory: cgroups
-        run: cargo test
+        run: cargo test --no-fail-fast
+  coverage:
+    runs-on: ubuntu-latest
+    name: Run test coverage
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry
+            ~/.cargo/git
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+        continue-on-error: true
+      - name: install cargo-llvm-cov
+        run: |
+          wget https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${CARGO_LLVM_COV_VERSION}/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz -qO- | tar -xzvf -
+          mv cargo-llvm-cov ~/.cargo/bin
+        env:
+          CARGO_LLVM_COV_VERSION: 0.1.5
+      - name: Update System Libraries
+        run: sudo apt-get -y update
+      - name: Install System Libraries
+        run: sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev
+      - name: Toolchain setup
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+          profile: minimal
+          components: llvm-tools-preview
+      - name: Run Test Coverage for youki
+        run: |
+          cargo llvm-cov clean --workspace
+          cargo llvm-cov --no-report
+          cargo llvm-cov --no-run --lcov --output-path ./coverage.lcov
+      - name: Run Test Coverage for cgroups
+        working-directory: ./cgroups
+        run: |
+          cargo llvm-cov clean --workspace
+          cargo llvm-cov --no-report
+          cargo llvm-cov --no-run --lcov --output-path ./coverage.lcov
+      - name: Upload Youki Code Coverage Results
+        uses: codecov/codecov-action@v2
+        with:
+          file: ./coverage.lcov
+      - name: Upload Cgroups Code Coverage Results
+        uses: codecov/codecov-action@v2
+        with:
+          file: ./cgroups/coverage.lcov
  integration_tests:
    runs-on: ubuntu-latest
    strategy:
--- a/Cargo.lock
+++ b/Cargo.lock
@ -106,9 +106,9 @@ dependencies = [

 [[package]]
 name = "clap"
-version = "3.0.0-beta.2"
+version = "3.0.0-beta.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142"
+checksum = "fcd70aa5597dbc42f7217a543f9ef2768b2ef823ba29036072d30e1d88e98406"
 dependencies = [
 "bitflags",
 "clap_derive",
@ -117,15 +117,14 @@ dependencies = [
 "os_str_bytes",
 "strsim",
 "textwrap",
- "unicode-width",
 "vec_map",
 ]

 [[package]]
 name = "clap_derive"
-version = "3.0.0-beta.2"
+version = "3.0.0-beta.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1"
+checksum = "0b5bb0d655624a0b8770d1c178fb8ffcb1f91cc722cb08f451e3dc72465421ac"
 dependencies = [
 "heck",
 "proc-macro-error",
@ -598,9 +597,9 @@ checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"

 [[package]]
 name = "os_str_bytes"
-version = "2.4.0"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85"
+checksum = "6acbef58a60fe69ab50510a55bc8cdd4d6cf2283d27ad338f54cb52747a9cf2d"

 [[package]]
 name = "parking_lot"
@ -693,18 +692,18 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"

 [[package]]
 name = "proc-macro2"
-version = "1.0.27"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
+checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d"
 dependencies = [
 "unicode-xid",
 ]

 [[package]]
 name = "procfs"
-version = "0.9.1"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab8809e0c18450a2db0f236d2a44ec0b4c1412d0eb936233579f0990faa5d5cd"
+checksum = "95e344cafeaeefe487300c361654bcfc85db3ac53619eeccced29f5ea18c4c70"
 dependencies = [
 "bitflags",
 "byteorder",
@ -864,9 +863,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

 [[package]]
 name = "syn"
-version = "1.0.72"
+version = "1.0.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
+checksum = "b7f58f7e8eaa0009c5fec437aabf511bd9933e4b2d7407bd05273c01a8906ea7"
 dependencies = [
 "proc-macro2",
 "quote",
@ -899,12 +898,9 @@ dependencies = [

 [[package]]
 name = "textwrap"
-version = "0.12.1"
+version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789"
-dependencies = [
- "unicode-width",
-]
+checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"

 [[package]]
 name = "thiserror"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -10,13 +10,13 @@ default = ["systemd_cgroups"]
 systemd_cgroups = ["systemd"]

 [dependencies.clap]
-version = "3.0.0-beta.2"
+version = "3.0.0-beta.4"
 default-features = false
-features = ["std", "suggestions", "derive"]
+features = ["std", "suggestions", "derive", "cargo"]

 [dependencies]
 nix = "0.22.0"
-procfs = "0.9.1"
+procfs = "0.10.1"
 # Waiting for new caps release, replace git with version on release
 caps = { git = "https://github.com/lucab/caps-rs", rev = "cb54844", features = ["serde_support"] }
 serde = { version = "1.0", features = ["derive"] }
--- a/README.md
+++ b/README.md
@ -4,6 +4,7 @@
 [![GitHub commit activity](https://img.shields.io/github/commit-activity/m/containers/youki)](https://github.com/containers/youki/graphs/commit-activity)
 [![GitHub contributors](https://img.shields.io/github/contributors/containers/youki)](https://github.com/containers/youki/graphs/contributors)
 [![Github CI](https://github.com/containers/youki/actions/workflows/main.yml/badge.svg?branch=main)](https://github.com/containers/youki/actions)
+[![codecov](https://codecov.io/gh/containers/youki/branch/main/graph/badge.svg)](https://codecov.io/gh/containers/youki)

 <p align="center">
  <img src="docs/youki.png" width="230" height="230">
@ -47,7 +48,7 @@ youki is not at the practical stage yet. However, it is getting closer to practi
 |    Seccomp     |             Filtering system calls              |                     WIP on [#25](https://github.com/containers/youki/issues/25)                     |
 |     Hooks      | Add custom processing during container creation |                                                  ✅                                                  |
 |    Rootless    |   Running a container without root privileges   | It works, but cgroups isn't supported. WIP on [#77](https://github.com/containers/youki/issues/77)  |
-| OCI Compliance |        Compliance with OCI Runtime Spec         |                                   44 out of 55 test cases passing                                   |
+| OCI Compliance |        Compliance with OCI Runtime Spec         |                                   47 out of 55 test cases passing                                   |

 # Design and implementation of youki
 ![sequence diagram of youki](docs/.drawio.svg)
--- a/cgroups/Cargo.lock
+++ b/cgroups/Cargo.lock
@ -446,9 +446,9 @@ dependencies = [

 [[package]]
 name = "procfs"
-version = "0.9.1"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab8809e0c18450a2db0f236d2a44ec0b4c1412d0eb936233579f0990faa5d5cd"
+checksum = "95e344cafeaeefe487300c361654bcfc85db3ac53619eeccced29f5ea18c4c70"
 dependencies = [
 "bitflags",
 "byteorder",
--- a/cgroups/Cargo.toml
+++ b/cgroups/Cargo.toml
@ -11,7 +11,7 @@ cgroupsv2_devices = ["rbpf", "libbpf-sys", "errno", "libc"]

 [dependencies]
 nix = "0.22.0"
-procfs = "0.9.1"
+procfs = "0.10.1"
 log = "0.4"
 anyhow = "1.0"
 oci-spec = "0.4.0"
--- a/cgroups/src/common.rs
+++ b/cgroups/src/common.rs
@ -1,5 +1,4 @@
 use std::{
-    env,
    fmt::{Debug, Display},
    fs::{self, File},
    io::{BufRead, BufReader, Write},
@ -7,7 +6,10 @@ use std::{
 };

 use anyhow::{bail, Context, Result};
-use nix::unistd::Pid;
+use nix::{
+    sys::statfs::{statfs, CGROUP2_SUPER_MAGIC, TMPFS_MAGIC},
+    unistd::Pid,
+};
 use oci_spec::runtime::{LinuxDevice, LinuxDeviceCgroup, LinuxDeviceType, LinuxResources};
 use procfs::process::Process;
 #[cfg(feature = "systemd_cgroups")]
@ -41,16 +43,18 @@ pub trait CgroupManager {
 }

 #[derive(Debug)]
-pub enum Cgroup {
-    V1,
-    V2,
+pub enum CgroupSetup {
+    Hybrid,
+    Legacy,
+    Unified,
 }

-impl Display for Cgroup {
+impl Display for CgroupSetup {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let print = match *self {
-            Cgroup::V1 => "v1",
-            Cgroup::V2 => "v2",
+        let print = match self {
+            CgroupSetup::Hybrid => "hybrid",
+            CgroupSetup::Legacy => "legacy",
+            CgroupSetup::Unified => "unified",
        };

        write!(f, "{}", print)
@ -108,92 +112,79 @@ pub fn read_cgroup_file<P: AsRef<Path>>(path: P) -> Result<String> {
    fs::read_to_string(path).with_context(|| format!("failed to open {:?}", path))
 }

-pub fn get_supported_cgroup_fs() -> Result<Vec<Cgroup>> {
-    let cgroup_mount = Process::myself()?
-        .mountinfo()?
-        .into_iter()
-        .find(|m| m.fs_type == "cgroup");
+/// Determines the cgroup setup of the system. Systems typically have one of
+/// three setups:
+/// - Unified: Pure cgroup v2 system.
+/// - Legacy: Pure cgroup v1 system.
+/// - Hybrid: Hybrid is basically a cgroup v1 system, except for
+///   an additional unified hierarchy which doesn't have any
+///   controllers attached. Resource control can purely be achieved
+///   through the cgroup v1 hierarchy, not through the cgroup v2 hierarchy.
+pub fn get_cgroup_setup() -> Result<CgroupSetup> {
+    let default_root = Path::new(DEFAULT_CGROUP_ROOT);
+    match default_root.exists() {
+        true => {
+            // If the filesystem is of type cgroup2, the system is in unified mode.
+            // If the filesystem is tmpfs instead the system is either in legacy or
+            // hybrid mode. If a cgroup2 filesystem has been mounted under the "unified"
+            // folder we are in hybrid mode, otherwise we are in legacy mode.
+            let stat = statfs(default_root).with_context(|| {
+                format!(
+                    "failed to stat default cgroup root {}",
+                    &default_root.display()
+                )
+            })?;
+            if stat.filesystem_type() == CGROUP2_SUPER_MAGIC {
+                return Ok(CgroupSetup::Unified);
+            }

-    let cgroup2_mount = Process::myself()?
-        .mountinfo()?
-        .into_iter()
-        .find(|m| m.fs_type == "cgroup2");
+            if stat.filesystem_type() == TMPFS_MAGIC {
+                let unified = Path::new("/sys/fs/cgroup/unified");
+                if Path::new(unified).exists() {
+                    let stat = statfs(unified)
+                        .with_context(|| format!("failed to stat {}", unified.display()))?;
+                    if stat.filesystem_type() == CGROUP2_SUPER_MAGIC {
+                        return Ok(CgroupSetup::Hybrid);
+                    }
+                }

-    let mut cgroups = vec![];
-    if cgroup_mount.is_some() {
-        cgroups.push(Cgroup::V1);
+                return Ok(CgroupSetup::Legacy);
+            }
+        }
+        false => bail!("non default cgroup root not supported"),
    }

-    if cgroup2_mount.is_some() {
-        cgroups.push(Cgroup::V2);
-    }
-
-    Ok(cgroups)
+    bail!("failed to detect cgroup setup");
 }

 pub fn create_cgroup_manager<P: Into<PathBuf>>(
    cgroup_path: P,
    systemd_cgroup: bool,
 ) -> Result<Box<dyn CgroupManager>> {
-    let cgroup_mount = Process::myself()?
-        .mountinfo()?
-        .into_iter()
-        .find(|m| m.fs_type == "cgroup");
+    let cgroup_setup = get_cgroup_setup()?;

-    let cgroup2_mount = Process::myself()?
-        .mountinfo()?
-        .into_iter()
-        .find(|m| m.fs_type == "cgroup2");
-
-    match (cgroup_mount, cgroup2_mount) {
-        (Some(_), None) => {
+    match cgroup_setup {
+        CgroupSetup::Legacy | CgroupSetup::Hybrid => {
            log::info!("cgroup manager V1 will be used");
            Ok(Box::new(v1::manager::Manager::new(cgroup_path.into())?))
        }
-        (None, Some(cgroup2)) => {
-            log::info!("cgroup manager V2 will be used");
+        CgroupSetup::Unified => {
            if systemd_cgroup {
                if !booted()? {
                    bail!("systemd cgroup flag passed, but systemd support for managing cgroups is not available");
                }
                log::info!("systemd cgroup manager will be used");
                return Ok(Box::new(v2::SystemDCGroupManager::new(
-                    cgroup2.mount_point,
+                    DEFAULT_CGROUP_ROOT.into(),
                    cgroup_path.into(),
                )?));
            }
+            log::info!("cgroup manager V2 will be used");
            Ok(Box::new(v2::manager::Manager::new(
-                cgroup2.mount_point,
+                DEFAULT_CGROUP_ROOT.into(),
                cgroup_path.into(),
            )?))
        }
-        (Some(_), Some(cgroup2)) => {
-            let cgroup_override = env::var("YOUKI_PREFER_CGROUPV2");
-            match cgroup_override {
-                Ok(v) if v == "true" => {
-                    log::info!("cgroup manager V2 will be used");
-                    if systemd_cgroup {
-                        if !booted()? {
-                            bail!("systemd cgroup flag passed, but systemd support for managing cgroups is not available");
-                        }
-                        log::info!("systemd cgroup manager will be used");
-                        return Ok(Box::new(v2::SystemDCGroupManager::new(
-                            cgroup2.mount_point,
-                            cgroup_path.into(),
-                        )?));
-                    }
-                    Ok(Box::new(v2::manager::Manager::new(
-                        cgroup2.mount_point,
-                        cgroup_path.into(),
-                    )?))
-                }
-                _ => {
-                    log::info!("cgroup manager V1 will be used");
-                    Ok(Box::new(v1::manager::Manager::new(cgroup_path.into())?))
-                }
-            }
-        }
-        _ => bail!("could not find cgroup filesystem"),
    }
 }

--- a/integration_test.sh
+++ b/integration_test.sh
@ -30,14 +30,14 @@ test_cases=(
  # This case includes checking for features that are excluded from linux kernel 5.0, so even runc doesn't pass it.
  # ref. https://github.com/docker/cli/pull/2908
  # "linux_cgroups_relative_blkio/linux_cgroups_relative_blkio.t"
-  "linux_cgroups_relative_cpus/linux_cgroups_relative_cpus.t" 
+  "linux_cgroups_relative_cpus/linux_cgroups_relative_cpus.t"
  "linux_cgroups_relative_devices/linux_cgroups_relative_devices.t"
  "linux_cgroups_relative_hugetlb/linux_cgroups_relative_hugetlb.t"
  "linux_cgroups_relative_memory/linux_cgroups_relative_memory.t"
  "linux_cgroups_relative_network/linux_cgroups_relative_network.t"
  "linux_cgroups_relative_pids/linux_cgroups_relative_pids.t"
  "linux_devices/linux_devices.t"
-  # "linux_masked_paths/linux_masked_paths.t"
+  "linux_masked_paths/linux_masked_paths.t"
  "linux_mount_label/linux_mount_label.t"
  # This test case hangs on the Github Action. Runtime-tools has an issue filed from 2019 that the clean up step hangs. Otherwise, the test case passes.
  # Ref: https://github.com/opencontainers/runtime-tools/issues/698
@ -52,7 +52,7 @@ test_cases=(
  "linux_sysctl/linux_sysctl.t"
  # "linux_uid_mappings/linux_uid_mappings.t"
  "misc_props/misc_props.t"
-  # "mounts/mounts.t"
+  "mounts/mounts.t"
  # "pidfile/pidfile.t"
  "poststart/poststart.t"
  "poststart_fail/poststart_fail.t"
@ -63,10 +63,10 @@ test_cases=(
  "process/process.t"
  "process_capabilities/process_capabilities.t"
  "process_capabilities_fail/process_capabilities_fail.t"
-  # "process_oom_score_adj/process_oom_score_adj.t"
+  "process_oom_score_adj/process_oom_score_adj.t"
  "process_rlimits/process_rlimits.t"
  "process_rlimits_fail/process_rlimits_fail.t"
-  # "process_user/process_user.t"
+  "process_user/process_user.t"
  "root_readonly_true/root_readonly_true.t"
  # Record the tests that runc also fails to pass below, maybe we will fix this by origin integration test, issue: https://github.com/containers/youki/issues/56
  # "start/start.t"
@ -78,7 +78,7 @@ check_enviroment() {
  if [[ $test_case =~ .*(memory|hugetlb).t ]]; then
    if [[ ! -e "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes" ]]; then
        return 1
-    fi 
+    fi
  fi
 }

@ -97,13 +97,13 @@ for case in "${test_cases[@]}"; do
  fi

  if [ $PATTERN != "." ] && [[ ! $case =~ $PATTERN ]]; then
-    continue    
-  fi 
+    continue
+  fi

  echo "Running $case"
  logfile="./log/$case.log"
  mkdir -p "$(dirname $logfile)"
-  sudo RUST_BACKTRACE=1 RUNTIME=${RUNTIME} ${ROOT}/integration_test/src/github.com/opencontainers/runtime-tools/validation/$case >$logfile 2>&1
+  sudo RUST_BACKTRACE=1 RUNTIME=${RUNTIME} ${ROOT}/integration_test/src/github.com/opencontainers/runtime-tools/validation/$case >$logfile 2>&1 || (cat $logfile && exit 1)
  if [ 0 -ne $(grep "not ok" $logfile | wc -l ) ]; then
      cat $logfile
      exit 1
--- a/src/commands/info.rs
+++ b/src/commands/info.rs
@ -17,6 +17,7 @@ impl Info {
        print_os();
        print_hardware();
        print_cgroups();
+        print_namespaces();

        Ok(())
    }
@ -105,12 +106,11 @@ pub fn print_hardware() {

 /// Print cgroups info of system
 pub fn print_cgroups() {
-    if let Ok(cgroup_fs) = cgroups::common::get_supported_cgroup_fs() {
-        let cgroup_fs: Vec<String> = cgroup_fs.into_iter().map(|c| c.to_string()).collect();
-        println!("{:<18}{}", "cgroup version", cgroup_fs.join(" and "));
+    if let Ok(cgroup_setup) = cgroups::common::get_cgroup_setup() {
+        println!("{:<18}{}", "Cgroup setup", cgroup_setup);
    }

-    println!("cgroup mounts");
+    println!("Cgroup mounts");
    if let Ok(v1_mounts) = cgroups::v1::util::list_subsystem_mount_points() {
        let mut v1_mounts: Vec<String> = v1_mounts
            .iter()
@ -128,3 +128,44 @@ pub fn print_cgroups() {
        println!("  {:<16}{}", "unified", mount_point.display());
    }
 }
+
+pub fn print_namespaces() {
+    let uname = nix::sys::utsname::uname();
+    let kernel_config = Path::new("/boot").join(format!("config-{}", uname.release()));
+    if !kernel_config.exists() {
+        return;
+    }
+
+    if let Ok(content) = fs::read_to_string(kernel_config) {
+        if let Some(ns_enabled) = find_parameter(&content, "CONFIG_NAMESPACES") {
+            if ns_enabled == "y" {
+                println!("{:<18}enabled", "Namespaces");
+            } else {
+                println!("{:<18}disabled", "Namespaces");
+                return;
+            }
+        }
+
+        // mount namespace is always enabled if namespaces are enabled
+        println!("  {:<16}enabled", "mount");
+        print_feature_status(&content, "CONFIG_UTS_NS", "uts");
+        print_feature_status(&content, "CONFIG_IPC_NS", "ipc");
+        print_feature_status(&content, "CONFIG_USER_NS", "user");
+        print_feature_status(&content, "CONFIG_PID_NS", "pid");
+        print_feature_status(&content, "CONFIG_NET_NS", "network");
+    }
+}
+
+fn print_feature_status(config: &str, feature: &str, display: &str) {
+    if let Some(status_flag) = find_parameter(config, feature) {
+        let status = if status_flag == "y" {
+            "enabled"
+        } else {
+            "disabled"
+        };
+
+        println!("  {:<16}{}", display, status);
+    } else {
+        println!("  {:<16}disabled", display);
+    }
+}
--- a/src/commands/list.rs
+++ b/src/commands/list.rs
@ -55,7 +55,7 @@ impl List {
                container.id(),
                pid,
                container.status(),
-                container.bundle(),
+                container.bundle().to_string_lossy(),
                created,
                user_name.to_string_lossy()
            ));
--- a/src/container/builder_impl.rs
+++ b/src/container/builder_impl.rs
@ -9,7 +9,7 @@ use crate::{
 use anyhow::{Context, Result};
 use cgroups;
 use oci_spec::runtime::Spec;
-use std::{fs, os::unix::prelude::RawFd, path::PathBuf};
+use std::{fs, io::Write, os::unix::prelude::RawFd, path::PathBuf};

 use super::{Container, ContainerStatus};

@ -49,11 +49,10 @@ impl<'a> ContainerBuilderImpl<'a> {
    }

    fn run_container(&mut self) -> Result<()> {
-        prctl::set_dumpable(false).unwrap();
-
        let linux = self.spec.linux.as_ref().context("no linux in spec")?;
        let cgroups_path = utils::get_cgroup_path(&linux.cgroups_path, &self.container_id);
        let cmanager = cgroups::common::create_cgroup_manager(&cgroups_path, self.use_systemd)?;
+        let process = self.spec.process.as_ref().context("No process in spec")?;

        if self.init {
            if let Some(hooks) = self.spec.hooks.as_ref() {
@ -67,10 +66,37 @@ impl<'a> ContainerBuilderImpl<'a> {

        // Need to create the notify socket before we pivot root, since the unix
        // domain socket used here is outside of the rootfs of container. During
-        // exec, need to create the socket before we exter into existing mount
+        // exec, need to create the socket before we enter into existing mount
        // namespace.
        let notify_socket: NotifyListener = NotifyListener::new(&self.notify_path)?;

+        // If Out-of-memory score adjustment is set in specification.  set the score
+        // value for the current process check
+        // https://dev.to/rrampage/surviving-the-linux-oom-killer-2ki9 for some more
+        // information.
+        //
+        // This has to be done before !dumpable because /proc/self/oom_score_adj
+        // is not writeable unless you're an privileged user (if !dumpable is
+        // set). All children inherit their parent's oom_score_adj value on
+        // fork(2) so this will always be propagated properly.
+        if let Some(oom_score_adj) = process.oom_score_adj {
+            log::debug!("Set OOM score to {}", oom_score_adj);
+            let mut f = fs::File::create("/proc/self/oom_score_adj")?;
+            f.write_all(oom_score_adj.to_string().as_bytes())?;
+        }
+
+        // Make the process non-dumpable, to avoid various race conditions that
+        // could cause processes in namespaces we're joining to access host
+        // resources (or potentially execute code).
+        //
+        // However, if the number of namespaces we are joining is 0, we are not
+        // going to be switching to a different security context. Thus setting
+        // ourselves to be non-dumpable only breaks things (like rootless
+        // containers), which is the recommendation from the kernel folks.
+        if linux.namespaces.is_some() {
+            prctl::set_dumpable(false).unwrap();
+        }
+
        // This init_args will be passed to the container init process,
        // therefore we will have to move all the variable by value. Since self
        // is a shared reference, we have to clone these variables here.
@ -83,6 +109,7 @@ impl<'a> ContainerBuilderImpl<'a> {
            notify_socket,
            preserve_fds: self.preserve_fds,
            container: self.container.clone(),
+            rootless: self.rootless.clone(),
        };
        let intermediate_pid = fork::container_fork(|| {
            // The fds in the pipe is duplicated during fork, so we first close
@ -94,7 +121,7 @@ impl<'a> ContainerBuilderImpl<'a> {
                .close()
                .context("Failed to close unused receiver")?;

-            init::container_intermidiate(init_args, receiver_from_main, sender_to_main)
+            init::container_intermediate(init_args, receiver_from_main, sender_to_main)
        })?;
        // Close down unused fds. The corresponding fds are duplicated to the
        // child process during fork.
@ -111,7 +138,12 @@ impl<'a> ContainerBuilderImpl<'a> {
        if self.rootless.is_some() {
            receiver_from_intermediate.wait_for_mapping_request()?;
            log::debug!("write mapping for pid {:?}", intermediate_pid);
-            utils::write_file(format!("/proc/{}/setgroups", intermediate_pid), "deny")?;
+            let rootless = self.rootless.as_ref().unwrap();
+            if !rootless.privileged {
+                // The main process is running as an unprivileged user and cannot write the mapping
+                // until "deny" has been written to setgroups. See CVE-2014-8989.
+                utils::write_file(format!("/proc/{}/setgroups", intermediate_pid), "deny")?;
+            }
            rootless::write_uid_mapping(intermediate_pid, self.rootless.as_ref())?;
            rootless::write_gid_mapping(intermediate_pid, self.rootless.as_ref())?;
            sender_to_intermediate.mapping_written()?;
@ -120,15 +152,15 @@ impl<'a> ContainerBuilderImpl<'a> {
        let init_pid = receiver_from_intermediate.wait_for_intermediate_ready()?;
        log::debug!("init pid is {:?}", init_pid);

-        cmanager
-            .add_task(init_pid)
-            .context("Failed to add tasks to cgroup manager")?;
-
        if self.rootless.is_none() && linux.resources.is_some() && self.init {
            let controller_opt = cgroups::common::ControllerOpt {
                resources: linux.resources.clone().unwrap(),
                ..Default::default()
            };
+            cmanager
+                .add_task(init_pid)
+                .context("Failed to add tasks to cgroup manager")?;
+
            cmanager
                .apply(&controller_opt)
                .context("Failed to apply resource limits through cgroup")?;
--- a/src/container/container.rs
+++ b/src/container/container.rs
@ -38,11 +38,11 @@ impl Container {
        container_id: &str,
        status: ContainerStatus,
        pid: Option<i32>,
-        bundle: &str,
+        bundle: &Path,
        container_root: &Path,
    ) -> Result<Self> {
        let container_root = fs::canonicalize(container_root)?;
-        let state = State::new(container_id, status, pid, bundle);
+        let state = State::new(container_id, status, pid, bundle.to_path_buf());
        Ok(Self {
            state,
            root: container_root,
@ -56,6 +56,7 @@ impl Container {
    pub fn status(&self) -> ContainerStatus {
        self.state.status
    }
+
    pub fn refresh_status(&mut self) -> Result<Self> {
        let new_status = match self.pid() {
            Some(pid) => {
@ -154,8 +155,8 @@ impl Container {
        None
    }

-    pub fn bundle(&self) -> String {
-        self.state.bundle.clone()
+    pub fn bundle(&self) -> &PathBuf {
+        &self.state.bundle
    }

    pub fn set_systemd(mut self, should_use: bool) -> Self {
@ -201,3 +202,35 @@ impl Container {
        Ok(spec)
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::env;
+
+    use super::*;
+    use anyhow::Result;
+
+    #[test]
+    fn test_set_id() -> Result<()> {
+        let dir = env::temp_dir();
+        let container = Container::new("container_id", ContainerStatus::Created, None, &dir, &dir)?;
+        let container = container.set_pid(1);
+        assert_eq!(container.pid(), Some(Pid::from_raw(1)));
+        Ok(())
+    }
+
+    #[test]
+    fn test_basic_getter() -> Result<()> {
+        let container = Container::new(
+            "container_id",
+            ContainerStatus::Created,
+            None,
+            &PathBuf::from("."),
+            &PathBuf::from("."),
+        )?;
+
+        assert_eq!(container.bundle(), &PathBuf::from("."));
+        assert_eq!(container.root, fs::canonicalize(PathBuf::from("."))?);
+        Ok(())
+    }
+}
--- a/src/container/init_builder.rs
+++ b/src/container/init_builder.rs
@ -1,7 +1,7 @@
 use anyhow::{bail, Context, Result};
 use nix::unistd;
 use oci_spec::runtime::Spec;
-use rootless::detect_rootless;
+use rootless::Rootless;
 use std::{
    fs,
    path::{Path, PathBuf},
@ -65,7 +65,7 @@ impl InitContainerBuilder {
            None
        };

-        let rootless = detect_rootless(&spec)?;
+        let rootless = Rootless::new(&spec)?;
        let mut builder_impl = ContainerBuilderImpl {
            init: true,
            syscall: self.base.syscall,
@ -121,7 +121,7 @@ impl InitContainerBuilder {
            &self.base.container_id,
            ContainerStatus::Creating,
            None,
-            self.bundle.as_path().to_str().unwrap(),
+            self.bundle.as_path(),
            container_dir,
        )?;
        container.save()?;
--- a/src/container/state.rs
+++ b/src/container/state.rs
@ -24,6 +24,7 @@ pub enum ContainerStatus {
    // The container process has paused
    Paused,
 }
+
 impl Default for ContainerStatus {
    fn default() -> Self {
        ContainerStatus::Creating
@ -84,7 +85,7 @@ pub struct State {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub pid: Option<i32>,
    // Bundle is the path to the container's bundle directory.
-    pub bundle: String,
+    pub bundle: PathBuf,
    // Annotations are key values associated with the container.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub annotations: Option<HashMap<String, String>>,
@ -105,14 +106,14 @@ impl State {
        container_id: &str,
        status: ContainerStatus,
        pid: Option<i32>,
-        bundle: &str,
+        bundle: PathBuf,
    ) -> Self {
        Self {
            oci_version: "v1.0.2".to_string(),
            id: container_id.to_string(),
            status,
            pid,
-            bundle: bundle.to_string(),
+            bundle,
            annotations: Some(HashMap::default()),
            created: None,
            creator: None,
@ -157,3 +158,58 @@ impl State {
        container_root.join(Self::STATE_FILE_PATH)
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_creating_status() {
+        let cstatus = ContainerStatus::default();
+        assert!(!cstatus.can_start());
+        assert!(!cstatus.can_delete());
+        assert!(!cstatus.can_kill());
+        assert!(!cstatus.can_pause());
+        assert!(!cstatus.can_resume());
+    }
+
+    #[test]
+    fn test_create_status() {
+        let cstatus = ContainerStatus::Created;
+        assert!(cstatus.can_start());
+        assert!(!cstatus.can_delete());
+        assert!(cstatus.can_kill());
+        assert!(!cstatus.can_pause());
+        assert!(!cstatus.can_resume());
+    }
+
+    #[test]
+    fn test_running_status() {
+        let cstatus = ContainerStatus::Running;
+        assert!(!cstatus.can_start());
+        assert!(!cstatus.can_delete());
+        assert!(cstatus.can_kill());
+        assert!(cstatus.can_pause());
+        assert!(!cstatus.can_resume());
+    }
+
+    #[test]
+    fn test_stopped_status() {
+        let cstatus = ContainerStatus::Stopped;
+        assert!(!cstatus.can_start());
+        assert!(cstatus.can_delete());
+        assert!(!cstatus.can_kill());
+        assert!(!cstatus.can_pause());
+        assert!(!cstatus.can_resume());
+    }
+
+    #[test]
+    fn test_paused_status() {
+        let cstatus = ContainerStatus::Paused;
+        assert!(!cstatus.can_start());
+        assert!(!cstatus.can_delete());
+        assert!(cstatus.can_kill());
+        assert!(!cstatus.can_pause());
+        assert!(cstatus.can_resume());
+    }
+}
--- a/src/container/tenant_builder.rs
+++ b/src/container/tenant_builder.rs
@ -5,19 +5,19 @@ use oci_spec::runtime::{
    Capabilities as SpecCapabilities, LinuxCapabilities, LinuxNamespace, LinuxNamespaceType,
    Process, Spec,
 };
+use procfs::process::Namespace;

 use std::{
    collections::HashMap,
    convert::TryFrom,
-    ffi::{CString, OsString},
    fs,
-    os::unix::prelude::{OsStrExt, RawFd},
+    os::unix::prelude::RawFd,
    path::{Path, PathBuf},
    str::FromStr,
 };

 use crate::capabilities::from_cap;
-use crate::{notify_socket::NotifySocket, rootless::detect_rootless, tty, utils};
+use crate::{notify_socket::NotifySocket, rootless::Rootless, tty, utils};

 use super::{builder::ContainerBuilder, builder_impl::ContainerBuilderImpl, Container};

@ -104,7 +104,7 @@ impl TenantContainerBuilder {
        let csocketfd = self.setup_tty_socket(&container_dir)?;

        let use_systemd = self.should_use_systemd(&container);
-        let rootless = detect_rootless(&spec)?;
+        let rootless = Rootless::new(&spec)?;

        let mut builder_impl = ContainerBuilderImpl {
            init: false,
@ -358,62 +358,3 @@ impl TenantContainerBuilder {
        }
    }
 }
-
-// Can be removed once https://github.com/eminence/procfs/pull/135 is available
-trait GetNamespace {
-    fn namespaces(&self) -> Result<Vec<Namespace>>;
-}
-
-impl GetNamespace for procfs::process::Process {
-    /// Describes namespaces to which the process with the corresponding PID belongs.
-    /// Doc reference: https://man7.org/linux/man-pages/man7/namespaces.7.html
-    fn namespaces(&self) -> Result<Vec<Namespace>> {
-        let proc_path = PathBuf::from(format!("/proc/{}", self.pid()));
-        let ns = proc_path.join("ns");
-        let mut namespaces = Vec::new();
-        for entry in fs::read_dir(ns)? {
-            let entry = entry?;
-            let path = entry.path();
-            let ns_type = entry.file_name();
-            let cstr = CString::new(path.as_os_str().as_bytes()).unwrap();
-
-            let mut stat = unsafe { std::mem::zeroed() };
-            if unsafe { libc::stat(cstr.as_ptr(), &mut stat) } != 0 {
-                bail!("Unable to stat {:?}", path);
-            }
-
-            namespaces.push(Namespace {
-                ns_type,
-                path,
-                identifier: stat.st_ino,
-                device_id: stat.st_dev,
-            })
-        }
-
-        Ok(namespaces)
-    }
-}
-
-/// Information about a namespace
-///
-/// See also the [Process::namespaces()] method
-#[derive(Debug, Clone)]
-pub struct Namespace {
-    /// Namespace type
-    pub ns_type: OsString,
-    /// Handle to the namespace
-    pub path: PathBuf,
-    /// Namespace identifier (inode number)
-    pub identifier: u64,
-    /// Device id of the namespace
-    pub device_id: u64,
-}
-
-impl PartialEq for Namespace {
-    fn eq(&self, other: &Self) -> bool {
-        // see https://lore.kernel.org/lkml/87poky5ca9.fsf@xmission.com/
-        self.identifier == other.identifier && self.device_id == other.device_id
-    }
-}
-
-impl Eq for Namespace {}
--- a/src/main.rs
+++ b/src/main.rs
@ -7,7 +7,7 @@ use std::path::PathBuf;

 use anyhow::bail;
 use anyhow::Result;
-use clap::Clap;
+use clap::{crate_version, Clap};

 use nix::sys::stat::Mode;
 use nix::unistd::getuid;
@ -25,14 +25,14 @@ use youki::commands::run;
 use youki::commands::spec_json;
 use youki::commands::start;
 use youki::commands::state;
-use youki::rootless::should_use_rootless;
+use youki::rootless::rootless_required;
 use youki::utils::{self, create_dir_all_with_mode};

 // High-level commandline option definition
 // This takes global options as well as individual commands as specified in [OCI runtime-spec](https://github.com/opencontainers/runtime-spec/blob/master/runtime.md)
 // Also check [runc commandline documentation](https://github.com/opencontainers/runc/blob/master/man/runc.8.md) for more explanation
 #[derive(Clap, Debug)]
-#[clap(version = "0.0.0", author = "youki team")]
+#[clap(version = crate_version!(), author = "youki team")]
 struct Opts {
    /// root directory to store container state
    #[clap(short, long)]
@ -53,33 +53,33 @@ struct Opts {
 // Also for a short information, check [runc commandline documentation](https://github.com/opencontainers/runc/blob/master/man/runc.8.md)
 #[derive(Clap, Debug)]
 enum SubCommand {
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Create(create::Create),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Start(start::Start),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Run(run::Run),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Exec(exec::Exec),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Kill(kill::Kill),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Delete(delete::Delete),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    State(state::State),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Info(info::Info),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Spec(spec_json::SpecJson),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    List(list::List),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Pause(pause::Pause),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Resume(resume::Resume),
-    #[clap(version = "0.0.0", author = "youki team")]
+    #[clap(version = crate_version!(), author = "youki team")]
    Events(events::Events),
-    #[clap(version = "0.0.0", author = "youki team", setting=clap::AppSettings::AllowLeadingHyphen)]
+    #[clap(version = crate_version!(), author = "youki team", setting=clap::AppSettings::AllowLeadingHyphen)]
    Ps(ps::Ps),
 }

@ -118,7 +118,7 @@ fn determine_root_path(root_path: Option<PathBuf>) -> Result<PathBuf> {
        return Ok(path);
    }

-    if !should_use_rootless() {
+    if !rootless_required() {
        let default = PathBuf::from("/run/youki");
        utils::create_dir_all(&default)?;
        return Ok(default);
--- a/src/process/channel.rs
+++ b/src/process/channel.rs
@ -230,48 +230,53 @@ fn new_pipe() -> Result<(Sender, Receiver)> {
 }

 #[cfg(test)]
+// Tests become unstable if not serial. The cause is not known.
 mod tests {
    use super::*;
    use anyhow::Context;
    use nix::sys::wait;
    use nix::unistd;
+    use serial_test::serial;

    #[test]
+    #[serial]
    fn test_channel_intermadiate_ready() -> Result<()> {
        let (sender, receiver) = &mut intermediate_to_main()?;
        match unsafe { unistd::fork()? } {
            unistd::ForkResult::Parent { child } => {
+                wait::waitpid(child, None)?;
                let pid = receiver
                    .wait_for_intermediate_ready()
                    .with_context(|| "Failed to wait for intermadiate ready")?;
+                receiver.close()?;
                assert_eq!(pid, child);
-                wait::waitpid(child, None)?;
            }
            unistd::ForkResult::Child => {
                let pid = unistd::getpid();
-                sender
-                    .intermediate_ready(pid)
-                    .with_context(|| "Failed to send intermadiate ready")?;
+                sender.intermediate_ready(pid)?;
+                sender.close()?;
                std::process::exit(0);
            }
        };
+
        Ok(())
    }

    #[test]
+    #[serial]
    fn test_channel_id_mapping_request() -> Result<()> {
        let (sender, receiver) = &mut intermediate_to_main()?;
        match unsafe { unistd::fork()? } {
            unistd::ForkResult::Parent { child } => {
-                receiver
-                    .wait_for_mapping_request()
-                    .with_context(|| "Failed to wait for mapping ack")?;
                wait::waitpid(child, None)?;
+                receiver.wait_for_mapping_request()?;
+                receiver.close()?;
            }
            unistd::ForkResult::Child => {
                sender
                    .identifier_mapping_request()
                    .with_context(|| "Failed to send mapping written")?;
+                sender.close()?;
                std::process::exit(0);
            }
        };
@ -280,14 +285,13 @@ mod tests {
    }

    #[test]
+    #[serial]
    fn test_channel_id_mapping_ack() -> Result<()> {
        let (sender, receiver) = &mut main_to_intermediate()?;
        match unsafe { unistd::fork()? } {
            unistd::ForkResult::Parent { child } => {
-                receiver
-                    .wait_for_mapping_ack()
-                    .with_context(|| "Failed to wait for mapping ack")?;
                wait::waitpid(child, None)?;
+                receiver.wait_for_mapping_ack()?;
            }
            unistd::ForkResult::Child => {
                sender
@ -301,26 +305,29 @@ mod tests {
    }

    #[test]
+    #[serial]
    fn test_channel_init_ready() -> Result<()> {
        let (sender, receiver) = &mut init_to_intermediate()?;
        match unsafe { unistd::fork()? } {
            unistd::ForkResult::Parent { child } => {
-                receiver
-                    .wait_for_init_ready()
-                    .with_context(|| "Failed to wait for init ready")?;
                wait::waitpid(child, None)?;
+                receiver.wait_for_init_ready()?;
+                receiver.close()?;
            }
            unistd::ForkResult::Child => {
                sender
                    .init_ready()
                    .with_context(|| "Failed to send init ready")?;
+                sender.close()?;
                std::process::exit(0);
            }
        };
+
        Ok(())
    }

    #[test]
+    #[serial]
    fn test_channel_intermedaite_graceful_exit() -> Result<()> {
        let (sender, receiver) = &mut intermediate_to_main()?;
        match unsafe { unistd::fork()? } {
@ -343,6 +350,7 @@ mod tests {
    }

    #[test]
+    #[serial]
    fn test_channel_init_graceful_exit() -> Result<()> {
        let (sender, receiver) = &mut init_to_intermediate()?;
        match unsafe { unistd::fork()? } {
--- a/src/process/init.rs
+++ b/src/process/init.rs
@ -7,14 +7,15 @@ use nix::{
    sys::statfs,
    unistd::{self, Gid, Uid},
 };
-use oci_spec::runtime::{LinuxNamespaceType, Spec};
+use oci_spec::runtime::{LinuxNamespaceType, Spec, User};
 use std::collections::HashMap;
 use std::{
    env,
    os::unix::{io::AsRawFd, prelude::RawFd},
 };
-use std::{fs, io::Write, path::Path, path::PathBuf};
+use std::{fs, path::Path, path::PathBuf};

+use crate::rootless::Rootless;
 use crate::{
    capabilities,
    container::Container,
@ -146,7 +147,41 @@ fn readonly_path(path: &str) -> Result<()> {
    Ok(())
 }

-pub struct ContainerInitArgs {
+// For files, bind mounts /dev/null over the top of the specified path.
+// For directories, mounts read-only tmpfs over the top of the specified path.
+fn masked_path(path: &str, mount_label: &Option<String>) -> Result<()> {
+    match nix_mount::<str, str, str, str>(
+        Some("/dev/null"),
+        path,
+        None::<&str>,
+        MsFlags::MS_BIND,
+        None::<&str>,
+    ) {
+        // ignore error if path is not exist.
+        Err(nix::errno::Errno::ENOENT) => {
+            log::warn!("masked path {:?} not exist", path);
+            return Ok(());
+        }
+        Err(nix::errno::Errno::ENOTDIR) => {
+            let label = match mount_label {
+                Some(l) => format!("context={}", l),
+                None => "".to_string(),
+            };
+            let _ = nix_mount(
+                Some("tmpfs"),
+                path,
+                Some("tmpfs"),
+                MsFlags::MS_RDONLY,
+                Some(label.as_str()),
+            );
+        }
+        Err(err) => bail!(err),
+        Ok(_) => {}
+    };
+    Ok(())
+}
+
+pub struct ContainerInitArgs<'a> {
    /// Flag indicating if an init or a tenant container should be created
    pub init: bool,
    /// Interface to operating system primitives
@ -163,9 +198,11 @@ pub struct ContainerInitArgs {
    pub preserve_fds: i32,
    /// Container state
    pub container: Option<Container>,
+    /// Options for rootless containers
+    pub rootless: Option<Rootless<'a>>,
 }

-pub fn container_intermidiate(
+pub fn container_intermediate(
    args: ContainerInitArgs,
    receiver_from_main: &mut channel::ReceiverFromMain,
    sender_to_main: &mut channel::SenderIntermediateToMain,
@ -175,17 +212,6 @@ pub fn container_intermidiate(
    let linux = spec.linux.as_ref().context("no linux in spec")?;
    let namespaces = Namespaces::from(linux.namespaces.as_ref());

-    // if Out-of-memory score adjustment is set in specification.  set the score
-    // value for the current process check
-    // https://dev.to/rrampage/surviving-the-linux-oom-killer-2ki9 for some more
-    // information
-    if let Some(ref process) = spec.process {
-        if let Some(oom_score_adj) = process.oom_score_adj {
-            let mut f = fs::File::create("/proc/self/oom_score_adj")?;
-            f.write_all(oom_score_adj.to_string().as_bytes())?;
-        }
-    }
-
    // if new user is specified in specification, this will be true and new
    // namespace will be created, check
    // https://man7.org/linux/man-pages/man7/user_namespaces.7.html for more
@ -362,6 +388,13 @@ pub fn container_init(
        }
    }

+    if let Some(paths) = &linux.masked_paths {
+        // mount masked path
+        for path in paths {
+            masked_path(path, &linux.mount_label).context("Failed to set masked path")?;
+        }
+    }
+
    let cwd = format!("{}", proc.cwd.display());
    let do_chdir = if cwd.is_empty() {
        false
@ -376,6 +409,9 @@ pub fn container_init(
        }
    };

+    set_supplementary_gids(&proc.user, &args.rootless)
+        .context("failed to set supplementary gids")?;
+
    command
        .set_id(Uid::from_raw(proc.user.uid), Gid::from_raw(proc.user.gid))
        .context("Failed to configure uid and gid")?;
@ -469,11 +505,70 @@ pub fn container_init(
    unreachable!();
 }

+// Before 3.19 it was possible for an unprivileged user to enter an user namespace,
+// become root and then call setgroups in order to drop membership in supplementary
+// groups. This allowed access to files which blocked access based on being a member
+// of these groups (see CVE-2014-8989)
+//
+// This leaves us with three scenarios:
+//
+// Unprivileged user starting a rootless container: The main process is running as an
+// unprivileged user and therefore cannot write the mapping until "deny" has been written
+// to /proc/{pid}/setgroups. Once written /proc/{pid}/setgroups cannot be reset and the
+// setgroups system call will be disabled for all processes in this user namespace. This
+// also means that we should detect if the user is unprivileged and additional gids have
+// been specified and bail out early as this can never work. This is not handled here,
+// but during the validation for rootless containers.
+//
+// Privileged user starting a rootless container: It is not necessary to write "deny" to
+// /proc/setgroups in order to create the gid mapping and therefore we don't. This means
+// that setgroups could be used to drop groups, but this is fine as the user is privileged
+// and could do so anyway.
+// We already have checked during validation if the specified supplemental groups fall into
+// the range that are specified in the gid mapping and bail out early if they do not.
+//
+// Privileged user starting a normal container: Just add the supplementary groups.
+//
+fn set_supplementary_gids(user: &User, rootless: &Option<Rootless>) -> Result<()> {
+    if let Some(additional_gids) = &user.additional_gids {
+        if additional_gids.is_empty() {
+            return Ok(());
+        }
+
+        let setgroups =
+            fs::read_to_string("/proc/self/setgroups").context("failed to read setgroups")?;
+        if setgroups.trim() == "deny" {
+            bail!("cannot set supplementary gids, setgroup is disabled");
+        }
+
+        let gids: Vec<Gid> = additional_gids
+            .iter()
+            .map(|gid| Gid::from_raw(*gid))
+            .collect();
+
+        match rootless {
+            Some(r) if r.privileged => {
+                nix::unistd::setgroups(&gids).context("failed to set supplementary gids")?;
+            }
+            None => {
+                nix::unistd::setgroups(&gids).context("failed to set supplementary gids")?;
+            }
+            // this should have been detected during validation
+            _ => unreachable!(
+                "unprivileged users cannot set supplementary gids in rootless container"
+            ),
+        }
+    }
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
    use anyhow::{bail, Result};
    use nix::{fcntl, sys, unistd};
+    use serial_test::serial;
    use std::fs;

    #[test]
@ -501,6 +596,7 @@ mod tests {
    }

    #[test]
+    #[serial]
    fn test_cleanup_file_descriptors() -> Result<()> {
        // Open a fd without the CLOEXEC flag. Rust automatically adds the flag,
        // so we use fcntl::open here for more control.
--- a/src/rootfs.rs
+++ b/src/rootfs.rs
@ -9,7 +9,7 @@ use nix::mount::mount as nix_mount;
 use nix::mount::MsFlags;
 use nix::sys::stat::{mknod, umask};
 use nix::sys::stat::{Mode, SFlag};
-use nix::unistd::{chdir, chown, close, getcwd};
+use nix::unistd::{chown, close};
 use nix::unistd::{Gid, Uid};
 use oci_spec::runtime::{LinuxDevice, LinuxDeviceType, Mount, Spec};
 use std::fs::OpenOptions;
@ -49,7 +49,7 @@ pub fn prepare_rootfs(spec: &Spec, rootfs: &Path, bind_devices: bool) -> Result<
            log::debug!("Mount... {:?}", mount);
            let (flags, data) = parse_mount(mount);
            let mount_label = linux.mount_label.as_ref();
-            if mount.typ.as_ref().context("no type in mount spec")? == "cgroup" {
+            if mount.typ == Some("cgroup".to_string()) {
                // skip
                log::warn!("A feature of cgroup is unimplemented.");
            } else if mount.destination == PathBuf::from("/dev") {
@ -68,17 +68,18 @@ pub fn prepare_rootfs(spec: &Spec, rootfs: &Path, bind_devices: bool) -> Result<
        }
    }

-    let olddir = getcwd()?;
-    chdir(rootfs)?;
    setup_default_symlinks(rootfs).context("Failed to setup default symlinks")?;
    if let Some(added_devices) = linux.devices.as_ref() {
-        create_devices(default_devices().iter().chain(added_devices), bind_devices)
+        create_devices(
+            rootfs,
+            default_devices().iter().chain(added_devices),
+            bind_devices,
+        )
    } else {
-        create_devices(default_devices().iter(), bind_devices)
+        create_devices(rootfs, default_devices().iter(), bind_devices)
    }?;
-    setup_ptmx(rootfs)?;
-    chdir(&olddir)?;

+    setup_ptmx(rootfs)?;
    Ok(())
 }

@ -89,8 +90,7 @@ fn setup_ptmx(rootfs: &Path) -> Result<()> {
        }
    }

-    symlink("pts/ptmx", "dev/ptmx").context("Failed to symlink ptmx")?;
-
+    symlink("pts/ptmx", rootfs.join("dev/ptmx")).context("failed to symlink ptmx")?;
    Ok(())
 }

@ -171,7 +171,7 @@ pub fn default_devices() -> Vec<LinuxDevice> {
    ]
 }

-fn create_devices<'a, I>(devices: I, bind: bool) -> Result<()>
+fn create_devices<'a, I>(rootfs: &Path, devices: I, bind: bool) -> Result<()>
 where
    I: Iterator<Item = &'a LinuxDevice>,
 {
@ -183,7 +183,7 @@ where
                    panic!("{} is not a valid device path", dev.path.display());
                }

-                bind_dev(dev)
+                bind_dev(rootfs, dev)
            })
            .collect::<Result<Vec<_>>>()?;
    } else {
@ -193,7 +193,7 @@ where
                    panic!("{} is not a valid device path", dev.path.display());
                }

-                mknod_dev(dev)
+                mknod_dev(rootfs, dev)
            })
            .collect::<Result<Vec<_>>>()?;
    }
@ -202,15 +202,17 @@ where
    Ok(())
 }

-fn bind_dev(dev: &LinuxDevice) -> Result<()> {
+fn bind_dev(rootfs: &Path, dev: &LinuxDevice) -> Result<()> {
+    let full_container_path = rootfs.join(dev.path.as_in_container()?);
+
    let fd = open(
-        &dev.path.as_in_container()?,
+        &full_container_path,
        OFlag::O_RDWR | OFlag::O_CREAT,
        Mode::from_bits_truncate(0o644),
    )?;
    close(fd)?;
    nix_mount(
-        Some(&*dev.path.as_in_container()?),
+        Some(&full_container_path),
        &dev.path,
        None::<&str>,
        MsFlags::MS_BIND,
@ -228,21 +230,23 @@ fn to_sflag(dev_type: LinuxDeviceType) -> SFlag {
    }
 }

-fn mknod_dev(dev: &LinuxDevice) -> Result<()> {
+fn mknod_dev(rootfs: &Path, dev: &LinuxDevice) -> Result<()> {
    fn makedev(major: i64, minor: i64) -> u64 {
        ((minor & 0xff)
            | ((major & 0xfff) << 8)
            | ((minor & !0xff) << 12)
            | ((major & !0xfff) << 32)) as u64
    }
+
+    let full_container_path = rootfs.join(dev.path.as_in_container()?);
    mknod(
-        &dev.path.as_in_container()?,
+        &full_container_path,
        to_sflag(dev.typ),
        Mode::from_bits_truncate(dev.file_mode.unwrap_or(0)),
        makedev(dev.major, dev.minor),
    )?;
    chown(
-        &dev.path.as_in_container()?,
+        &full_container_path,
        dev.uid.map(Uid::from_raw),
        dev.gid.map(Gid::from_raw),
    )?;
@ -257,9 +261,9 @@ fn mount_to_container(
    data: &str,
    label: Option<&String>,
 ) -> Result<()> {
-    let typ = m.typ.as_ref().context("no type in mount spec")?;
+    let typ = m.typ.as_deref();
    let d = if let Some(l) = label {
-        if typ != "proc" && typ != "sysfs" {
+        if typ != Some("proc") && typ != Some("sysfs") {
            if data.is_empty() {
                format!("context=\"{}\"", l)
            } else {
@ -278,7 +282,7 @@ fn mount_to_container(
    );
    let dest = Path::new(&dest_for_host);
    let source = m.source.as_ref().context("no source in mount spec")?;
-    let src = if typ == "bind" {
+    let src = if typ == Some("bind") {
        let src = canonicalize(source)?;
        let dir = if src.is_file() {
            Path::new(&dest).parent().unwrap()
@ -301,12 +305,11 @@ fn mount_to_container(
        PathBuf::from(source)
    };

-    if let Err(errno) = nix_mount(Some(&*src), dest, Some(&*typ.as_str()), flags, Some(&*d)) {
+    if let Err(errno) = nix_mount(Some(&*src), dest, typ, flags, Some(&*d)) {
        if !matches!(errno, Errno::EINVAL) {
            bail!("mount of {:?} failed", m.destination);
        }
-
-        nix_mount(Some(&*src), dest, Some(&*typ.as_str()), flags, Some(data))?;
+        nix_mount(Some(&*src), dest, typ, flags, Some(data))?;
    }

    if flags.contains(MsFlags::MS_BIND)
@ -359,15 +362,15 @@ fn parse_mount(m: &Mount) -> (MsFlags, String) {
                "rbind" => Some((false, MsFlags::MS_BIND | MsFlags::MS_REC)),
                "unbindable" => Some((false, MsFlags::MS_UNBINDABLE)),
                "runbindable" => Some((false, MsFlags::MS_UNBINDABLE | MsFlags::MS_REC)),
-                "private" => Some((false, MsFlags::MS_PRIVATE)),
-                "rprivate" => Some((false, MsFlags::MS_PRIVATE | MsFlags::MS_REC)),
-                "shared" => Some((false, MsFlags::MS_SHARED)),
-                "rshared" => Some((false, MsFlags::MS_SHARED | MsFlags::MS_REC)),
-                "slave" => Some((false, MsFlags::MS_SLAVE)),
-                "rslave" => Some((false, MsFlags::MS_SLAVE | MsFlags::MS_REC)),
-                "relatime" => Some((false, MsFlags::MS_RELATIME)),
+                "private" => Some((true, MsFlags::MS_PRIVATE)),
+                "rprivate" => Some((true, MsFlags::MS_PRIVATE | MsFlags::MS_REC)),
+                "shared" => Some((true, MsFlags::MS_SHARED)),
+                "rshared" => Some((true, MsFlags::MS_SHARED | MsFlags::MS_REC)),
+                "slave" => Some((true, MsFlags::MS_SLAVE)),
+                "rslave" => Some((true, MsFlags::MS_SLAVE | MsFlags::MS_REC)),
+                "relatime" => Some((true, MsFlags::MS_RELATIME)),
                "norelatime" => Some((true, MsFlags::MS_RELATIME)),
-                "strictatime" => Some((false, MsFlags::MS_STRICTATIME)),
+                "strictatime" => Some((true, MsFlags::MS_STRICTATIME)),
                "nostrictatime" => Some((true, MsFlags::MS_STRICTATIME)),
                _ => None,
            } {
--- a/src/rootless.rs
+++ b/src/rootless.rs
@ -18,6 +18,42 @@ pub struct Rootless<'a> {
    pub gid_mappings: Option<&'a Vec<LinuxIdMapping>>,
    /// Info on the user namespaces
    user_namespace: Option<LinuxNamespace>,
+    /// Is rootless container requested by a privileged
+    /// user
+    pub privileged: bool,
+}
+
+impl<'a> Rootless<'a> {
+    pub fn new(spec: &'a Spec) -> Result<Option<Rootless<'a>>> {
+        let linux = spec.linux.as_ref().context("no linux in spec")?;
+        let namespaces = Namespaces::from(linux.namespaces.as_ref());
+        let user_namespace = namespaces.get(LinuxNamespaceType::User);
+
+        // If conditions requires us to use rootless, we must either create a new
+        // user namespace or enter an exsiting.
+        if rootless_required() && user_namespace.is_none() {
+            bail!("rootless container requires valid user namespace definition");
+        }
+
+        if user_namespace.is_some() && user_namespace.unwrap().path.is_none() {
+            log::debug!("rootless container should be created");
+            log::warn!(
+                "resource constraints and multi id mapping is unimplemented for rootless containers"
+            );
+
+            validate(spec).context("The spec failed to comply to rootless requirement")?;
+            let mut rootless = Rootless::from(linux);
+            if let Some((uid_binary, gid_binary)) = lookup_map_binaries(linux)? {
+                rootless.newuidmap = Some(uid_binary);
+                rootless.newgidmap = Some(gid_binary);
+            }
+
+            Ok(Some(rootless))
+        } else {
+            log::debug!("This is NOT a rootless container");
+            Ok(None)
+        }
+    }
 }

 impl<'a> From<&'a Linux> for Rootless<'a> {
@ -30,47 +66,13 @@ impl<'a> From<&'a Linux> for Rootless<'a> {
            uid_mappings: linux.uid_mappings.as_ref(),
            gid_mappings: linux.gid_mappings.as_ref(),
            user_namespace: user_namespace.cloned(),
+            privileged: nix::unistd::geteuid().is_root(),
        }
    }
 }

-// If user namespace is detected, then we are going into rootless.
-// If we are not root, check if we are user namespace.
-pub fn detect_rootless(spec: &Spec) -> Result<Option<Rootless>> {
-    let linux = spec.linux.as_ref().context("no linux in spec")?;
-    let namespaces = Namespaces::from(linux.namespaces.as_ref());
-    let user_namespace = namespaces.get(LinuxNamespaceType::User);
-    // If conditions requires us to use rootless, we must either create a new
-    // user namespace or enter an exsiting.
-    if should_use_rootless() && user_namespace.is_none() {
-        bail!("Rootless container requires valid user namespace definition");
-    }
-
-    // Go through rootless procedure only when entering into a new user namespace
-    let rootless = if user_namespace.is_some() && user_namespace.unwrap().path.is_none() {
-        log::debug!("rootless container should be created");
-        log::warn!(
-            "resource constraints and multi id mapping is unimplemented for rootless containers"
-        );
-        validate(spec).context("The spec failed to comply to rootless requirement")?;
-        let linux = spec.linux.as_ref().context("no linux in spec")?;
-        let mut rootless = Rootless::from(linux);
-        if let Some((uid_binary, gid_binary)) = lookup_map_binaries(linux)? {
-            rootless.newuidmap = Some(uid_binary);
-            rootless.newgidmap = Some(gid_binary);
-        }
-
-        Some(rootless)
-    } else {
-        log::debug!("This is NOT a rootless container");
-        None
-    };
-
-    Ok(rootless)
-}
-
 /// Checks if rootless mode should be used
-pub fn should_use_rootless() -> bool {
+pub fn rootless_required() -> bool {
    if !nix::unistd::geteuid().is_root() {
        return true;
    }
@ -114,6 +116,30 @@ fn validate(spec: &Spec) -> Result<()> {
        gid_mappings,
    )?;

+    if let Some(process) = &spec.process {
+        if let Some(additional_gids) = &process.user.additional_gids {
+            let privileged = nix::unistd::geteuid().is_root();
+
+            match (privileged, additional_gids.is_empty()) {
+                (true, false) => {
+                    for gid in additional_gids {
+                        if !is_id_mapped(*gid, gid_mappings) {
+                            bail!("gid {} is specified as supplementary group, but is not mapped in the user namespace", gid);
+                        }
+                    }
+                }
+                (false, false) => {
+                    bail!(
+                        "user is {} (unprivileged). Supplementary groups cannot be set in \
+                        a rootless container for this user due to CVE-2014-8989",
+                        nix::unistd::geteuid()
+                    )
+                }
+                _ => {}
+            }
+        }
+    }
+
    Ok(())
 }

@ -125,11 +151,11 @@ fn validate_mounts(
    for mount in mounts {
        if let Some(options) = &mount.options {
            for opt in options {
-                if opt.starts_with("uid=") && !is_id_mapped(&opt[4..], uid_mappings)? {
+                if opt.starts_with("uid=") && !is_id_mapped(opt[4..].parse()?, uid_mappings) {
                    bail!("Mount {:?} specifies option {} which is not mapped inside the rootless container", mount, opt);
                }

-                if opt.starts_with("gid=") && !is_id_mapped(&opt[4..], gid_mappings)? {
+                if opt.starts_with("gid=") && !is_id_mapped(opt[4..].parse()?, gid_mappings) {
                    bail!("Mount {:?} specifies option {} which is not mapped inside the rootless container", mount, opt);
                }
            }
@ -139,11 +165,10 @@ fn validate_mounts(
    Ok(())
 }

-fn is_id_mapped(id: &str, mappings: &[LinuxIdMapping]) -> Result<bool> {
-    let id = id.parse::<u32>()?;
-    Ok(mappings
+fn is_id_mapped(id: u32, mappings: &[LinuxIdMapping]) -> bool {
+    mappings
        .iter()
-        .any(|m| id >= m.container_id && id <= m.container_id + m.size))
+        .any(|m| id >= m.container_id && id <= m.container_id + m.size)
 }

 /// Looks up the location of the newuidmap and newgidmap binaries which
@ -177,7 +202,7 @@ fn lookup_map_binary(binary: &str) -> Result<Option<PathBuf>> {
 pub fn write_uid_mapping(target_pid: Pid, rootless: Option<&Rootless>) -> Result<()> {
    log::debug!("Write UID mapping for {:?}", target_pid);
    if let Some(rootless) = rootless {
-        if let Some(uid_mappings) = rootless.gid_mappings {
+        if let Some(uid_mappings) = rootless.uid_mappings {
            return write_id_mapping(
                &format!("/proc/{}/uid_map", target_pid),
                uid_mappings,
--- a/test_framework/Cargo.lock
+++ b/test_framework/Cargo.lock
@ -0,0 +1,14 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "anyhow"
+version = "1.0.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486"
+
+[[package]]
+name = "test_framework"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+]
--- a/test_framework/Cargo.toml
+++ b/test_framework/Cargo.toml
@ -0,0 +1,9 @@
+[package]
+name = "test_framework"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.42"
--- a/test_framework/Readme.md
+++ b/test_framework/Readme.md
@ -0,0 +1,51 @@
+# Test Framework for the Integration test
+
+This is a simple test framework which provides various structs to setup and run tests.
+
+## Docs
+
+This crate provides following things.
+
+#### TestResult
+
+A Simple enum, similar to Rust Result, but Ok has no associated value, and has Skip variant to indicate that a test is skipped.
+
+#### Trait Testable
+
+This trait indicates that something can be used as a test. This is the smallest individual unit of the framework.
+
+The implementor must implement three functions :
+
+- get_name : returns name of the test.
+- can_run : returns boolean indicating that if the particular test can be run or not. Defaults to returning true.
+- run : runs the actual test, and returns a TestResult.
+
+#### Trait TestableGroup
+
+This trait indicates that something is a group of multiple tests. Primarily used for grouping tests, as well as providing namespacing.
+
+The implementor must implement three functions :
+
+- get_name : returns name of the test group
+- run_all : run all of the tests belonging to this group, and return vector of tuples, each pair having name of test and its result.
+- run_selected : takes slice of test names which are to be run, and should run only those tests. Return vector of tuples, each pair having name of test and its result.
+
+#### Struct Test
+
+Provides a simple template for a simple test, implements Testable. This is intended to quickly create tests which are always run, and do not require state information. The new function takes name and Boxed function, which is the test function.
+
+#### Struct ConditionalTest
+
+Provides a simple template for test which is to be run conditionally. Implements Testable, and is intended to be used for stateless tests, which may or may not run depending on some condition (system config, env var etc.) The new function takes name, a Boxed function which is condition function, which returns a boolean indicating if the test can be run or not, and another Boxed function, which is the test function.
+
+#### Struct TestGroup
+
+Provides a simple template for a test group. This implement TestableGroup. The new function takes the name of the test group, and add function takes vector of Testables. This is intended to used for grouping of simple, stateless tests.
+
+#### Struct TestManager
+
+This is the core manager for running of the tests. This stores test groups, controls running of them, and printing of results. It has following functions :
+
+- add_test_group : adds a TestableGroup.
+- run_all : runs all the tests in all test groups which can be run (whose can_run returns true) and prints their results to stdout
+- run_selected : takes a vector of tuples of the form (group-name, optional vector of test names) . Then runs only selected tests. If the optional vector is not present (None) then runs all tests in the group, or else runs only the selected tests from the group.
--- a/test_framework/src/conditional_test.rs
+++ b/test_framework/src/conditional_test.rs
@ -0,0 +1,40 @@
+///! Contains definition for a tests which should be conditionally run
+use crate::testable::{TestResult, Testable};
+
+// type aliases for test function signature
+type TestFn = dyn Fn() -> TestResult;
+// type alias for function signature for function which checks if a test can be run or not
+type CheckFn = dyn Fn() -> bool;
+
+/// Basic Template structure for tests which need to be run conditionally
+pub struct ConditionalTest {
+    /// name of the test
+    name: String,
+    /// actual test function
+    test_fn: Box<TestFn>,
+    /// function to check if a test can be run or not
+    check_fn: Box<CheckFn>,
+}
+
+impl ConditionalTest {
+    /// Create a new condition test
+    pub fn new(name: &str, check_fn: Box<CheckFn>, test_fn: Box<TestFn>) -> Self {
+        ConditionalTest {
+            name: name.to_string(),
+            check_fn,
+            test_fn,
+        }
+    }
+}
+
+impl Testable for ConditionalTest {
+    fn get_name(&self) -> String {
+        self.name.clone()
+    }
+    fn can_run(&self) -> bool {
+        (self.check_fn)()
+    }
+    fn run(&self) -> TestResult {
+        (self.test_fn)()
+    }
+}
--- a/test_framework/src/lib.rs
+++ b/test_framework/src/lib.rs
@ -0,0 +1,10 @@
+mod conditional_test;
+mod test;
+mod test_group;
+mod test_manager;
+mod testable;
+pub use conditional_test::ConditionalTest;
+pub use test::Test;
+pub use test_group::TestGroup;
+pub use test_manager::TestManager;
+pub use testable::{TestResult, Testable, TestableGroup};
--- a/test_framework/src/test.rs
+++ b/test_framework/src/test.rs
@ -0,0 +1,32 @@
+///! Contains definition for a simple and commonly usable test structure
+use crate::testable::{TestResult, Testable};
+
+// type alias for the test function
+type TestFn = dyn Sync + Send + Fn() -> TestResult;
+
+/// Basic Template structure for a test
+pub struct Test {
+    /// name of the test
+    name: String,
+    /// Actual test function
+    test_fn: Box<TestFn>,
+}
+
+impl Test {
+    /// create new test
+    pub fn new(name: &str, test_fn: Box<TestFn>) -> Self {
+        Test {
+            name: name.to_string(),
+            test_fn,
+        }
+    }
+}
+
+impl Testable for Test {
+    fn get_name(&self) -> String {
+        self.name.clone()
+    }
+    fn run(&self) -> TestResult {
+        (self.test_fn)()
+    }
+}
--- a/test_framework/src/test_group.rs
+++ b/test_framework/src/test_group.rs
@ -0,0 +1,63 @@
+///! Contains structure for a test group
+use crate::testable::{TestResult, Testable, TestableGroup};
+use std::collections::BTreeMap;
+
+/// Stores tests belonging to a group
+pub struct TestGroup {
+    /// name of the test group
+    name: String,
+    /// tests belonging to this group
+    tests: BTreeMap<String, Box<dyn Testable + 'static + Sync + Send>>,
+}
+
+impl TestGroup {
+    /// create a new test group
+    pub fn new(name: &str) -> Self {
+        TestGroup {
+            name: name.to_string(),
+            tests: BTreeMap::new(),
+        }
+    }
+
+    /// add a test to the group
+    pub fn add(&mut self, tests: Vec<impl Testable + 'static + Sync + Send>) {
+        tests.into_iter().for_each(|t| {
+            self.tests.insert(t.get_name(), Box::new(t));
+        });
+    }
+}
+
+impl TestableGroup for TestGroup {
+    /// get name of the test group
+    fn get_name(&self) -> String {
+        self.name.clone()
+    }
+    /// run all the test from the test group
+    fn run_all(&self) -> Vec<(String, TestResult)> {
+        self.tests
+            .iter()
+            .map(|(_, t)| {
+                if t.can_run() {
+                    (t.get_name(), t.run())
+                } else {
+                    (t.get_name(), TestResult::Skip)
+                }
+            })
+            .collect()
+    }
+
+    /// run selected test from the group
+    fn run_selected(&self, selected: &[&str]) -> Vec<(String, TestResult)> {
+        self.tests
+            .iter()
+            .filter(|(name, _)| selected.contains(&name.as_str()))
+            .map(|(_, t)| {
+                if t.can_run() {
+                    (t.get_name(), t.run())
+                } else {
+                    (t.get_name(), TestResult::Skip)
+                }
+            })
+            .collect()
+    }
+}
--- a/test_framework/src/test_manager.rs
+++ b/test_framework/src/test_manager.rs
@ -0,0 +1,88 @@
+///! This exposes the main control wrapper to control the tests
+use crate::testable::{TestResult, TestableGroup};
+use std::collections::BTreeMap;
+
+/// This manages all test groups, and thus the tests
+pub struct TestManager<'a> {
+    test_groups: BTreeMap<String, &'a dyn TestableGroup>,
+}
+
+impl<'a> Default for TestManager<'a> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<'a> TestManager<'a> {
+    /// Create new TestManager
+    pub fn new() -> Self {
+        TestManager {
+            test_groups: BTreeMap::new(),
+        }
+    }
+
+    /// add a test group to the test manager
+    pub fn add_test_group(&mut self, tg: &'a dyn TestableGroup) {
+        self.test_groups.insert(tg.get_name(), tg);
+    }
+
+    /// Prints the given test results, usually used to print
+    /// results of a test group
+    fn print_test_result(&self, name: &str, res: Vec<(&String, &TestResult)>) {
+        println!("# Start group {}", name);
+        let len = res.len();
+        for (idx, (name, res)) in res.iter().enumerate() {
+            print!("{} / {} : {} : ", idx + 1, len, name);
+            match res {
+                TestResult::Ok => {
+                    println!("ok");
+                }
+                TestResult::Skip => {
+                    println!("skipped");
+                }
+                TestResult::Err(e) => {
+                    println!("not ok\n\t{}", e);
+                }
+            }
+        }
+        println!("\n# End group {}", name);
+    }
+
+    /// Run all tests from given group
+    fn run_test_group(&self, name: &str, tg: &'a dyn TestableGroup) {
+        let results = tg.run_all();
+        let mut test_vec = Vec::new();
+        for (name, res) in results.iter() {
+            test_vec.push((name, res));
+        }
+        self.print_test_result(name, test_vec);
+    }
+
+    /// Run all tests from all tests group
+    pub fn run_all(&self) {
+        for (name, tg) in self.test_groups.iter() {
+            self.run_test_group(name, *tg);
+        }
+    }
+
+    /// Run only selected tests
+    pub fn run_selected(&self, tests: Vec<(String, Option<Vec<&str>>)>) {
+        for (test_group_name, tests) in tests.iter() {
+            if let Some(tg) = self.test_groups.get(test_group_name) {
+                match tests {
+                    None => self.run_test_group(test_group_name, *tg),
+                    Some(tests) => {
+                        let results = tg.run_selected(tests);
+                        let mut test_vec = Vec::new();
+                        for (name, res) in results.iter() {
+                            test_vec.push((name, res));
+                        }
+                        self.print_test_result(test_group_name, test_vec);
+                    }
+                }
+            } else {
+                eprintln!("Error : Test Group {} not found, skipping", test_group_name);
+            }
+        }
+    }
+}
--- a/test_framework/src/testable.rs
+++ b/test_framework/src/testable.rs
@ -0,0 +1,42 @@
+///! Contains Basic setup for testing, testable trait and its result type
+use anyhow::{Error, Result};
+
+#[derive(Debug)]
+/// Enum indicating result of the test. This is like an extended std::result,
+/// which includes a Skip variant to indicate that a test was skipped, and the Ok variant has no associated value
+pub enum TestResult {
+    /// Test was ok
+    Ok,
+    /// Test needed to be skipped
+    Skip,
+    /// Test was error
+    Err(Error),
+}
+
+impl<T> From<Result<T>> for TestResult {
+    fn from(result: Result<T>) -> Self {
+        match result {
+            Ok(_) => TestResult::Ok,
+            Err(err) => TestResult::Err(err),
+        }
+    }
+}
+
+/// This trait indicates that something can be run as a test, or is 'testable'
+/// This forms the basis of the framework, as all places where tests are done,
+/// expect structs which implement this
+pub trait Testable {
+    fn get_name(&self) -> String;
+    fn can_run(&self) -> bool {
+        true
+    }
+    fn run(&self) -> TestResult;
+}
+
+/// This trait indicates that something forms a group of tests.
+/// Test groups are used to group tests in sensible manner as well as provide namespacing to tests
+pub trait TestableGroup {
+    fn get_name(&self) -> String;
+    fn run_all(&self) -> Vec<(String, TestResult)>;
+    fn run_selected(&self, selected: &[&str]) -> Vec<(String, TestResult)>;
+}
--- a/youki_integration_test/Cargo.lock
+++ b/youki_integration_test/Cargo.lock
@ -1,7 +1,5 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
-
 [[package]]
 name = "adler"
 version = "1.0.2"
@ -10,9 +8,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"

 [[package]]
 name = "anyhow"
-version = "1.0.42"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486"
+checksum = "28ae2b3dec75a406790005a200b1bd89785afc02517a00ca99ecfe093ee9e6cf"

 [[package]]
 name = "autocfg"
@ -22,9 +20,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"

 [[package]]
 name = "bitflags"
-version = "1.2.1"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

 [[package]]
 name = "cfg-if"
@ -32,6 +30,36 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

+[[package]]
+name = "clap"
+version = "3.0.0-beta.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142"
+dependencies = [
+ "bitflags",
+ "clap_derive",
+ "indexmap",
+ "lazy_static",
+ "os_str_bytes",
+ "strsim",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
+[[package]]
+name = "clap_derive"
+version = "3.0.0-beta.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1"
+dependencies = [
+ "heck",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.2.1"
@ -43,9 +71,9 @@ dependencies = [

 [[package]]
 name = "filetime"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8"
+checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98"
 dependencies = [
 "cfg-if",
 "libc",
@ -77,10 +105,41 @@ dependencies = [
 ]

 [[package]]
-name = "libc"
-version = "0.2.95"
+name = "hashbrown"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+
+[[package]]
+name = "heck"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+dependencies = [
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.101"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"

 [[package]]
 name = "miniz_oxide"
@ -92,6 +151,18 @@ dependencies = [
 "autocfg",
 ]

+[[package]]
+name = "once_cell"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
+
+[[package]]
+name = "os_str_bytes"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.10"
@ -99,10 +170,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"

 [[package]]
-name = "rand"
-version = "0.8.3"
+name = "proc-macro-error"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
 dependencies = [
 "libc",
 "rand_chacha",
@ -122,9 +235,9 @@ dependencies = [

 [[package]]
 name = "rand_core"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
 dependencies = [
 "getrandom",
 ]
@ -140,18 +253,35 @@ dependencies = [

 [[package]]
 name = "redox_syscall"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
+checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
 dependencies = [
 "bitflags",
 ]

 [[package]]
-name = "tar"
-version = "0.4.35"
+name = "strsim"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d779dc6aeff029314570f666ec83f19df7280bb36ef338442cfa8c604021b80"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
+[[package]]
+name = "syn"
+version = "1.0.75"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f58f7e8eaa0009c5fec437aabf511bd9933e4b2d7407bd05273c01a8906ea7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "tar"
+version = "0.4.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f5515d3add52e0bbdcad7b83c388bb36ba7b754dda3b5f5bc2d38640cdba5c"
 dependencies = [
 "filetime",
 "libc",
@ -159,10 +289,38 @@ dependencies = [
 ]

 [[package]]
-name = "testanything"
-version = "0.2.1"
+name = "test_framework"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d11c9f0b80f86baebe13b3555c1dc5e1824deae0f710b4df8277b80b707a0a84"
+checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"

 [[package]]
 name = "uuid"
@ -170,6 +328,18 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"

+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
+[[package]]
+name = "version_check"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
+
 [[package]]
 name = "wasi"
 version = "0.10.2+wasi-snapshot-preview1"
@ -212,9 +382,13 @@ name = "youki_integration_test"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "clap",
+ "clap_derive",
 "flate2",
+ "lazy_static",
+ "once_cell",
 "rand",
 "tar",
- "testanything",
+ "test_framework",
 "uuid",
 ]
--- a/youki_integration_test/Cargo.toml
+++ b/youki_integration_test/Cargo.toml
@ -3,10 +3,21 @@ name = "youki_integration_test"
 version = "0.1.0"
 edition = "2018"

+[dependencies.clap]
+version = "=3.0.0-beta.2"
+default-features = false
+features = ["std", "suggestions", "derive"]
+
+[dependencies.clap_derive]
+version = "=3.0.0-beta.2"
+default-features = true
+
 [dependencies]
 uuid = "0.8"
 rand = "0.8.0"
 tar = "0.4"
 flate2 = "1.0"
-testanything = "0.2.1"
+test_framework = { version = "0.1.0", path = "../test_framework"}
 anyhow = "1.0"
+lazy_static = "1.4.0"
+once_cell = "1.8.0"
--- a/youki_integration_test/README.md
+++ b/youki_integration_test/README.md
@ -1,10 +1,33 @@
 # Integration test

-## Usage
-Here is a preview implementation of the integration test.
+This provides a test suite to test low level OCI spec compliant container runtime

-```
+## Usage
+
+```sh
+# in root folder
+$ ./build.sh
+$ cd youki_integration_test
 $ cp ../youki .
 $ ./build.sh
-$ sudo ./youki_integration_test
+# currently root access is required
+$ sudo ./youki_integration_test -r ./youki
 ```
+
+This provides following commandline options :
+
+- --runtime (-r) : Required. Takes path of runtime executable to be tested. If the path is not valid, the program exits.
+- --tests (-t) : Optional. Takes list of tests to be run, and runs only those tests. Format for it is : `test-grp-1::test-1,test-2 <space> test-grp-2 <space> test-grp-3::test-3 ...`. The test groups with no specific tests specified, (test-grp-2 in the example) , will run all of its tests, and in other cases, only selected tests will be run. Test groups not mentioned will be ignored.
+
+Currently, there are following test groups and tests :
+
+- lifecycle
+  - create
+  - start
+  - kill
+  - state
+  - delete
+- create
+  - empty_id
+  - valid_id
+  - duplicate_id
--- a/youki_integration_test/src/command/create.rs
+++ b/youki_integration_test/src/command/create.rs
@ -1,19 +0,0 @@
-use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
-
-// TODO Allow to receive arguments.
-// TODO Wrapping up the results
-pub fn exec(project_path: &Path, id: &str) -> bool {
-    let status = Command::new(project_path.join(PathBuf::from("youki")))
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
-        .arg("-r")
-        .arg(project_path.join("integration-workspace").join("youki"))
-        .arg("create")
-        .arg(id)
-        .arg("--bundle")
-        .arg(project_path.join("integration-workspace").join("bundle"))
-        .status()
-        .expect("failed to execute process");
-    status.success()
-}
--- a/youki_integration_test/src/command/delete.rs
+++ b/youki_integration_test/src/command/delete.rs
@ -1,17 +0,0 @@
-use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
-
-// TODO Allow to receive arguments.
-// TODO Wrapping up the results
-pub fn exec(project_path: &Path, id: &str) -> bool {
-    let status = Command::new(project_path.join(PathBuf::from("youki")))
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
-        .arg("-r")
-        .arg(project_path.join("integration-workspace").join("youki"))
-        .arg("delete")
-        .arg(id)
-        .status()
-        .expect("failed to execute process");
-    status.success()
-}
--- a/youki_integration_test/src/command/kill.rs
+++ b/youki_integration_test/src/command/kill.rs
@ -1,18 +0,0 @@
-use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
-
-// TODO Allow to receive arguments.
-// TODO Wrapping up the results
-pub fn exec(project_path: &Path, id: &str) -> bool {
-    let status = Command::new(project_path.join(PathBuf::from("youki")))
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
-        .arg("-r")
-        .arg(project_path.join("integration-workspace").join("youki"))
-        .arg("kill")
-        .arg(id)
-        .arg("9")
-        .status()
-        .expect("failed to execute process");
-    status.success()
-}
--- a/youki_integration_test/src/command/mod.rs
+++ b/youki_integration_test/src/command/mod.rs
@ -1,6 +0,0 @@
-pub mod create;
-pub mod delete;
-pub mod kill;
-pub mod start;
-pub mod state;
-pub mod youki;
--- a/youki_integration_test/src/command/start.rs
+++ b/youki_integration_test/src/command/start.rs
@ -1,17 +0,0 @@
-use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
-
-// TODO Allow to receive arguments.
-// TODO Wrapping up the results
-pub fn exec(project_path: &Path, id: &str) -> bool {
-    let status = Command::new(project_path.join(PathBuf::from("youki")))
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
-        .arg("-r")
-        .arg(project_path.join("integration-workspace").join("youki"))
-        .arg("start")
-        .arg(id)
-        .status()
-        .expect("failed to execute process");
-    status.success()
-}
--- a/youki_integration_test/src/command/state.rs
+++ b/youki_integration_test/src/command/state.rs
@ -1,17 +0,0 @@
-use std::path::{Path, PathBuf};
-use std::process::{Command, Stdio};
-
-// TODO Allow to receive arguments.
-// TODO Wrapping up the results
-pub fn exec(project_path: &Path, id: &str) -> bool {
-    let status = Command::new(project_path.join(PathBuf::from("youki")))
-        .stdout(Stdio::null())
-        .stderr(Stdio::null())
-        .arg("-r")
-        .arg(project_path.join("integration-workspace").join("youki"))
-        .arg("state")
-        .arg(id)
-        .status()
-        .expect("failed to execute process");
-    status.success()
-}
--- a/youki_integration_test/src/command/youki.rs
+++ b/youki_integration_test/src/command/youki.rs
@ -1,45 +0,0 @@
-use std::path::{Path, PathBuf};
-
-use crate::support::generate_uuid;
-
-use super::{create, delete, kill, start, state};
-
-pub struct Container {
-    project_path: PathBuf,
-    container_id: String,
-}
-
-impl Container {
-    pub fn new(project_path: &Path) -> Self {
-        Container {
-            project_path: project_path.to_owned(),
-            container_id: generate_uuid().to_string(),
-        }
-    }
-    pub fn with_container_id(project_path: &Path, container_id: &str) -> Self {
-        Container {
-            project_path: project_path.to_owned(),
-            container_id: container_id.to_string(),
-        }
-    }
-
-    pub fn create(&self) -> bool {
-        create::exec(&self.project_path, &self.container_id)
-    }
-
-    pub fn start(&self) -> bool {
-        start::exec(&self.project_path, &self.container_id)
-    }
-
-    pub fn state(&self) -> bool {
-        state::exec(&self.project_path, &self.container_id)
-    }
-
-    pub fn kill(&self) -> bool {
-        kill::exec(&self.project_path, &self.container_id)
-    }
-
-    pub fn delete(&self) -> bool {
-        delete::exec(&self.project_path, &self.container_id)
-    }
-}
--- a/youki_integration_test/src/lib.rs
+++ b/youki_integration_test/src/lib.rs
@ -1,2 +1,2 @@
-pub mod command;
 pub mod support;
+pub mod tests;
--- a/youki_integration_test/src/main.rs
+++ b/youki_integration_test/src/main.rs
@ -1,115 +1,73 @@
-use anyhow::{bail, Result};
-use std::path::Path;
-
-mod command;
 mod support;
+mod tests;
+
+use anyhow::{bail, Result};
+use clap::Clap;
+use std::path::PathBuf;
+use test_framework::TestManager;

 use crate::support::cleanup_test;
-use crate::support::create_project_path;
-use crate::support::generate_uuid;
+use crate::support::get_project_path;
 use crate::support::initialize_test;
-use crate::support::print_test_results;
-use crate::support::test_builder;
+use crate::support::set_runtime_path;
+use crate::tests::lifecycle::{ContainerCreate, ContainerLifecycle};

-use crate::command::youki::Container;
+#[derive(Clap, Debug)]
+#[clap(version = "0.0.1", author = "youki team")]
+struct Opts {
+    /// path for the container runtime to be tested
+    #[clap(short, long)]
+    runtime: PathBuf,
+    /// selected tests to be run, format should be
+    /// space separated groups, eg
+    /// -t group1::test1,test3 group2 group3::test5
+    #[clap(short, long, multiple = true, value_delimiter = " ")]
+    tests: Option<Vec<String>>,
+}
+
+// parse test string given in commandline option as pair of testgroup name and tests belonging to that
+fn parse_tests(tests: &[String]) -> Vec<(String, Option<Vec<&str>>)> {
+    let mut ret = Vec::with_capacity(tests.len());
+    for test in tests {
+        if test.contains("::") {
+            let (mod_name, test_names) = test.split_once("::").unwrap();
+            let _tests = test_names.split(',').collect();
+            ret.push((mod_name.to_owned(), Some(_tests)));
+        } else {
+            ret.push((test.to_owned(), None));
+        }
+    }
+    ret
+}

 fn main() -> Result<()> {
-    let project_path = create_project_path();
-    if initialize_test(&project_path).is_err() {
-        bail!("Can not initilize test.")
-    }
-    life_cycle_test(&project_path);
-    if cleanup_test(&project_path).is_err() {
-        bail!("Can not cleanup test.")
-    }
+    let opts: Opts = Opts::parse();
+
+    let path = std::fs::canonicalize(opts.runtime).expect("Invalid runtime path");
+    set_runtime_path(&path);
+
+    let mut tm = TestManager::new();
+    let project_path = get_project_path();
+
+    let cl = ContainerLifecycle::new(&project_path);
+    let cc = ContainerCreate::new(&project_path);
+
+    tm.add_test_group(&cl);
+    tm.add_test_group(&cc);

    if initialize_test(&project_path).is_err() {
        bail!("Can not initilize test.")
    }
-    container_create_test(&project_path);
+
+    if let Some(tests) = opts.tests {
+        let tests_to_run = parse_tests(&tests);
+        tm.run_selected(tests_to_run);
+    } else {
+        tm.run_all();
+    }
+
    if cleanup_test(&project_path).is_err() {
        bail!("Can not cleanup test.")
    }
    Ok(())
 }
-
-// This tests the entire lifecycle of the container.
-fn life_cycle_test(project_path: &Path) {
-    let container_runtime = Container::new(project_path);
-
-    let create_test = test_builder(
-        container_runtime.create(),
-        "Create a new container test",
-        "This operation must create a new container.",
-    );
-    let state_test = test_builder(
-        container_runtime.state(),
-        "Execute state test",
-        "This operation must state the container.",
-    );
-    let start_test = test_builder(
-        container_runtime.start(),
-        "Execute start test",
-        "This operation must start the container.",
-    );
-    let state_again_test = test_builder(
-        container_runtime.state(),
-        "Execute state test",
-        "This operation must state the container.",
-    );
-    let kill_test = test_builder(
-        container_runtime.kill(),
-        "Execute kill test",
-        "This operation must kill the container.",
-    );
-    let delete_test = test_builder(
-        container_runtime.delete(),
-        "Execute delete test",
-        "This operation must delete the container.",
-    );
-
-    // print to stdout
-    print_test_results(
-        "Create comand test suite",
-        vec![
-            create_test,
-            state_test,
-            start_test,
-            state_again_test,
-            kill_test,
-            delete_test,
-        ],
-    );
-}
-
-// This is a test of the create command.
-// It follows the `opencontainers/runtime-tools` test case.
-fn container_create_test(project_path: &Path) {
-    let container_runtime_with_empty_id = Container::with_container_id(project_path, "");
-    let empty_id_test = test_builder(
-        !container_runtime_with_empty_id.create(),
-        "create with no ID test",
-        "This operation MUST generate an error if it is not provided a path to the bundle and the container ID to associate with the container.",
-    );
-
-    let uuid = generate_uuid();
-    let container_runtime_with_id = Container::with_container_id(project_path, &uuid.to_string());
-    let with_id_test = test_builder(
-        container_runtime_with_id.create(),
-        "create with ID test",
-        "This operation MUST create a new container.",
-    );
-
-    let container_id_with_exist_id = Container::with_container_id(project_path, &uuid.to_string());
-    let exist_id_test = test_builder(
-        !container_id_with_exist_id.create(),
-        "create with an already existing ID test",
-        "If the ID provided is not unique across all containers within the scope of the runtime, or is not valid in any other way, the implementation MUST generate an error and a new container MUST NOT be created.",
-    );
-
-    // print to stdout
-    print_test_results(
-        "Create comand test suite",
-        vec![empty_id_test, with_id_test, exist_id_test],
-    );
-}
--- a/youki_integration_test/src/support.rs
+++ b/youki_integration_test/src/support.rs
@ -1,14 +1,22 @@
 use flate2::read::GzDecoder;
+use once_cell::sync::OnceCell;
 use rand::Rng;
 use std::fs::File;
-use std::io;
 use std::path::PathBuf;
 use std::{env, fs, path::Path};
 use tar::Archive;
-use testanything::tap_suite_builder::TapSuiteBuilder;
-use testanything::{tap_test::TapTest, tap_test_builder::TapTestBuilder};
 use uuid::Uuid;

+static RUNTIME_PATH: OnceCell<PathBuf> = OnceCell::new();
+
+pub fn set_runtime_path(path: &Path) {
+    RUNTIME_PATH.set(path.to_owned()).unwrap();
+}
+
+pub fn get_runtime_path() -> &'static PathBuf {
+    RUNTIME_PATH.get().expect("Runtime path is not set")
+}
+
 pub fn initialize_test(project_path: &Path) -> Result<(), std::io::Error> {
    prepare_test_workspace(project_path)
 }
@ -17,7 +25,7 @@ pub fn cleanup_test(project_path: &Path) -> Result<(), std::io::Error> {
    delete_test_workspace(project_path)
 }

-pub fn create_project_path() -> PathBuf {
+pub fn get_project_path() -> PathBuf {
    let current_dir_path_result = env::current_dir();
    match current_dir_path_result {
        Ok(path_buf) => path_buf,
@ -43,29 +51,6 @@ pub fn generate_uuid() -> Uuid {
    }
 }

-pub fn test_builder(status: bool, name: &str, diagnostic: &str) -> TapTest {
-    TapTestBuilder::new()
-        .name(name)
-        .passed(status)
-        .diagnostics(&[diagnostic])
-        .finalize()
-}
-
-pub fn print_test_results(test_name: &str, tests: Vec<TapTest>) {
-    let tap_suite = TapSuiteBuilder::new()
-        .name(test_name)
-        .tests(tests)
-        .finalize();
-
-    // print to stdout
-    println!("# Start {}", test_name);
-    match tap_suite.print(io::stdout().lock()) {
-        Ok(_) => {}
-        Err(reason) => eprintln!("{}", reason),
-    }
-    println!("\n# End {}", test_name);
-}
-
 // Temporary files to be used for testing are created in the `integration-workspace`.
 fn prepare_test_workspace(project_path: &Path) -> Result<(), std::io::Error> {
    let integration_test_workspace_path = project_path.join("integration-workspace");
--- a/youki_integration_test/src/tests/lifecycle/container_create.rs
+++ b/youki_integration_test/src/tests/lifecycle/container_create.rs
@ -0,0 +1,79 @@
+use super::{create, kill};
+use crate::support::generate_uuid;
+use std::path::{Path, PathBuf};
+use test_framework::{TestResult, TestableGroup};
+
+pub struct ContainerCreate {
+    project_path: PathBuf,
+    container_id: String,
+}
+
+impl ContainerCreate {
+    pub fn new(project_path: &Path) -> Self {
+        ContainerCreate {
+            project_path: project_path.to_owned(),
+            container_id: generate_uuid().to_string(),
+        }
+    }
+
+    // runtime should not create container with empty id
+    fn create_empty_id(&self) -> TestResult {
+        let temp = create::create(&self.project_path, "");
+        match temp {
+            TestResult::Ok => TestResult::Err(anyhow::anyhow!(
+                "Container should not have been created with empty id, but was created."
+            )),
+            TestResult::Err(_) => TestResult::Ok,
+            TestResult::Skip => TestResult::Skip,
+        }
+    }
+
+    // runtime should create container with valid id
+    fn create_valid_id(&self) -> TestResult {
+        let temp = create::create(&self.project_path, &self.container_id);
+        if let TestResult::Ok = temp {
+            kill::kill(&self.project_path, &self.container_id);
+        }
+        temp
+    }
+
+    // runtime should not create container with is that already exists
+    fn create_duplicate_id(&self) -> TestResult {
+        let id = generate_uuid().to_string();
+        let _ = create::create(&self.project_path, &id);
+        let temp = create::create(&self.project_path, &id);
+        kill::kill(&self.project_path, &id);
+        match temp {
+            TestResult::Ok => TestResult::Err(anyhow::anyhow!(
+                "Container should not have been created with same id, but was created."
+            )),
+            TestResult::Err(_) => TestResult::Ok,
+            TestResult::Skip => TestResult::Skip,
+        }
+    }
+}
+
+impl TestableGroup for ContainerCreate {
+    fn get_name(&self) -> String {
+        "create".to_owned()
+    }
+    fn run_all(&self) -> Vec<(String, TestResult)> {
+        vec![
+            ("empty_id".to_owned(), self.create_empty_id()),
+            ("valid_id".to_owned(), self.create_valid_id()),
+            ("duplicate_id".to_owned(), self.create_duplicate_id()),
+        ]
+    }
+    fn run_selected(&self, selected: &[&str]) -> Vec<(String, TestResult)> {
+        let mut ret = Vec::new();
+        for name in selected {
+            match *name {
+                "empty_id" => ret.push(("empty_id".to_owned(), self.create_empty_id())),
+                "valid_id" => ret.push(("valid_id".to_owned(), self.create_valid_id())),
+                "duplicate_id" => ret.push(("duplicate_id".to_owned(), self.create_duplicate_id())),
+                _ => eprintln!("No test named {} in lifecycle", name),
+            };
+        }
+        ret
+    }
+}
--- a/youki_integration_test/src/tests/lifecycle/container_lifecycle.rs
+++ b/youki_integration_test/src/tests/lifecycle/container_lifecycle.rs
@ -0,0 +1,69 @@
+use std::path::{Path, PathBuf};
+
+use crate::support::generate_uuid;
+use test_framework::{TestResult, TestableGroup};
+
+use super::{create, delete, kill, start, state};
+
+pub struct ContainerLifecycle {
+    project_path: PathBuf,
+    container_id: String,
+}
+
+impl ContainerLifecycle {
+    pub fn new(project_path: &Path) -> Self {
+        ContainerLifecycle {
+            project_path: project_path.to_owned(),
+            container_id: generate_uuid().to_string(),
+        }
+    }
+
+    pub fn create(&self) -> TestResult {
+        create::create(&self.project_path, &self.container_id)
+    }
+
+    pub fn start(&self) -> TestResult {
+        start::start(&self.project_path, &self.container_id)
+    }
+
+    pub fn state(&self) -> TestResult {
+        state::state(&self.project_path, &self.container_id)
+    }
+
+    pub fn kill(&self) -> TestResult {
+        kill::kill(&self.project_path, &self.container_id)
+    }
+
+    pub fn delete(&self) -> TestResult {
+        delete::delete(&self.project_path, &self.container_id)
+    }
+}
+
+impl TestableGroup for ContainerLifecycle {
+    fn get_name(&self) -> String {
+        "lifecycle".to_owned()
+    }
+    fn run_all(&self) -> Vec<(String, TestResult)> {
+        vec![
+            ("create".to_owned(), self.create()),
+            ("start".to_owned(), self.start()),
+            ("kill".to_owned(), self.kill()),
+            ("state".to_owned(), self.state()),
+            ("delete".to_owned(), self.delete()),
+        ]
+    }
+    fn run_selected(&self, selected: &[&str]) -> Vec<(String, TestResult)> {
+        let mut ret = Vec::new();
+        for name in selected {
+            match *name {
+                "create" => ret.push(("create".to_owned(), self.create())),
+                "start" => ret.push(("start".to_owned(), self.start())),
+                "kill" => ret.push(("kill".to_owned(), self.kill())),
+                "state" => ret.push(("state".to_owned(), self.state())),
+                "delete" => ret.push(("delete".to_owned(), self.delete())),
+                _ => eprintln!("No test named {} in lifecycle", name),
+            };
+        }
+        ret
+    }
+}
--- a/youki_integration_test/src/tests/lifecycle/create.rs
+++ b/youki_integration_test/src/tests/lifecycle/create.rs
@ -0,0 +1,37 @@
+use crate::support::get_runtime_path;
+use std::io;
+use std::path::Path;
+use std::process::{Command, Stdio};
+use test_framework::TestResult;
+
+// There are still some issues here
+// in case we put stdout and stderr as piped
+// the youki process created halts indefinitely
+// which is why we pass null, and use wait instead of wait_with_output
+pub fn create(project_path: &Path, id: &str) -> TestResult {
+    let res = Command::new(get_runtime_path())
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .arg("--root")
+        .arg(project_path.join("integration-workspace").join("youki"))
+        .arg("create")
+        .arg(id)
+        .arg("--bundle")
+        .arg(project_path.join("integration-workspace").join("bundle"))
+        .spawn()
+        .expect("Cannot execute create command")
+        .wait();
+    match res {
+        io::Result::Ok(status) => {
+            if status.success() {
+                TestResult::Ok
+            } else {
+                TestResult::Err(anyhow::anyhow!(
+                    "Error : create exited with nonzero status : {}",
+                    status
+                ))
+            }
+        }
+        io::Result::Err(e) => TestResult::Err(anyhow::Error::new(e)),
+    }
+}
--- a/youki_integration_test/src/tests/lifecycle/delete.rs
+++ b/youki_integration_test/src/tests/lifecycle/delete.rs
@ -0,0 +1,19 @@
+use super::get_result_from_output;
+use crate::support::get_runtime_path;
+use std::path::Path;
+use std::process::{Command, Stdio};
+use test_framework::TestResult;
+
+pub fn delete(project_path: &Path, id: &str) -> TestResult {
+    let res = Command::new(get_runtime_path())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .arg("--root")
+        .arg(project_path.join("integration-workspace").join("youki"))
+        .arg("delete")
+        .arg(id)
+        .spawn()
+        .expect("failed to execute delete command")
+        .wait_with_output();
+    get_result_from_output(res)
+}
--- a/youki_integration_test/src/tests/lifecycle/kill.rs
+++ b/youki_integration_test/src/tests/lifecycle/kill.rs
@ -0,0 +1,30 @@
+use super::get_result_from_output;
+use crate::support::get_runtime_path;
+use std::path::Path;
+use std::process::{Command, Stdio};
+use std::thread::sleep;
+use std::time::Duration;
+use test_framework::TestResult;
+
+// By experimenting, somewhere around 50 is enough for youki process
+// to get the kill signal and shut down
+// here we add a little buffer time as well
+const SLEEP_TIME: u64 = 75;
+
+pub fn kill(project_path: &Path, id: &str) -> TestResult {
+    let res = Command::new(get_runtime_path())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .arg("--root")
+        .arg(project_path.join("integration-workspace").join("youki"))
+        .arg("kill")
+        .arg(id)
+        .arg("9")
+        .spawn()
+        .expect("failed to execute kill command")
+        .wait_with_output();
+    // sleep a little, so the youki process actually gets the signal and shuts down
+    // otherwise, the tester moves on to next tests before the youki has gotten signal, and delete test can fail
+    sleep(Duration::from_millis(SLEEP_TIME));
+    get_result_from_output(res)
+}
--- a/youki_integration_test/src/tests/lifecycle/mod.rs
+++ b/youki_integration_test/src/tests/lifecycle/mod.rs
@ -0,0 +1,11 @@
+mod container_create;
+mod container_lifecycle;
+mod create;
+mod delete;
+mod kill;
+mod start;
+mod state;
+mod util;
+pub use container_create::ContainerCreate;
+pub use container_lifecycle::ContainerLifecycle;
+pub use util::get_result_from_output;
--- a/youki_integration_test/src/tests/lifecycle/start.rs
+++ b/youki_integration_test/src/tests/lifecycle/start.rs
@ -0,0 +1,19 @@
+use super::get_result_from_output;
+use crate::support::get_runtime_path;
+use std::path::Path;
+use std::process::{Command, Stdio};
+use test_framework::TestResult;
+
+pub fn start(project_path: &Path, id: &str) -> TestResult {
+    let res = Command::new(get_runtime_path())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .arg("--root")
+        .arg(project_path.join("integration-workspace").join("youki"))
+        .arg("start")
+        .arg(id)
+        .spawn()
+        .expect("failed to execute start command")
+        .wait_with_output();
+    get_result_from_output(res)
+}
--- a/youki_integration_test/src/tests/lifecycle/state.rs
+++ b/youki_integration_test/src/tests/lifecycle/state.rs
@ -0,0 +1,41 @@
+use crate::support::get_runtime_path;
+use std::io;
+use std::path::Path;
+use std::process::{Command, Stdio};
+use test_framework::TestResult;
+
+pub fn state(project_path: &Path, id: &str) -> TestResult {
+    let res = Command::new(get_runtime_path())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .arg("--root")
+        .arg(project_path.join("integration-workspace").join("youki"))
+        .arg("state")
+        .arg(id)
+        .spawn()
+        .expect("failed to execute state command")
+        .wait_with_output();
+    match res {
+        io::Result::Ok(output) => {
+            let stderr = String::from_utf8(output.stderr).unwrap();
+            let stdout = String::from_utf8(output.stdout).unwrap();
+            if stderr.contains("Error") || stderr.contains("error") {
+                TestResult::Err(anyhow::anyhow!(
+                    "Error :\nstdout : {}\nstderr : {}",
+                    stdout,
+                    stderr
+                ))
+            } else {
+                // confirm that the status is stopped, as this is executed after the kill command
+                if !(stdout.contains(&format!(r#""id": "{}""#, id))
+                    && stdout.contains(r#""status": "stopped""#))
+                {
+                    TestResult::Err(anyhow::anyhow!("Expected state stopped, got : {}", stdout))
+                } else {
+                    TestResult::Ok
+                }
+            }
+        }
+        io::Result::Err(e) => TestResult::Err(anyhow::Error::new(e)),
+    }
+}
--- a/youki_integration_test/src/tests/lifecycle/util.rs
+++ b/youki_integration_test/src/tests/lifecycle/util.rs
@ -0,0 +1,21 @@
+use std::{io, process};
+use test_framework::TestResult;
+
+pub fn get_result_from_output(res: io::Result<process::Output>) -> TestResult {
+    match res {
+        io::Result::Ok(output) => {
+            let stderr = String::from_utf8(output.stderr).unwrap();
+            if stderr.contains("Error") || stderr.contains("error") {
+                let stdout = String::from_utf8(output.stdout).unwrap();
+                TestResult::Err(anyhow::anyhow!(
+                    "Error :\nstdout : {}\nstderr : {}",
+                    stdout,
+                    stderr
+                ))
+            } else {
+                TestResult::Ok
+            }
+        }
+        io::Result::Err(e) => TestResult::Err(anyhow::Error::new(e)),
+    }
+}
--- a/youki_integration_test/src/tests/mod.rs
+++ b/youki_integration_test/src/tests/mod.rs
@ -0,0 +1 @@
+pub mod lifecycle;