vmm security capabilities

[0xc1c4da]

I noticed under the host Nix configuration running the hardened.nix profile, microvm.nix services were failing because vmms would not run without explicitly setting CAP_SYS_ADMIN on the executable.

According to their documentation, crosvm & cloud-hypervisor also suggest CAP_NET_ADMIN, similar to the qemu-bridge-helper wrapper .

Should this be done with mkSetcapProgram ?

[astro]

I actually prefer doing network setup in privileged systemd-networkd, letting the unprivileged MicroVMs just open /dev/net/tun which doesn't require CAP_SYS_ADMIN. Would that work for you, too?

[astro]

If you really want this, yes, it has to be through a setcap wrapper as setcap does not work on /nix/store.

In the spirit of hardening, I still advise against gifting your Hypervisor with additional capabilities because it is the attack surface for MicroVMs. Network configuration with CAP_SYS_ADMIN is better done away from user-supplied data, eg. in networkd.

I guess the culprit is your network interface configuration?

[0xc1c4da]

I am not doing any networking at this stage, I don't believe CAP_SYS_ADMIN is required for networking.

Under the hardened profile, it appears to be a requirement for running unprivileged crosvm when trying to start a vm? cloud-hypervisor doesn't appear to have this requirement

From a quick search I could only see a reference here: https://crosvm.dev/doc/fuse/mount/fn.mount.html

[astro]

Thanks for the research! That may explain why I was having trouble with 9p shares on crosvm...

[astro]

With this we still don't get crosvm to boot:

diff --git flake.nix flake.nix
index 497d6f6..4005e60 100644
--- flake.nix
+++ flake.nix
@@ -103,8 +103,7 @@
           let
             hypervisorsWith9p = [
               "qemu"
-              # currently broken:
-              # "crosvm"
+              "crosvm"
             ];
             hypervisorsWithUserNet = [ "qemu" "kvmtool" ];
             makeExample = { system, hypervisor, config ? {} }:
diff --git lib/runners/crosvm.nix lib/runners/crosvm.nix
index d6375e3..be2e7bd 100644
--- lib/runners/crosvm.nix
+++ lib/runners/crosvm.nix
@@ -9,6 +9,11 @@ let
   inherit (microvmConfig) vcpu mem balloonMem user interfaces volumes shares socket devices;
   inherit (microvmConfig.crosvm) pivotRoot extraArgs;
   mktuntap = pkgs.callPackage ../../pkgs/mktuntap.nix {};
+  crosvmBinary =
+    if builtins.any ({ proto, ... }: proto == "9p") shares
+    # crosvm with CAP_SYS_ADMIN setup by the microvm.host module
+    then "/run/wrappers/bin/crosvm"
+    else "${pkgs.crosvm}/bin/crosvm";
   interfaceFdOffset = 3;
   kernelPath = {
     x86_64-linux = "${kernel.dev}/vmlinux";
@@ -35,7 +40,7 @@ in {
       ])) microvmConfig.interfaces)
       ++
       [
-        "${pkgs.crosvm}/bin/crosvm" "run"
+        crosvmBinary "run"
         "-m" (toString (mem + balloonMem))
         "-c" (toString vcpu)
         "-r" bootDisk
diff --git nixos-modules/host.nix nixos-modules/host.nix
index 1115bdb..a5457bd 100644
--- nixos-modules/host.nix
+++ nixos-modules/host.nix
@@ -299,6 +299,14 @@ in
       capabilities = "cap_net_admin+ep";
     };

+    # crosvm mounts in its own namespace, requiring CAP_SYS_ADMIN
+    security.wrappers.crosvm = {
+      source = "${pkgs.crosvm}/bin/crosvm";
+      owner = "root";
+      group = "root";
+      capabilities = "cap_sys_admin+ep";
+    };
+
     # You must define this file with your bridge interfaces if you
     # intend to use qemu-bridge-helper through a `type = "bridge"`
     # interface.