Implemented rewrite to real path in dump_path

This fixes #254 by adding a function, get_real_store_dir, to libnixstore
to support non-root nix stores, where the physical and logical paths
differ.

prettier code

Applied formatter and clippy recommendation

Fixed string comparison

Added test cases, fixed web serve paths

flake.nix

@@ -43,6 +43,7 @@
               t00-simple = import ./tests/t00-simple.nix testArgs;
               t01-signing = import ./tests/t01-signing.nix testArgs;
               t02-varnish = import ./tests/t02-varnish.nix testArgs;
+              t03-chroot = import ./tests/t03-chroot.nix testArgs;
             } // {
             clippy = config.packages.harmonia.override ({
               enableClippy = true;

harmonia/src/config.rs

@@ -1,5 +1,6 @@
 use std::fs::read_to_string;
 
+use crate::store::Store;
 use anyhow::{Context, Result};
 use base64::{engine::general_purpose, Engine};
 use serde::Deserialize;
@@ -33,8 +34,11 @@ pub(crate) struct Config {
     pub(crate) priority: usize,
     #[serde(default)]
     pub(crate) sign_key_path: Option<String>,
-    #[serde(default)]
+
+    #[serde(skip)]
     pub(crate) secret_key: Option<String>,
+    #[serde(skip)]
+    pub(crate) store: Store,
 }
 
 fn get_secret_key(sign_key_path: Option<&str>) -> Result<Option<String>> {
@@ -65,5 +69,6 @@ pub(crate) fn load() -> Result<Config> {
     )
     .with_context(|| format!("Couldn't parse config file '{settings_file}'"))?;
     settings.secret_key = get_secret_key(settings.sign_key_path.as_deref())?;
+    settings.store = Store::new();
     Ok(settings)
 }

harmonia/src/main.rs

@@ -11,6 +11,7 @@ mod narinfo;
 mod narlist;
 mod root;
 mod serve;
+mod store;
 mod version;
 
 fn nixhash(hash: &str) -> Option<String> {

harmonia/src/nar.rs

@@ -15,6 +15,7 @@ use sync::mpsc::Sender;
 use tokio::fs::File;
 use tokio::io::AsyncReadExt;
 
+use crate::config::Config;
 use crate::{cache_control_max_age_1y, some_or_404};
 use std::ffi::{OsStr, OsString};
 use tokio::{sync, task};
@@ -269,9 +270,9 @@ async fn dump_symlink(frame: &Frame, tx: &Sender<Result<Bytes, ThreadSafeError>>
     Ok(())
 }
 
-async fn dump_path(path: &Path, tx: &Sender<Result<Bytes, ThreadSafeError>>) -> Result<()> {
+async fn dump_path(path: PathBuf, tx: &Sender<Result<Bytes, ThreadSafeError>>) -> Result<()> {
     write_byte_slices(tx, &[b"nix-archive-1"]).await?;
-    let mut stack = vec![Frame::new(path.to_owned()).await?];
+    let mut stack = vec![Frame::new(path).await?];
 
     while let Some(frame) = stack.last_mut() {
         let file_type = frame.metadata.file_type();
@@ -325,6 +326,7 @@ pub(crate) async fn get(
     path: web::Path<PathParams>,
     req: HttpRequest,
     q: web::Query<NarRequest>,
+    settings: web::Data<Config>,
 ) -> Result<HttpResponse, Box<dyn Error>> {
     // Extract the narhash from the query parameter, and bail out if it's missing or invalid.
     let narhash = some_or_404!(Some(path.narhash.as_str()));
@@ -386,7 +388,11 @@ pub(crate) async fn get(
 
         let (tx2, mut rx2) = tokio::sync::mpsc::channel::<Result<Bytes, ThreadSafeError>>(1000);
         task::spawn(async move {
-            let err = dump_path(Path::new(&store_path), &tx2).await;
+            // If Nix is set to a non-root store, physical store paths will differ from
+            // logical paths. Below we check if that is the case, and rewrite to physical
+            // before dumping.
+
+            let err = dump_path(settings.store.get_real_path(&store_path), &tx2).await;
             if let Err(err) = err {
                 log::error!("Error dumping path {}: {:?}", store_path, err);
             }
@@ -426,7 +432,7 @@ pub(crate) async fn get(
         });
     } else {
         task::spawn(async move {
-            let err = dump_path(Path::new(&store_path), &tx).await;
+            let err = dump_path(settings.store.get_real_path(&store_path), &tx).await;
             if let Err(err) = err {
                 log::error!("Error dumping path {}: {:?}", store_path, err);
             }
@@ -443,12 +449,14 @@ pub(crate) async fn get(
 #[cfg(test)]
 mod test {
     use super::*;
-    use std::{path::PathBuf, process::Command};
+    use crate::store::Store;
+    use std::process::Command;
 
-    async fn dump_to_vec(path: PathBuf) -> Result<Vec<u8>> {
+    async fn dump_to_vec(path: String) -> Result<Vec<u8>> {
+        let store = Store::new();
         let (tx, mut rx) = tokio::sync::mpsc::channel::<Result<Bytes, ThreadSafeError>>(1000);
         task::spawn(async move {
-            let e = dump_path(&path, &tx).await;
+            let e = dump_path(store.get_real_path(&path), &tx).await;
             if let Err(e) = e {
                 eprintln!("Error dumping path: {:?}", e);
             }
@@ -521,7 +529,7 @@ mod test {
 
         std::os::unix::fs::symlink("sometarget", dir.join("symlink"))?;
 
-        let nar_dump = dump_to_vec(dir.to_path_buf()).await?;
+        let nar_dump = dump_to_vec(dir.to_str().unwrap().to_owned()).await?;
         let res = Command::new("nix-store")
             .arg("--dump")
             .arg(dir)

harmonia/src/serve.rs

@@ -8,7 +8,9 @@ use askama_escape::{escape as escape_html_entity, Html};
 use percent_encoding::{utf8_percent_encode, CONTROLS};
 use std::fmt::Write;
 
-use crate::{nixhash, some_or_404, ServerResult, BOOTSTRAP_SOURCE, CARGO_NAME, CARGO_VERSION};
+use crate::{
+    config::Config, nixhash, some_or_404, ServerResult, BOOTSTRAP_SOURCE, CARGO_NAME, CARGO_VERSION,
+};
 
 /// Returns percent encoded file URL path.
 macro_rules! encode_file_url {
@@ -48,7 +50,7 @@ fn file_size(bytes: u64) -> String {
 
 pub(crate) fn directory_listing(url_prefix: &Path, fs_path: &Path) -> ServerResult {
     let path_without_store = fs_path
-        .strip_prefix(libnixstore::get_store_dir())
+        .strip_prefix(libnixstore::get_real_store_dir())
         .unwrap_or(fs_path);
     let index_of = format!(
         "Index of {}",
@@ -123,11 +125,15 @@ pub(crate) fn directory_listing(url_prefix: &Path, fs_path: &Path) -> ServerResu
         .body(html))
 }
 
-pub(crate) async fn get(path: web::Path<(String, PathBuf)>, req: HttpRequest) -> ServerResult {
+pub(crate) async fn get(
+    path: web::Path<(String, PathBuf)>,
+    req: HttpRequest,
+    settings: web::Data<Config>,
+) -> ServerResult {
     let (hash, dir) = path.into_inner();
     let dir = dir.strip_prefix("/").unwrap_or(&dir);
 
-    let store_path = PathBuf::from(some_or_404!(nixhash(&hash)));
+    let store_path = settings.store.get_real_path(&some_or_404!(nixhash(&hash)));
     let full_path = if dir == Path::new("") {
         store_path.clone()
     } else {
@@ -137,7 +143,7 @@ pub(crate) async fn get(path: web::Path<(String, PathBuf)>, req: HttpRequest) ->
         .canonicalize()
         .with_context(|| format!("cannot resolve nix store path: {}", full_path.display()))?;
 
-    if !full_path.starts_with(libnixstore::get_store_dir()) {
+    if !full_path.starts_with(settings.store.real_store()) {
         return Ok(HttpResponse::NotFound().finish());
     }
 

harmonia/src/store.rs

@@ -0,0 +1,42 @@
+use std::path::PathBuf;
+
+#[derive(Default, Debug)]
+pub struct Store {
+    virtual_store: String,
+    real_store: Option<String>,
+}
+
+impl Store {
+    pub fn new() -> Self {
+        let real_store = libnixstore::get_real_store_dir();
+        let virtual_store = libnixstore::get_store_dir();
+
+        if virtual_store == real_store {
+            return Self {
+                virtual_store,
+                real_store: None,
+            };
+        }
+        Self {
+            virtual_store,
+            real_store: Some(real_store),
+        }
+    }
+    pub fn get_real_path(&self, virtual_path: &str) -> PathBuf {
+        if let Some(real_store) = &self.real_store {
+            if virtual_path.starts_with(&self.virtual_store) {
+                return PathBuf::from(format!(
+                    "{}{}",
+                    real_store,
+                    &virtual_path[self.virtual_store.len()..]
+                ));
+            }
+        }
+        PathBuf::from(virtual_path)
+    }
+    pub fn real_store(&self) -> &str {
+        self.real_store
+            .as_ref()
+            .map_or(&self.virtual_store, |s| s.as_str())
+    }
+}

libnixstore/include/nix.h

@@ -14,6 +14,7 @@ rust::String sign_string(rust::Str secret_key, rust::Str msg);
 bool check_signature(rust::Str public_key, rust::Str sig, rust::Str msg);
 InternalDrv derivation_from_path(rust::Str drv_path);
 rust::String get_store_dir();
+rust::String get_real_store_dir();
 rust::String get_build_log(rust::Str derivation_path);
 rust::String get_nar_list(rust::Str store_path);
 

libnixstore/src/lib.rs

@@ -48,6 +48,7 @@ mod ffi {
         fn check_signature(public_key: &str, sig: &str, msg: &str) -> Result<bool>;
         fn derivation_from_path(drv_path: &str) -> Result<InternalDrv>;
         fn get_store_dir() -> String;
+        fn get_real_store_dir() -> String;
         fn get_build_log(derivation_path: &str) -> Result<String>;
         fn get_nar_list(store_path: &str) -> Result<String>;
     }
@@ -218,6 +219,13 @@ pub fn get_store_dir() -> String {
     ffi::get_store_dir()
 }
 
+#[inline]
+#[must_use]
+/// Returns the physical path to the nix store.
+pub fn get_real_store_dir() -> String {
+    ffi::get_real_store_dir()
+}
+
 #[inline]
 #[must_use]
 /// Return the build log of the specified store path, if available, or null otherwise.

libnixstore/src/nix.cpp

@@ -6,6 +6,7 @@
 #include <nix/globals.hh>
 #include <nix/shared.hh>
 #include <nix/store-api.hh>
+#include <nix/local-fs-store.hh>
 #include <nix/log-store.hh>
 #include <nix/content-address.hh>
 #include <nix/util.hh>
@@ -192,6 +193,16 @@ rust::String get_store_dir() {
   return nix::settings.nixStore;
 }
 
+rust::String get_real_store_dir() {
+  auto store = get_store();
+  auto *fsstore = dynamic_cast<nix::LocalFSStore *>(&(*store));
+
+  if (fsstore != nullptr)
+    return fsstore->getRealStoreDir();
+  else
+    return get_store_dir();
+}
+
 rust::String get_build_log(rust::Str derivation_path) {
   auto store = get_store();
   auto path = store->parseStorePath(STRING_VIEW(derivation_path));

tests/t03-chroot.nix

@@ -0,0 +1,66 @@
+(import ./lib.nix)
+  ({ ... }:
+  {
+    name = "t03-chroot";
+
+    nodes = {
+      harmonia = { pkgs, ... }:
+        {
+          imports = [ ../module.nix ];
+
+          services.harmonia-dev.enable = true;
+          # We need to manipulate the target store first
+          systemd.services."harmonia-dev".wantedBy = pkgs.lib.mkForce [ ];
+
+          networking.firewall.allowedTCPPorts = [ 5000 ];
+          nix.settings.store = "/guest?read-only=1";
+          nix.extraOptions = ''
+            experimental-features = nix-command read-only-local-store
+          '';
+        };
+
+      client01 = { lib, ... }:
+        {
+          nix.settings.require-sigs = false;
+          nix.settings.substituters = lib.mkForce [ "http://harmonia:5000" ];
+          nix.extraOptions = ''
+            experimental-features = nix-command
+          '';
+        };
+    };
+
+    testScript =
+      ''
+        import json
+        start_all()
+
+        harmonia.wait_until_succeeds("echo 'test contents' > /my-file")
+        harmonia.wait_until_succeeds("mkdir /my-dir && cp /my-file /my-dir/")
+        f = harmonia.wait_until_succeeds("nix store --store /guest add-file /my-file")
+        d = harmonia.wait_until_succeeds("nix store --store /guest add-path /my-dir")
+        harmonia.systemctl("start harmonia-dev.service")
+        harmonia.wait_for_unit("harmonia-dev.service")
+
+        client01.wait_until_succeeds("curl -f http://harmonia:5000/version")
+        client01.succeed("curl -f http://harmonia:5000/nix-cache-info")
+
+        client01.wait_until_succeeds(f"nix copy --from http://harmonia:5000/ {f}")
+        client01.succeed(f"grep 'test contents' {f}")
+
+        dhash = d.removeprefix("/nix/store/")
+        dhash = dhash[:dhash.find('-')]
+        out = client01.wait_until_succeeds(f"curl -v http://harmonia:5000/{dhash}.ls")
+        data = json.loads(out)
+        print(out)
+        assert data["version"] == 1, "version is not correct"
+        assert data["root"]["entries"]["my-file"]["type"] == "regular", "expect my-file file in listing"
+
+        out = client01.wait_until_succeeds(f"curl -v http://harmonia:5000/serve/{dhash}/")
+        print(out)
+        assert "my-file" in out, "my-file not in listing"
+
+        out = client01.wait_until_succeeds(f"curl -v http://harmonia:5000/serve/{dhash}/my-file").strip()
+        print(out)
+        assert "test contents" == out, f"expected 'test contents', got '{out}'"
+      '';
+  })