[WasmFS] Add virtual backend utilities (#18248)

tlively · web-flow · commit 3a64fd404f08 · 2022-11-29T15:52:56.000-06:00
Add generic subclasses of DataFile, Directory, and Symlink that wrap other files
and pass operations through to them. Use the new utilities in a rewrite of the
ignore-case virtual backend that cleanly separates the underlying "real" file
system from the virtual file system. This also fixes outstanding issues with the
ignore case backend in which the original, non-normalized file names were not
preserved.
diff --git a/system/lib/wasmfs/backends/ignore_case_backend.cpp b/system/lib/wasmfs/backends/ignore_case_backend.cpp
@@ -3,12 +3,19 @@
 // University of Illinois/NCSA Open Source License.  Both these licenses can be
 // found in the LICENSE file.
 
-// This file defines the Ignore Case Backend of the new file system.
-// It is a virtual backend that normalizes all file paths to lower case.
+// A virtual backend that adapts any underlying backend to be
+// case-insensitive. IgnoreCaseDirectory intercepts all directory operations,
+// normalizes paths to be lower case, then forwards the operations with the new
+// paths to the underlying backend. It stores the original, non-normalized names
+// internally so they can be returned later, giving the appearance of a
+// case-insensitive but case-preserving file system.
+//
+// See the comment in virtual.h for an explanation of why DataFiles and Symlinks
+// must have no-op wrappers.
 
 #include "backend.h"
 #include "file.h"
-#include "memory_backend.h"
+#include "virtual.h"
 #include "wasmfs.h"
 
 namespace {
@@ -20,105 +27,178 @@ std::string normalize(const std::string& name) {
   }
   return result;
 }
-} // namespace
+
+} // anonymous namespace
 
 namespace wasmfs {
 
-// Problem: Child entries are stored both in IgnoreCaseDirectory and in baseDirectory.
-//
-// Original name case preservation could be possible if MemoryDirectory::entries
-// were accesible. Then, we would store the original case there but search
-// ignoring case.
-//
-class IgnoreCaseDirectory : public MemoryDirectory {
-  using BaseClass = MemoryDirectory;
-  std::shared_ptr<Directory> baseDirectory;
+class IgnoreCaseDirectory : public VirtualDirectory {
+
+  struct ChildInfo {
+    std::string originalName;
+    std::shared_ptr<File> child;
+  };
+
+  // Map normalized names to virtual files and their non-normalized names.
+  std::map<std::string, ChildInfo> children;
 
 public:
-  IgnoreCaseDirectory(std::shared_ptr<Directory> base, backend_t proxyBackend)
-    : BaseClass(base->locked().getMode(), proxyBackend), baseDirectory(base) {}
+  IgnoreCaseDirectory(std::shared_ptr<Directory> real, backend_t backend)
+    : VirtualDirectory(real, backend) {}
+
+  std::shared_ptr<File> getChild(const std::string& name) override;
+  std::shared_ptr<DataFile> insertDataFile(const std::string& name,
+                                           mode_t mode) override;
+  std::shared_ptr<Directory> insertDirectory(const std::string& name,
+                                             mode_t mode) override;
+  std::shared_ptr<Symlink> insertSymlink(const std::string& name,
+                                         const std::string& target) override;
+  int insertMove(const std::string& name, std::shared_ptr<File> file) override;
+  int removeChild(const std::string& name) override;
+  ssize_t getNumEntries() override { return real->locked().getNumEntries(); }
+  Directory::MaybeEntries getEntries() override;
+  std::string getName(std::shared_ptr<File> file) override;
+  bool maintainsFileIdentity() override { return true; }
+};
 
-  std::shared_ptr<File> getChild(const std::string& name) override {
-    return BaseClass::getChild(normalize(name));
+// Wrap a real file in an IgnoreCase virtual file of the same kind.
+std::shared_ptr<DataFile> virtualize(std::shared_ptr<DataFile> data,
+                                     backend_t backend) {
+  return std::make_shared<VirtualDataFile>(data, backend);
+}
+
+std::shared_ptr<Directory> virtualize(std::shared_ptr<Directory> dir,
+                                      backend_t backend) {
+  return std::make_shared<IgnoreCaseDirectory>(dir, backend);
+}
+
+std::shared_ptr<Symlink> virtualize(std::shared_ptr<Symlink> link,
+                                    backend_t backend) {
+  return std::make_shared<VirtualSymlink>(link, backend);
+}
+
+std::shared_ptr<File> virtualize(std::shared_ptr<File> file,
+                                 backend_t backend) {
+  if (auto data = file->dynCast<DataFile>()) {
+    return virtualize(data, backend);
+  } else if (auto dir = file->dynCast<Directory>()) {
+    return virtualize(dir, backend);
+  } else if (auto link = file->dynCast<Symlink>()) {
+    return virtualize(link, backend);
   }
+  WASMFS_UNREACHABLE("unexpected file kind");
+}
 
-  std::shared_ptr<DataFile> insertDataFile(const std::string& name,
-                                           mode_t mode) override {
-    auto name2 = normalize(name);
-    auto baseDirLocked = baseDirectory->locked();
-    auto child = baseDirLocked.insertDataFile(name2, mode);
-    if (child) {
-      insertChild(name2, child);
-      // Directory::Hanlde needs a parent
-      child->locked().setParent(cast<Directory>());
-    }
-    return child;
+std::shared_ptr<File> IgnoreCaseDirectory::getChild(const std::string& name) {
+  auto normalized = normalize(name);
+  if (auto it = children.find(normalized); it != children.end()) {
+    return it->second.child;
   }
+  auto child = real->locked().getChild(normalized);
+  if (!child) {
+    return nullptr;
+  }
+  child = virtualize(child, getBackend());
+  children[normalized] = {name, child};
+  return child;
+}
 
-  std::shared_ptr<Directory> insertDirectory(const std::string& name,
-                                             mode_t mode) override {
-    auto name2 = normalize(name);
-    auto baseDirLocked = baseDirectory->locked();
-    if (!baseDirLocked.getParent())
-      baseDirLocked.setParent(parent.lock()); // Directory::Hanlde needs a parent
-    auto baseChild = baseDirLocked.insertDirectory(name2, mode);
-    auto child = std::make_shared<IgnoreCaseDirectory>(baseChild, getBackend());
-    insertChild(name2, child);
-    return child;
+std::shared_ptr<DataFile>
+IgnoreCaseDirectory::insertDataFile(const std::string& name, mode_t mode) {
+  auto normalized = normalize(name);
+  auto file = real->locked().insertDataFile(normalized, mode);
+  if (!file) {
+    return nullptr;
   }
+  file = virtualize(file, getBackend());
+  children[normalized] = {name, file};
+  return file;
+}
 
-  std::shared_ptr<Symlink> insertSymlink(const std::string& name,
-                                         const std::string& target) override {
-    auto name2 = normalize(name);
-    auto child = baseDirectory->locked().insertSymlink(name2, target);
-    if (child) {
-      insertChild(name2, child);
-      // Directory::Hanlde needs a parent
-      child->locked().setParent(cast<Directory>());
-    }
-    return child;
+std::shared_ptr<Directory>
+IgnoreCaseDirectory::insertDirectory(const std::string& name, mode_t mode) {
+  auto normalized = normalize(name);
+  auto dir = real->locked().insertDirectory(normalized, mode);
+  if (!dir) {
+    return nullptr;
   }
+  dir = virtualize(dir, getBackend());
+  children[normalized] = {name, dir};
+  return dir;
+}
 
-  int insertMove(const std::string& name, std::shared_ptr<File> file) override {
-    auto newName = normalize(name);
-    // Remove entry with the new name (if any) from this directory.
-    if (auto err = removeChild(newName))
-      return err;
-    auto oldParent = file->locked().getParent()->locked();
-    auto oldName = normalize(oldParent.getName(file));
-    // Move in underlying directory.
-    if (auto err = baseDirectory->locked().insertMove(newName, file))
-      return err;
-    // Ensure old file was removed.
-    if (auto err = oldParent.removeChild(oldName))
-      return err;
-    // Cache file with the new name in this directory.
-    insertChild(newName, file);
-    file->locked().setParent(cast<Directory>());
-    return 0;
+std::shared_ptr<Symlink>
+IgnoreCaseDirectory::insertSymlink(const std::string& name,
+                                   const std::string& target) {
+  auto normalized = normalize(name);
+  auto link = real->locked().insertSymlink(normalized, target);
+  if (!link) {
+    return nullptr;
   }
+  link = virtualize(link, getBackend());
+  children[normalized] = {name, link};
+  return link;
+}
 
-  int removeChild(const std::string& name) override {
-    auto name2 = normalize(name);
-    if (auto err = BaseClass::removeChild(name2))
-      return err;
-    return baseDirectory->locked().removeChild(name2);
+int IgnoreCaseDirectory::insertMove(const std::string& name,
+                                    std::shared_ptr<File> file) {
+  auto normalized = normalize(name);
+  if (auto err = real->locked().insertMove(normalized, devirtualize(file))) {
+    return err;
+  }
+  auto oldParent =
+    std::static_pointer_cast<IgnoreCaseDirectory>(file->locked().getParent());
+  auto& oldChildren = oldParent->children;
+  // Delete the entry in the old parent.
+  for (auto it = oldChildren.begin(); it != oldChildren.end(); ++it) {
+    if (it->second.child == file) {
+      oldChildren.erase(it);
+      break;
+    }
+  }
+  // Unlink the overwritten entry if it exists.
+  auto [it, inserted] = children.insert({normalized, {name, file}});
+  if (!inserted) {
+    it->second.child->locked().setParent(nullptr);
+    it->second = {name, file};
   }
 
-  ssize_t getNumEntries() override { return baseDirectory->locked().getNumEntries(); }
+  return 0;
+}
 
-  // TODO: preserve original name, denormalize, and use it here
-  Directory::MaybeEntries getEntries() override {
-    return baseDirectory->locked().getEntries();
+int IgnoreCaseDirectory::removeChild(const std::string& name) {
+  auto normalized = normalize(name);
+  if (auto err = real->locked().removeChild(normalized)) {
+    return err;
   }
+  auto it = children.find(normalized);
+  assert(it != children.end());
+  it->second.child->locked().setParent(nullptr);
+  children.erase(it);
+  return 0;
+}
 
-  // TODO: preserve original name, denormalize, and use it here
-  std::string getName(std::shared_ptr<File> file) override {
-    return BaseClass::getName(file);
+Directory::MaybeEntries IgnoreCaseDirectory::getEntries() {
+  auto entries = real->locked().getEntries();
+  if (auto err = entries.getError()) {
+    return entries;
   }
+  for (auto& entry : *entries) {
+    if (auto it = children.find(entry.name); it != children.end()) {
+      entry.name = it->second.originalName;
+    }
+  }
+  return entries;
+}
 
-  bool maintainsFileIdentity() override { return true; }
-};
+std::string IgnoreCaseDirectory::getName(std::shared_ptr<File> file) {
+  for (auto& [_, info] : children) {
+    if (info.child == file) {
+      return info.originalName;
+    }
+  }
+  return "";
+}
 
 class IgnoreCaseBackend : public Backend {
   backend_t backend;
@@ -129,16 +209,21 @@ class IgnoreCaseBackend : public Backend {
   }
 
   std::shared_ptr<DataFile> createFile(mode_t mode) override {
-    return backend->createFile(mode);
+    return virtualize(backend->createFile(mode), this);
   }
 
   std::shared_ptr<Directory> createDirectory(mode_t mode) override {
-    return std::make_shared<IgnoreCaseDirectory>(backend->createDirectory(mode),
-                                                 this);
+    auto real = backend->createDirectory(mode);
+    // Inserts into the real backing directory won't work if it doesn't appear
+    // to be linked, so give it a parent.
+    // TODO: Break this reference cycle in a destructor somewhere.
+    real->locked().setParent(real);
+    auto ret = virtualize(real, this);
+    return ret;
   }
 
   std::shared_ptr<Symlink> createSymlink(std::string target) override {
-    return backend->createSymlink(normalize(target));
+    return virtualize(backend->createSymlink(target), this);
   }
 };
 
@@ -148,12 +233,14 @@ backend_t createIgnoreCaseBackend(std::function<backend_t()> createBackend) {
 }
 
 extern "C" {
+
 // C API for creating ignore case backend.
 backend_t wasmfs_create_icase_backend(backend_constructor_t create_backend,
                                       void* arg) {
   return createIgnoreCaseBackend(
     [create_backend, arg]() { return create_backend(arg); });
 }
-}
+
+} // extern "C"
 
 } // namespace wasmfs
diff --git a/system/lib/wasmfs/file.cpp b/system/lib/wasmfs/file.cpp
@@ -178,12 +178,13 @@ int Directory::Handle::insertMove(const std::string& name,
       it->second.file->locked().setParent(nullptr);
       it->second = entry;
     }
-    file->locked().setParent(getDir());
   } else {
     // This backend doesn't use the dcache.
     assert(getDir()->maintainsFileIdentity());
   }
 
+  file->locked().setParent(getDir());
+
   // TODO: Moving mount points probably shouldn't update the mtime.
   auto now = time(NULL);
   oldParent->locked().setMTime(now);
diff --git a/system/lib/wasmfs/file.h b/system/lib/wasmfs/file.h
@@ -181,6 +181,10 @@ class Directory : public File {
       return 0;
     }
 
+    std::vector<Entry>& operator*() {
+      return *std::get_if<std::vector<Entry>>(this);
+    }
+
     std::vector<Entry>* operator->() {
       return std::get_if<std::vector<Entry>>(this);
     }
@@ -253,7 +257,7 @@ class Directory : public File {
   //  1. Ensuring that all insert* and getChild calls returning a particular
   //     file return the same File object.
   //
-  //  2. Clearing the File's parent field in `removeChild`.
+  //  2. Clearing unlinked Files' parents in `removeChild` and `insertMove`.
   //
   //  3. Implementing `getName`, since it cannot be implemented in terms of the
   //     dcache.
diff --git a/system/lib/wasmfs/memory_backend.h b/system/lib/wasmfs/memory_backend.h
@@ -15,6 +15,7 @@
 #include <emscripten/threading.h>
 
 namespace wasmfs {
+
 // This class describes a file that lives in Wasm Memory.
 class MemoryDataFile : public DataFile {
   std::vector<uint8_t> buffer;
diff --git a/system/lib/wasmfs/virtual.h b/system/lib/wasmfs/virtual.h
diff --git a/test/other/test_fs_icase.cpp b/test/other/test_fs_icase.cpp

Original file line number	Diff line number	Diff line change
`@@ -178,12 +178,13 @@ int Directory::Handle::insertMove(const std::string& name,`
`178`	`178`	`it->second.file->locked().setParent(nullptr);`
`179`	`179`	`it->second = entry;`
`180`	`180`	`}`
`181`		`- file->locked().setParent(getDir());`
`182`	`181`	`} else {`
`183`	`182`	`// This backend doesn't use the dcache.`
`184`	`183`	`assert(getDir()->maintainsFileIdentity());`
`185`	`184`	`}`
`186`	`185`
	`186`	`+ file->locked().setParent(getDir());`
	`187`	`+`
`187`	`188`	`// TODO: Moving mount points probably shouldn't update the mtime.`
`188`	`189`	`auto now = time(NULL);`
`189`	`190`	`oldParent->locked().setMTime(now);`