Support module-based search paths in backend

Summary: Support inclusion of toplevel module site packages in backend.

Reviewed By: dkgi

Differential Revision: D32599884

fbshipit-source-id: 79c875de3ccc19df26fe40cb8a05bd61d20237fe

Author: shannonzhu

source/searchPath.ml

@@ -14,6 +14,10 @@ type t =
       root: Path.t;
       subdirectory: string;
     }
+  | Submodule of {
+      root: Path.t;
+      submodule: string;
+    }
 [@@deriving sexp, compare, hash]
 
 type search_result = {
@@ -26,27 +30,35 @@ let equal = [%compare.equal: t]
 let get_root path =
   match path with
   | Root root -> root
-  | Subdirectory { root; _ } -> root
+  | Subdirectory { root; _ }
+  | Submodule { root; _ } ->
+      root
 
 
 let to_path path =
   match path with
   | Root root -> root
-  | Subdirectory { root; subdirectory } -> Path.create_relative ~root ~relative:subdirectory
+  | Subdirectory { root; subdirectory = relative }
+  | Submodule { root; submodule = relative } ->
+      Path.create_relative ~root ~relative
 
 
 let pp formatter = function
   | Root root -> Path.pp formatter root
-  | Subdirectory { root; subdirectory } ->
-      Format.fprintf formatter "%a$%s" Path.pp root subdirectory
+  | Subdirectory { root; subdirectory = relative }
+  | Submodule { root; submodule = relative } ->
+      Format.fprintf formatter "%a$%s" Path.pp root relative
 
 
 let show path = Format.asprintf "%a" pp path
 
 let create serialized =
   match String.split serialized ~on:'$' with
   | [root] -> Root (Path.create_absolute root)
-  | [root; subdirectory] -> Subdirectory { root = Path.create_absolute root; subdirectory }
+  | [root; path] -> (
+      match String.split path ~on:'.' with
+      | [subdirectory] -> Subdirectory { root = Path.create_absolute root; subdirectory }
+      | _ -> Submodule { root = Path.create_absolute root; submodule = path })
   | _ -> failwith (Format.asprintf "Unable to create search path from %s" serialized)
 
 
@@ -58,14 +70,22 @@ let normalize = function
           root = Path.create_absolute ~follow_symbolic_links:true (Path.absolute root);
           subdirectory;
         }
+  | Submodule { root; submodule } ->
+      Submodule
+        { root = Path.create_absolute ~follow_symbolic_links:true (Path.absolute root); submodule }
 
 
 let create_normalized serialized = create serialized |> normalize
 
 let search_for_path ~search_paths path =
   let under_root search_path =
     let open Option in
-    if Path.directory_contains ~directory:(to_path search_path) path then
+    let found =
+      match search_path with
+      | Submodule _ -> Path.equal (to_path search_path) path
+      | _ -> Path.directory_contains ~directory:(to_path search_path) path
+    in
+    if found then
       let root = get_root search_path in
       Path.get_relative_to_root ~root ~path
       >>| (fun relative -> Path.create_relative ~root ~relative)

source/searchPath.mli

@@ -13,6 +13,10 @@ type t =
       root: Path.t;
       subdirectory: string;
     }
+  | Submodule of {
+      root: Path.t;
+      submodule: string;
+    }
 [@@deriving sexp, compare, hash, show, eq]
 
 type search_result = {

source/service/check.ml

@@ -25,9 +25,20 @@ let check
     if not (Path.is_directory directory) then
       raise (Invalid_argument (Format.asprintf "`%a` is not a directory" Path.pp directory))
   in
+  let check_path_exists path =
+    if not (Path.file_exists path) then
+      raise (Invalid_argument (Format.asprintf "`%a` is not a valid path" Path.pp path))
+  in
+  let check_search_path_exists search_path =
+    match search_path with
+    | SearchPath.Root _
+    | SearchPath.Subdirectory _ ->
+        check_directory_exists (SearchPath.to_path search_path)
+    | SearchPath.Submodule _ -> check_path_exists (SearchPath.to_path search_path)
+  in
   source_paths |> List.map ~f:SearchPath.to_path |> List.iter ~f:check_directory_exists;
   check_directory_exists project_root;
-  search_paths |> List.map ~f:SearchPath.to_path |> List.iter ~f:check_directory_exists;
+  search_paths |> List.iter ~f:check_search_path_exists;
   (* Profiling helper *)
   Profiling.track_shared_memory_usage ~name:"Before module tracking" ();
 

source/test/searchPathTest.ml

@@ -23,6 +23,10 @@ let test_create_search_path _ =
     ~cmp:SearchPath.equal
     (SearchPath.Subdirectory { root; subdirectory = "subdirectory" })
     (SearchPath.create (Path.show root ^ "$subdirectory"));
+  assert_equal
+    ~cmp:SearchPath.equal
+    (SearchPath.Submodule { root; submodule = "submodule.py" })
+    (SearchPath.create (Path.show root ^ "$submodule.py"));
   assert_raises (Failure "Unable to create search path from too$many$levels") (fun () ->
       SearchPath.create "too$many$levels")
 
@@ -38,6 +42,8 @@ let test_show_search_path _ =
   assert_round_trip (SearchPath.Subdirectory { root = !"/foo"; subdirectory = "bar" });
   assert_round_trip (SearchPath.Subdirectory { root = !"/foo/bar"; subdirectory = "baz" });
   assert_round_trip (SearchPath.Subdirectory { root = !"/foo"; subdirectory = "bar/baz" });
+  assert_round_trip (SearchPath.Submodule { root = !"/foo"; submodule = "bar.py" });
+  assert_round_trip (SearchPath.Submodule { root = !"/foo"; submodule = "bar/baz.py" });
   ()
 
 
@@ -46,11 +52,12 @@ let test_normalize context =
   let bad_root = "nonexist/directory" in
   let good_subroot = good_root ^ "/subroot" in
   Sys_utils.mkdir_no_fail good_subroot;
-  let create_input ?subdirectory root =
+  let create_input ?subdirectory ?submodule root =
     let search_path =
-      match subdirectory with
-      | None -> SearchPath.Root !root
-      | Some subdirectory -> SearchPath.Subdirectory { root = !root; subdirectory }
+      match subdirectory, submodule with
+      | Some subdirectory, _ -> SearchPath.Subdirectory { root = !root; subdirectory }
+      | _, Some submodule -> SearchPath.Submodule { root = !root; submodule }
+      | _ -> SearchPath.Root !root
     in
     SearchPath.show search_path
   in
@@ -90,6 +97,7 @@ let test_normalize context =
   assert_success ~normalize:false ~expected:true (create_input ~subdirectory:"subroot" good_root);
   assert_success ~normalize:false ~expected:true (create_input ~subdirectory:"nosubroot" good_root);
   assert_success ~normalize:false ~expected:true (create_input ~subdirectory:"subroot" bad_root);
+  assert_success ~normalize:false ~expected:true (create_input ~submodule:"subroot" bad_root);
 
   (* Normalized creation depends on filesystem state. *)
   assert_success ~normalize:true ~expected:true (create_input good_root);
@@ -98,6 +106,8 @@ let test_normalize context =
   assert_success ~normalize:true ~expected:true (create_input ~subdirectory:"subroot" good_root);
   assert_success ~normalize:true ~expected:false (create_input ~subdirectory:"nosubroot" good_root);
   assert_success ~normalize:true ~expected:false (create_input ~subdirectory:"subroot" bad_root);
+  assert_success ~normalize:true ~expected:true (create_input ~submodule:"subroot" good_root);
+  assert_success ~normalize:true ~expected:false (create_input ~submodule:"subroot" bad_root);
 
   ()
 
@@ -116,6 +126,7 @@ let test_search_for_path context =
       SearchPath.Subdirectory
         { root = Path.create_relative ~root ~relative:"b"; subdirectory = "c" };
       SearchPath.Subdirectory { root; subdirectory = "b" };
+      SearchPath.Submodule { root; submodule = "b.py" };
     ]
   in
   assert_path
@@ -129,7 +140,8 @@ let test_search_for_path context =
   assert_path
     ~search_paths
     ~path:(Path.create_relative ~root ~relative:"b/other/file.py")
-    ~expected:"b/other/file.py"
+    ~expected:"b/other/file.py";
+  assert_path ~search_paths ~path:(Path.create_relative ~root ~relative:"b.py") ~expected:"b.py"
 
 
 let () =