Add Typesense (#44)

Author: ruslandoga

Diff for: .github/workflows/main.yml

@@ -24,6 +24,9 @@ jobs:
       - name: Check mix format
         run: mix format --check-formatted
 
+      - name: Start Typesense
+        run: docker compose up -d typesense
+
       - name: Run tests
         run: |
-          mix test
+          mix test --include typesense

Diff for: compose.yml

@@ -0,0 +1,6 @@
+services:
+  typesense:
+    image: typesense/typesense:27.1
+    command: --data-dir /tmp --api-key=hexdocs
+    ports:
+      - 8108:8108

Diff for: config/config.exs

@@ -4,9 +4,13 @@ config :hexdocs,
   port: "4002",
   hexpm_url: "http://localhost:4000",
   hexpm_secret: "2cd6d09334d4b00a2be4d532342b799b",
+  typesense_url: "http://localhost:8108",
+  typesense_api_key: "hexdocs",
+  typesense_collection: "hexdocs",
   hexpm_impl: Hexdocs.Hexpm.Impl,
   store_impl: Hexdocs.Store.Local,
   cdn_impl: Hexdocs.CDN.Local,
+  search_impl: Hexdocs.Search.Local,
   source_repo_impl: Hexdocs.SourceRepo.GitHub,
   tmp_dir: "tmp",
   queue_id: "test",

Diff for: config/dev.exs

@@ -5,4 +5,5 @@ config :hexdocs,
   hexpm_url: "http://localhost:4000",
   hexpm_impl: Hexdocs.Hexpm.Impl,
   store_impl: Hexdocs.Store.Local,
-  cdn_impl: Hexdocs.CDN.Local
+  cdn_impl: Hexdocs.CDN.Local,
+  search_impl: Hexdocs.Search.Local

Diff for: config/prod.exs

@@ -4,6 +4,7 @@ config :hexdocs,
   hexpm_impl: Hexdocs.Hexpm.Impl,
   store_impl: Hexdocs.Store.Impl,
   cdn_impl: Hexdocs.CDN.Fastly,
+  search_impl: Hexdocs.Search.Typesense,
   queue_producer: BroadwaySQS.Producer,
   gcs_put_debounce: 3000
 

Diff for: config/runtime.exs

@@ -5,6 +5,9 @@ if config_env() == :prod do
     port: System.fetch_env!("HEXDOCS_PORT"),
     hexpm_url: System.fetch_env!("HEXDOCS_HEXPM_URL"),
     hexpm_secret: System.fetch_env!("HEXDOCS_HEXPM_SECRET"),
+    typesense_url: System.fetch_env!("HEXDOCS_TYPESENSE_URL"),
+    typesense_api_key: System.fetch_env!("HEXDOCS_TYPESENSE_API_KEY"),
+    typesense_collection: System.fetch_env!("HEXDOCS_TYPESENSE_COLLECTION"),
     fastly_key: System.fetch_env!("HEXDOCS_FASTLY_KEY"),
     fastly_hexdocs: System.fetch_env!("HEXDOCS_FASTLY_HEXDOCS"),
     queue_id: System.fetch_env!("HEXDOCS_QUEUE_ID"),

Diff for: config/test.exs

@@ -6,6 +6,7 @@ config :hexdocs,
   hexpm_impl: Hexdocs.HexpmMock,
   store_impl: Hexdocs.Store.Local,
   cdn_impl: Hexdocs.CDN.Local,
+  search_impl: Hexdocs.Search.Local,
   source_repo_impl: Hexdocs.SourceRepo.Mock
 
 config :logger, level: :warning

Diff for: lib/hexdocs/http.ex

@@ -23,6 +23,10 @@ defmodule Hexdocs.HTTP do
     |> read_response()
   end
 
+  def post(url, headers, body, opts \\ []) do
+    :hackney.post(url, headers, body, opts)
+  end
+
   def delete(url, headers) do
     :hackney.delete(url, headers)
     |> read_response()

Diff for: lib/hexdocs/queue.ex

@@ -125,8 +125,12 @@ defmodule Hexdocs.Queue do
               update_package_sitemap(repository, key, package, files)
             end
 
+            if repository == "hexpm" do
+              update_search_index(key, package, version, files)
+            end
+
             elapsed = System.os_time(:millisecond) - start
-            Logger.info("FINISHED UPLOADING DOCS #{key} #{elapsed}ms")
+            Logger.info("FINISHED UPLOADING AND INDEXING DOCS #{key} #{elapsed}ms")
 
           {:error, reason} ->
             Logger.error("Failed unpack #{repository}/#{package} #{version}: #{reason}")
@@ -149,6 +153,10 @@ defmodule Hexdocs.Queue do
         Hexdocs.Bucket.delete(repository, package, version, all_versions)
         update_index_sitemap(repository, key)
 
+        if repository == "hexpm" do
+          Hexdocs.Search.delete(package, version)
+        end
+
         elapsed = System.os_time(:millisecond) - start
         Logger.info("FINISHED DELETING DOCS #{key} #{elapsed}ms")
         :ok
@@ -228,6 +236,14 @@ defmodule Hexdocs.Queue do
     :ok
   end
 
+  defp update_search_index(key, package, version, files) do
+    with {proglang, items} <- Hexdocs.Search.find_search_items(package, version, files) do
+      Logger.info("UPDATING SEARCH INDEX #{key}")
+      Hexdocs.Search.index(package, version, proglang, items)
+      Logger.info("UPDATED SEARCH INDEX #{key}")
+    end
+  end
+
   @doc false
   def paths_for_sitemaps() do
     key_regex = ~r"docs/(.*)-(.*).tar.gz$"

Diff for: lib/hexdocs/search/local.ex

@@ -0,0 +1,9 @@
+defmodule Hexdocs.Search.Local do
+  @behaviour Hexdocs.Search
+
+  @impl true
+  def index(_package, _version, _proglang, _items), do: :ok
+
+  @impl true
+  def delete(_package, _version), do: :ok
+end

Diff for: lib/hexdocs/search/search.ex

@@ -0,0 +1,93 @@
+defmodule Hexdocs.Search do
+  require Logger
+
+  @type package :: String.t()
+  @type version :: Version.t()
+  @type proglang :: String.t()
+  @type search_items :: [map]
+
+  @callback index(package, version, proglang, search_items) :: :ok
+  @callback delete(package, version) :: :ok
+
+  defp impl, do: Application.fetch_env!(:hexdocs, :search_impl)
+
+  @spec index(package, version, proglang, search_items) :: :ok
+  def index(package, version, proglang, search_items) do
+    impl().index(package, version, proglang, search_items)
+  end
+
+  @spec delete(package, version) :: :ok
+  def delete(package, version) do
+    impl().delete(package, version)
+  end
+
+  @spec find_search_items(package, version, [{Path.t(), content :: iodata}]) ::
+          {proglang, search_items} | nil
+  def find_search_items(package, version, files) do
+    search_data_js =
+      Enum.find_value(files, fn {path, content} ->
+        case Path.basename(path) do
+          "search_data-" <> _digest -> content
+          _other -> nil
+        end
+      end)
+
+    unless search_data_js do
+      Logger.info("Failed to find search data for #{package} #{version}")
+    end
+
+    search_data_json =
+      case search_data_js do
+        "searchData=" <> json ->
+          json
+
+        _ when is_binary(search_data_js) ->
+          Logger.error("Unexpected search_data format for #{package} #{version}")
+          nil
+
+        nil ->
+          nil
+      end
+
+    search_data =
+      if search_data_json do
+        try do
+          :json.decode(search_data_json)
+        catch
+          _kind, reason ->
+            Logger.error(
+              "Failed to decode search data json for #{package} #{version}: " <>
+                inspect(reason)
+            )
+
+            nil
+        end
+      end
+
+    case search_data do
+      %{"items" => [_ | _] = search_items} ->
+        proglang = Map.get(search_data, "proglang") || proglang(search_items)
+        {proglang, search_items}
+
+      nil ->
+        nil
+
+      _ ->
+        Logger.error(
+          "Failed to extract search items and proglang from search data for #{package} #{version}"
+        )
+
+        nil
+    end
+  end
+
+  defp proglang(search_items) do
+    if Enum.any?(search_items, &elixir_module?/1), do: "elixir", else: "erlang"
+  end
+
+  defp elixir_module?(%{"type" => "module", "title" => <<first_letter, _::binary>>})
+       when first_letter in ?A..?Z,
+       do: true
+
+  defp elixir_module?(_), do: false
+end

Diff for: lib/hexdocs/search/typesense.ex

@@ -0,0 +1,105 @@
+defmodule Hexdocs.Search.Typesense do
+  @moduledoc false
+  require Logger
+  alias Hexdocs.HTTP
+
+  @behaviour Hexdocs.Search
+
+  @impl true
+  def index(package, version, proglang, search_items) do
+    full_package = full_package(package, version)
+
+    ndjson =
+      Enum.map(search_items, fn item ->
+        json =
+          Map.take(item, ["type", "ref", "title", "doc"])
+          |> Map.put("package", full_package)
+          |> Map.put("proglang", proglang)
+          |> :json.encode()
+
+        [json, ?\n]
+      end)
+
+    url = url("collections/#{collection()}/documents/import?action=create")
+    headers = [{"x-typesense-api-key", api_key()}]
+
+    case HTTP.post(url, headers, ndjson, [:with_body]) do
+      {:ok, 200, _resp_headers, ndjson} ->
+        ndjson
+        |> String.split("\n")
+        |> Enum.each(fn json ->
+          case :json.decode(json) do
+            %{"success" => true} ->
+              :ok
+
+            %{"success" => false, "error" => error, "document" => document} ->
+              Logger.error(
+                "Failed to index search item for #{package} #{version} for document #{inspect(document)}: #{inspect(error)}"
+              )
+          end
+        end)
+
+      {:ok, status, _resp_headers, _body} ->
+        Logger.error("Failed to index search items for #{package} #{version}: status=#{status}")
+
+      {:error, reason} ->
+        Logger.error("Failed to index search items #{package} #{version}: #{inspect(reason)}")
+    end
+  end
+
+  @impl true
+  def delete(package, version) do
+    full_package = full_package(package, version)
+
+    query = URI.encode_query([{"filter_by", "package:#{full_package}"}])
+    url = url("collections/#{collection()}/documents?" <> query)
+    headers = [{"x-typesense-api-key", api_key()}]
+
+    case HTTP.delete(url, headers) do
+      {:ok, 200, _resp_headers, _body} ->
+        :ok
+
+      {:ok, status, _resp_headers, _body} ->
+        Logger.error("Failed to delete search items for #{package} #{version}: status=#{status}")
+
+      {:error, reason} ->
+        Logger.error(
+          "Failed to delete search items for #{package} #{version}: #{inspect(reason)}"
+        )
+    end
+  end
+
+  @spec collection :: String.t()
+  def collection do
+    Application.fetch_env!(:hexdocs, :typesense_collection)
+  end
+
+  @spec collection_schema :: map
+  def collection_schema(collection \\ collection()) do
+    %{
+      "fields" => [
+        %{"facet" => true, "name" => "proglang", "type" => "string"},
+        %{"facet" => true, "name" => "type", "type" => "string"},
+        %{"name" => "title", "type" => "string"},
+        %{"name" => "doc", "type" => "string"},
+        %{"facet" => true, "name" => "package", "type" => "string"}
+      ],
+      "name" => collection,
+      "token_separators" => [".", "_", "-", " ", ":", "@", "/"]
+    }
+  end
+
+  @spec api_key :: String.t()
+  def api_key do
+    Application.fetch_env!(:hexdocs, :typesense_api_key)
+  end
+
+  defp full_package(package, version) do
+    "#{package}-#{version}"
+  end
+
+  defp url(path) do
+    base_url = Application.fetch_env!(:hexdocs, :typesense_url)
+    Path.join(base_url, path)
+  end
+end