fix(engine): isolate search store from async load crashes

apps/engine/lib/engine/application.ex

@@ -17,6 +17,7 @@ defmodule Engine.Application do
           Engine.Build.CaptureServer,
           Engine.Plugin.Runner.Supervisor,
           Engine.Plugin.Runner.Coordinator,
+          {Task.Supervisor, name: Engine.TaskSupervisor},
           Engine.Search.Store.Backends.Ets,
           {Engine.Search.Store,
            [

apps/engine/lib/engine/search/store.ex

@@ -154,9 +154,18 @@ defmodule Engine.Search.Store do
 
   # handle the result from `State.async_load/1`
   def handle_info({ref, result}, {update_ref, %State{async_load_ref: ref} = state}) do
+    Process.demonitor(ref, [:flush])
     {:noreply, {update_ref, State.async_load_complete(state, result)}}
   end
 
+  def handle_info(
+        {:DOWN, ref, :process, _pid, reason},
+        {update_ref, %State{async_load_ref: ref} = state}
+      ) do
+    Logger.error("Search index async load crashed: #{inspect(reason)}")
+    {:noreply, {update_ref, %State{state | async_load_ref: nil}}}
+  end
+
   def handle_info(:flush_updates, {_, %State{} = state}) do
     {:ok, state} = State.flush_buffered_updates(state)
     ref = schedule_flush()

apps/engine/lib/engine/search/store/state.ex

@@ -234,7 +234,7 @@ defmodule Engine.Search.Store.State do
 
   defp prepare_backend_async(%__MODULE__{async_load_ref: nil} = state, backend_result) do
     task =
-      Task.async(fn ->
+      Task.Supervisor.async_nolink(Engine.TaskSupervisor, fn ->
         case state.backend.prepare(backend_result) do
           {:ok, :empty} ->
             Logger.info("backend reports empty")

apps/engine/test/engine/code_intelligence/references_test.exs

@@ -20,6 +20,7 @@ defmodule Engine.CodeIntelligence.ReferencesTest do
     Backends.Ets.destroy_all(project)
     Engine.set_project(project)
 
+    start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor})
     start_supervised!(Document.Store)
     start_supervised!(Engine.Dispatch)
     start_supervised!(Backends.Ets)

apps/engine/test/engine/dispatch/handlers/indexer_test.exs

@@ -28,6 +28,7 @@ defmodule Engine.Dispatch.Handlers.IndexingTest do
 
     patch(Engine.Dispatch, :erpc_cast, fn Expert.Progress, _function, _args -> true end)
 
+    start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor})
     start_supervised!(Engine.Dispatch)
     start_supervised!(Commands.Reindex)
     start_supervised!(Search.Store.Backends.Ets)

apps/engine/test/engine/search/store/backends/ets_test.exs

@@ -61,6 +61,7 @@ defmodule Engine.Search.Store.Backend.EtsTest do
   end
 
   defp start_supervised_store(%Project{} = project, create_fn, update_fn, backend) do
+    start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor})
     start_supervised!(Dispatch)
     start_supervised!(Backends.Ets)
     start_supervised!({Store, [project, create_fn, update_fn, backend]})

apps/engine/test/engine/search/store_test.exs

@@ -368,6 +368,7 @@ defmodule Engine.Search.StoreTest do
   defp with_a_started_store(project, backend) do
     destroy_backend(backend, project)
 
+    start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor})
     start_supervised!(Dispatch)
     start_supervised!(backend)
     start_supervised!({Store, [project, &default_create/1, &default_update/2, backend]})
@@ -400,6 +401,7 @@ defmodule Engine.Search.StoreTest do
     setup %{project: project} do
       destroy_backend(Ets, project)
 
+      start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor})
       start_supervised!(Dispatch)
       start_supervised!(Ets)
 
@@ -458,4 +460,37 @@ defmodule Engine.Search.StoreTest do
       assert received_project == project
     end
   end
+
+  describe "async load crash recovery" do
+    setup %{project: project} do
+      destroy_backend(Ets, project)
+
+      start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor})
+      start_supervised!(Dispatch)
+      start_supervised!(Ets)
+
+      crashing_create = fn _project ->
+        raise "index build exploded"
+      end
+
+      start_supervised!({Store, [project, crashing_create, &default_update/2, Ets]})
+
+      assert_eventually alive?()
+
+      on_exit(fn ->
+        after_each_test(Ets, project)
+      end)
+
+      {:ok, project: project}
+    end
+
+    test "store survives when async load task crashes" do
+      pid = Process.whereis(Store)
+      Store.enable()
+      # Allow time for the task to crash and the DOWN message to be processed.
+      Process.sleep(100)
+
+      assert Process.alive?(pid)
+    end
+  end
 end