diff --git a/apps/engine/lib/engine/application.ex b/apps/engine/lib/engine/application.ex index 014aaf60..2dfb0a70 100644 --- a/apps/engine/lib/engine/application.ex +++ b/apps/engine/lib/engine/application.ex @@ -17,6 +17,7 @@ defmodule Engine.Application do Engine.Build.CaptureServer, Engine.Plugin.Runner.Supervisor, Engine.Plugin.Runner.Coordinator, + {Task.Supervisor, name: Engine.TaskSupervisor}, Engine.Search.Store.Backends.Ets, {Engine.Search.Store, [ diff --git a/apps/engine/lib/engine/search/store.ex b/apps/engine/lib/engine/search/store.ex index a490fefe..e19412a8 100644 --- a/apps/engine/lib/engine/search/store.ex +++ b/apps/engine/lib/engine/search/store.ex @@ -154,9 +154,18 @@ defmodule Engine.Search.Store do # handle the result from `State.async_load/1` def handle_info({ref, result}, {update_ref, %State{async_load_ref: ref} = state}) do + Process.demonitor(ref, [:flush]) {:noreply, {update_ref, State.async_load_complete(state, result)}} end + def handle_info( + {:DOWN, ref, :process, _pid, reason}, + {update_ref, %State{async_load_ref: ref} = state} + ) do + Logger.error("Search index async load crashed: #{inspect(reason)}") + {:noreply, {update_ref, %State{state | async_load_ref: nil}}} + end + def handle_info(:flush_updates, {_, %State{} = state}) do {:ok, state} = State.flush_buffered_updates(state) ref = schedule_flush() diff --git a/apps/engine/lib/engine/search/store/state.ex b/apps/engine/lib/engine/search/store/state.ex index db9f7c38..84a378f3 100644 --- a/apps/engine/lib/engine/search/store/state.ex +++ b/apps/engine/lib/engine/search/store/state.ex @@ -234,7 +234,7 @@ defmodule Engine.Search.Store.State do defp prepare_backend_async(%__MODULE__{async_load_ref: nil} = state, backend_result) do task = - Task.async(fn -> + Task.Supervisor.async_nolink(Engine.TaskSupervisor, fn -> case state.backend.prepare(backend_result) do {:ok, :empty} -> Logger.info("backend reports empty") diff --git a/apps/engine/test/engine/code_intelligence/references_test.exs b/apps/engine/test/engine/code_intelligence/references_test.exs index 9e9ba968..728bf49e 100644 --- a/apps/engine/test/engine/code_intelligence/references_test.exs +++ b/apps/engine/test/engine/code_intelligence/references_test.exs @@ -20,6 +20,7 @@ defmodule Engine.CodeIntelligence.ReferencesTest do Backends.Ets.destroy_all(project) Engine.set_project(project) + start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor}) start_supervised!(Document.Store) start_supervised!(Engine.Dispatch) start_supervised!(Backends.Ets) diff --git a/apps/engine/test/engine/dispatch/handlers/indexer_test.exs b/apps/engine/test/engine/dispatch/handlers/indexer_test.exs index 74cfa8e7..baff56cf 100644 --- a/apps/engine/test/engine/dispatch/handlers/indexer_test.exs +++ b/apps/engine/test/engine/dispatch/handlers/indexer_test.exs @@ -28,6 +28,7 @@ defmodule Engine.Dispatch.Handlers.IndexingTest do patch(Engine.Dispatch, :erpc_cast, fn Expert.Progress, _function, _args -> true end) + start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor}) start_supervised!(Engine.Dispatch) start_supervised!(Commands.Reindex) start_supervised!(Search.Store.Backends.Ets) diff --git a/apps/engine/test/engine/search/store/backends/ets_test.exs b/apps/engine/test/engine/search/store/backends/ets_test.exs index 6c313248..d2ee68c0 100644 --- a/apps/engine/test/engine/search/store/backends/ets_test.exs +++ b/apps/engine/test/engine/search/store/backends/ets_test.exs @@ -61,6 +61,7 @@ defmodule Engine.Search.Store.Backend.EtsTest do end defp start_supervised_store(%Project{} = project, create_fn, update_fn, backend) do + start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor}) start_supervised!(Dispatch) start_supervised!(Backends.Ets) start_supervised!({Store, [project, create_fn, update_fn, backend]}) diff --git a/apps/engine/test/engine/search/store_test.exs b/apps/engine/test/engine/search/store_test.exs index 966b2ddd..4f19e846 100644 --- a/apps/engine/test/engine/search/store_test.exs +++ b/apps/engine/test/engine/search/store_test.exs @@ -368,6 +368,7 @@ defmodule Engine.Search.StoreTest do defp with_a_started_store(project, backend) do destroy_backend(backend, project) + start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor}) start_supervised!(Dispatch) start_supervised!(backend) start_supervised!({Store, [project, &default_create/1, &default_update/2, backend]}) @@ -400,6 +401,7 @@ defmodule Engine.Search.StoreTest do setup %{project: project} do destroy_backend(Ets, project) + start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor}) start_supervised!(Dispatch) start_supervised!(Ets) @@ -458,4 +460,37 @@ defmodule Engine.Search.StoreTest do assert received_project == project end end + + describe "async load crash recovery" do + setup %{project: project} do + destroy_backend(Ets, project) + + start_supervised!({Task.Supervisor, name: Engine.TaskSupervisor}) + start_supervised!(Dispatch) + start_supervised!(Ets) + + crashing_create = fn _project -> + raise "index build exploded" + end + + start_supervised!({Store, [project, crashing_create, &default_update/2, Ets]}) + + assert_eventually alive?() + + on_exit(fn -> + after_each_test(Ets, project) + end) + + {:ok, project: project} + end + + test "store survives when async load task crashes" do + pid = Process.whereis(Store) + Store.enable() + # Allow time for the task to crash and the DOWN message to be processed. + Process.sleep(100) + + assert Process.alive?(pid) + end + end end