Add genserver to process the keyword

Author: rosle

lib/google_crawler/application.ex

@@ -13,7 +13,8 @@ defmodule GoogleCrawler.Application do
       # Start the endpoint when the application starts
       GoogleCrawlerWeb.Endpoint,
       # The supervisor for crawler background job
-      {Task.Supervisor, name: GoogleCrawler.TaskSupervisor}
+      {Task.Supervisor, name: GoogleCrawler.TaskSupervisor},
+      GoogleCrawler.FetchKeywordWorker
     ]
 
     # See https://hexdocs.pm/elixir/Supervisor.html

lib/google_crawler/search.ex

@@ -72,7 +72,7 @@ defmodule GoogleCrawler.Search do
   def create_and_search_keyword(attrs \\ %{}, user) do
     case create_keyword(attrs, user) do
       {:ok, %Keyword{} = keyword} ->
-        GoogleCrawler.Task.perform(GoogleCrawler.Search.SearchKeywordTask, keyword)
+        GoogleCrawler.FetchKeywordWorker.fetch_and_scrap(keyword.id)
         {:ok, %Keyword{}}
 
       {:error, %Ecto.Changeset{} = changeset} ->

lib/google_crawler/search/fetch_keyword_worker.ex

@@ -0,0 +1,81 @@
+defmodule GoogleCrawler.FetchKeywordWorker do
+  use GenServer
+
+  alias GoogleCrawler.Search
+  alias GoogleCrawler.Search.Keyword
+
+  @max_retry_count 3
+
+  # Client
+
+  def start_link(_args) do
+    GenServer.start_link(__MODULE__, %{}, name: __MODULE__)
+  end
+
+  def fetch_and_scrap(keyword_id) do
+    GenServer.call(__MODULE__, {:fetch_and_scrap, keyword_id})
+  end
+
+  # Server Callbacks
+
+  def init(state) do
+    {:ok, state}
+  end
+
+  def handle_call({:fetch_and_scrap, keyword_id}, _from, state) do
+    IO.puts "Handle call #{keyword_id}"
+
+    keyword = Search.get_keyword(keyword_id)
+    Search.update_keyword(keyword, %{status: :in_progress})
+
+    # start task and store the state of the task with the retry count
+    # as a tuple of {keyword, retry_count} -> {%Keyword{}, 1}
+    task = start_task(keyword)
+    new_state = Map.put(state, task.ref, {keyword, 0})
+
+    {:reply, :ok, new_state}
+  end
+
+  def handle_info({ref, result}, state) do
+    IO.puts "Handle info | success"
+
+    {keyword, _retry_count} = Map.get(state, ref)
+    Search.update_keyword(keyword, %{
+      status: :completed,
+      raw_html_result: result.raw_html_result
+    })
+
+    # Demonitor the task and remove from the state
+    Process.demonitor(ref, [:flush])
+    new_state = Map.delete(state, ref)
+
+    {:noreply, new_state}
+  end
+
+  def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
+    IO.puts "Handle info | failed"
+
+    {keyword, retry_count} = Map.get(state, ref)
+    new_state = if retry_count < @max_retry_count do
+                  IO.puts "Retry... #{retry_count}"
+                  task = start_task(keyword)
+
+                  state
+                  |> Map.delete(ref)
+                  |> Map.put(task.ref, {keyword, retry_count + 1})
+                else
+                  IO.puts "Done with failed..."
+                  Search.update_keyword(keyword, %{status: :failed})
+
+                  Map.delete(state, ref)
+                end
+
+    {:noreply, new_state}
+  end
+
+  defp start_task(%Keyword{} = keyword) do
+    Task.Supervisor.async_nolink(GoogleCrawler.TaskSupervisor, fn ->
+      GoogleCrawler.Search.SearchKeywordTask.perform(keyword)
+    end)
+  end
+end

lib/google_crawler/search/keyword.ex

@@ -1,6 +1,6 @@
 import EctoEnum
 
-defenum(GoogleCrawler.Search.Keyword.Status, in_queue: 0, in_progress: 1, completed: 2, failed: 3)
+defenum(GoogleCrawler.Search.Keyword.Status, in_queue: 0, in_progress: 1, failed: 2, completed: 3)
 
 defmodule GoogleCrawler.Search.Keyword do
   use Ecto.Schema

lib/google_crawler/search/page_fetcher.ex

@@ -2,7 +2,6 @@ defmodule GoogleCrawler.Search.PageFetcher do
   @url "https://www.google.com/search?q="
 
   def fetch(keyword) do
-    IO.puts "Performing search ... #{@url <> URI.encode(keyword)}"
     case HTTPoison.get(@url <> URI.encode(keyword)) do
       {:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
         {:ok, body}

lib/google_crawler/search/search_keyword_task.ex

@@ -1,22 +1,14 @@
 defmodule GoogleCrawler.Search.SearchKeywordTask do
-  alias GoogleCrawler.Search
   alias GoogleCrawler.Search.Keyword
   alias GoogleCrawler.Search.PageFetcher
   alias GoogleCrawler.Search.PageScrapper
 
   def perform(%Keyword{} = keyword) do
-    Search.update_keyword(keyword, %{status: :in_progress})
-
     case PageFetcher.fetch(keyword.keyword) do
       {:ok, body} ->
-        result = PageScrapper.scrap(body)
-        Search.update_keyword(keyword, %{
-          status: :completed,
-          raw_html_result: result.raw_html_result
-        })
+        PageScrapper.scrap(body)
 
       {:error, reason} ->
-        Search.update_keyword(keyword, %{ status: :failed })
         raise "Keyword search failed: #{reason}"
     end
   end

test/google_crawler/search_test.exs

@@ -30,7 +30,7 @@ defmodule GoogleCrawler.SearchTest do
 
       assert {:ok, %Keyword{} = keyword} = Search.create_keyword(keyword_attrs, user)
       assert keyword.keyword == "elixir"
-      assert keyword.status == :in_queue
+      assert keyword.status == :created
       assert keyword.user_id == user.id
     end
 
@@ -45,9 +45,20 @@ defmodule GoogleCrawler.SearchTest do
       keyword = KeywordFactory.create()
       keyword_attrs = %{keyword: "new", status: :in_progress}
 
-      assert {:ok, %Keyword{} = keyword} = Search.update_keyword(keyword, keyword_attrs)
-      assert keyword.keyword == "new"
-      assert keyword.status == :in_progress
+      # assert {:ok, %Keyword{} = keyword} = Search.update_keyword(keyword, keyword_attrs)
+      # assert keyword.keyword == "new"
+      # assert keyword.status == :in_progress
+
+      Search.update_keyword(keyword, %{status: :in_progress})
+      Search.update_keyword(keyword, %{status: :failed})
+      Search.update_keyword(keyword, %{status: :in_progress})
+      Search.update_keyword(keyword, %{status: :failed})
+      Search.update_keyword(keyword, %{status: :in_progress})
+      Search.update_keyword(keyword, %{status: :failed})
+      Search.update_keyword(keyword, %{status: :in_progress})
+      Search.update_keyword(keyword, %{status: :completed})
+
+      IO.inspect Search.get_keyword(keyword.id)
     end
 
     test "update_keyword/2 with invalid data returns error changeset" do