Add failed status and retry the job

Author: rosle

lib/google_crawler/search.ex

@@ -72,8 +72,7 @@ defmodule GoogleCrawler.Search do
   def create_and_search_keyword(attrs \\ %{}, user) do
     case create_keyword(attrs, user) do
       {:ok, %Keyword{} = keyword} ->
-        search_task = fn -> GoogleCrawler.Search.SearchKeywordTask.perform(keyword) end
-        Task.Supervisor.start_child(GoogleCrawler.TaskSupervisor, search_task)
+        GoogleCrawler.Task.perform(GoogleCrawler.Search.SearchKeywordTask, keyword)
         {:ok, %Keyword{}}
 
       {:error, %Ecto.Changeset{} = changeset} ->

lib/google_crawler/search/keyword.ex

@@ -1,6 +1,6 @@
 import EctoEnum
 
-defenum(GoogleCrawler.Search.Keyword.Status, in_queue: 0, in_progress: 1, completed: 2)
+defenum(GoogleCrawler.Search.Keyword.Status, in_queue: 0, in_progress: 1, completed: 2, failed: 3)
 
 defmodule GoogleCrawler.Search.Keyword do
   use Ecto.Schema

lib/google_crawler/search/page_fetcher.ex

@@ -2,7 +2,8 @@ defmodule GoogleCrawler.Search.PageFetcher do
   @url "https://www.google.com/search?q="
 
   def fetch(keyword) do
-    case HTTPoison.get(@url <> keyword) do
+    IO.puts "Performing search ... #{@url <> URI.encode(keyword)}"
+    case HTTPoison.get(@url <> URI.encode(keyword)) do
       {:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
         {:ok, body}
 

lib/google_crawler/search/search_keyword_task.ex

@@ -5,22 +5,19 @@ defmodule GoogleCrawler.Search.SearchKeywordTask do
   alias GoogleCrawler.Search.PageScrapper
 
   def perform(%Keyword{} = keyword) do
-    # Update the status to in progress
     Search.update_keyword(keyword, %{status: :in_progress})
 
     case PageFetcher.fetch(keyword.keyword) do
       {:ok, body} ->
-        # Store the result to the db
         result = PageScrapper.scrap(body)
-        IO.inspect result
         Search.update_keyword(keyword, %{
           status: :completed,
           raw_html_result: result.raw_html_result
         })
 
       {:error, reason} ->
-        # TODO: Retry the task
-        IO.inspect(reason)
+        Search.update_keyword(keyword, %{ status: :failed })
+        raise "Keyword search failed: #{reason}"
     end
   end
 end

lib/google_crawler/task.ex

@@ -0,0 +1,7 @@
+defmodule GoogleCrawler.Task do
+  def perform(task, args) do
+    Task.Supervisor.start_child(GoogleCrawler.TaskSupervisor, fn ->
+      task.perform(args)
+    end, restart: :transient)
+  end
+end