[#21] [Backend] As a system, I can scrap data from Google search result page

Gemfile

@@ -16,6 +16,7 @@ gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] # Windows doe
 # gem 'kredis' # Use Kredis to get higher-level data types in Redis
 # gem 'bcrypt' # Use Active Model has_secure_password
 gem 'devise' # Flexible authentication solution for Rails with Warden
+gem 'httparty' # A HTTP client for Ruby.
 
 # Authentications & Authorizations
 gem 'pundit' # Minimal authorization through OO design and pure Ruby classes

Gemfile.lock

@@ -187,6 +187,9 @@ GEM
     globalid (1.1.0)
       activesupport (>= 5.0)
     hashdiff (1.0.1)
+    httparty (0.21.0)
+      mini_mime (>= 1.0.0)
+      multi_xml (>= 0.5.2)
     i18n (1.13.0)
       concurrent-ruby (~> 1.0)
     i18n-js (3.9.0)
@@ -226,6 +229,7 @@ GEM
     mini_mime (1.1.2)
     minitest (5.18.0)
     msgpack (1.7.1)
+    multi_xml (0.6.0)
     nap (1.1.0)
     net-imap (0.3.4)
       date
@@ -468,6 +472,7 @@ DEPENDENCIES
   ffaker
   figaro
   foreman
+  httparty
   i18n-js (= 3.9.0)
   jsbundling-rails
   json_matchers

app/jobs/google/search_keyword_job.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Google
+  class ClientServiceError < StandardError; end
+
+  class SearchKeywordJob < ApplicationJob
+    queue_as :default
+
+    def perform(search_stat_id:)
+      search_stat = SearchStat.find search_stat_id
+      html_result = Google::ClientService.new(keyword: search_stat.keyword).call
+      parsed_attributes = ParserService.new(html_response: html_result).call
+
+      update_search_stat(search_stat, parsed_attributes)
+    rescue ActiveRecord::RecordNotFound, ClientServiceError, ArgumentError, ActiveRecord::RecordInvalid
+      update_search_stat_status search_stat, :failed
+    end
+
+    def update_search_stat(search_stat, attributes)
+      SearchStat.transaction do
+        search_stat.result_links.create(attributes[:result_links])
+
+        search_stat.update! attributes.except(:result_links)
+      end
+    end
+
+    def update_search_stat_status(search_stat, status)
+      search_stat.update! status: status
+    end
+  end
+end

app/services/google/client_service.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Google
+  class ClientService
+    USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '\
+                 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36'
+
+    BASE_SEARCH_URL = 'https://www.google.com/search'
+
+    SUCCESS_STATUS_CODE = '200'
+
+    def initialize(keyword:, lang: 'en')
+      @escaped_keyword = CGI.escape(keyword)
+      @uri = URI("#{BASE_SEARCH_URL}?q=#{@escaped_keyword}&hl=#{lang}&gl=#{lang}")
+    end
+
+    def call
+      result = HTTParty.get(@uri, { headers: { 'User-Agent' => USER_AGENT } })
+
+      raise ClientServiceError unless valid_result? result
+
+      result
+    rescue HTTParty::Error, Timeout::Error, SocketError, ClientServiceError => e
+      Rails.logger.error "Error: Query Google with '#{@escaped_keyword}' thrown an error: #{e}"
+
+      raise ClientServiceError, 'Error fetching HTML result'
+    end
+
+    private
+
+    def valid_result?(result)
+      return false unless result
+      return true if result.response.code == SUCCESS_STATUS_CODE
+    end
+  end
+end

app/services/google/parser_service.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Google
+  class ParserService
+    NON_ADS_RESULT_SELECTOR = 'a[data-ved]:not([role]):not([jsaction]):not(.adwords):not(.footer-links)'
+    AD_CONTAINER_ID = 'tads'
+    ADWORDS_CLASS = 'adwords'
+
+    def initialize(html_response:)
+      @html = html_response
+
+      @document = Nokogiri::HTML.parse(html_response) if html_response.body
+    end
+
+    # Parse html data and return a hash with the results
+    def call
+      return unless valid?
+
+      mark_adword_links
+      mark_footer_links
+
+      present_parsed_data
+    end
+
+    private
+
+    attr_reader :html, :document
+
+    def valid?
+      html.present? && document.present?
+    end
+
+    def mark_adword_links
+      # Add a class to all AdWords link for easier manipulation
+      document.css('div[data-text-ad] a[data-ved]').add_class(ADWORDS_CLASS)
+    end
+
+    def mark_footer_links
+      # Mark footer links to identify them
+      document.css('#footcnt a').add_class('footer-links')
+    end
+
+    def present_parsed_data
+      {
+        top_ad_count: ads_top_count,
+        ad_count: ads_page_count,
+        non_ad_count: non_ads_result_count,
+        total_result_count: total_link_count,
+        raw_response: html,
+        result_links: result_links,
+        status: :completed
+      }
+    end
+
+    def ads_top_count
+      document.css("##{AD_CONTAINER_ID} .#{ADWORDS_CLASS}").count
+    end
+
+    def ads_page_count
+      document.css(".#{ADWORDS_CLASS}").count
+    end
+
+    def ads_top_urls
+      document.css("##{AD_CONTAINER_ID} .#{ADWORDS_CLASS}").filter_map { |a_tag| a_tag['href'].presence }
+    end
+
+    def ads_page_urls
+      document.css(".#{ADWORDS_CLASS}").filter_map { |a_tag| a_tag['href'].presence }
+    end
+
+    def non_ads_result_count
+      document.css(NON_ADS_RESULT_SELECTOR).count { |a_tag| a_tag['href'].presence }
+    end
+
+    def non_ads_urls
+      document.css(NON_ADS_RESULT_SELECTOR).filter_map { |a_tag| a_tag['href'].presence }
+    end
+
+    def total_link_count
+      document.css('a').count
+    end
+
+    def result_links
+      results = result_link_map(ads_top_urls, :ads_top)
+      results += result_link_map(non_ads_urls, :non_ads)
+
+      results
+    end
+
+    def result_link_map(urls, type)
+      urls.map { |url| { url: url, link_type: type } }
+    end
+  end
+end

db/seeds.rb

@@ -15,5 +15,5 @@
 end
 
 10.times do
-  Fabricate.times(100, :search_stat, user: user)
+  Fabricate.times(100, :search_stat_parsed_with_links, user: user)
 end

spec/fabricators/search_stat_fabricator.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-demo_user = User.create(email: 'user@demo.com', password: 'Secret@11')
-
-Fabricator(:search_stat) do
+Fabricator(:search_stat, class_name: SearchStat) do
   keyword { FFaker::Lorem.word }
   ad_count { rand(1..10) }
   link_count { rand(1..60) }
@@ -11,5 +9,9 @@
   top_ad_count { rand(1..5) }
   status { rand(1..3) }
   raw_response { FFaker::HTMLIpsum.body }
-  user_id { demo_user.id }
+  user { Fabricate(:user) }
+end
+
+Fabricator(:search_stat_parsed_with_links, from: :search_stat) do
+  result_links(count: FFaker.rand(10) + 1)
 end