Db optimization task 4

.gitignore

@@ -25,3 +25,4 @@
 
 # Ignore master key for decrypting credentials and more.
 /config/master.key
+.env

.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--require rails_helper

Gemfile

@@ -7,10 +7,19 @@ gem 'rails', '~> 5.2.3'
 gem 'pg', '>= 0.18', '< 2.0'
 gem 'puma', '~> 3.11'
 gem 'bootsnap', '>= 1.1.0', require: false
+gem 'oj'
+gem 'activerecord-import'
+gem 'newrelic_rpm'
+gem 'pghero'
+gem 'pg_query', '>= 0.9.0'
+gem 'ruby-prof'
+gem 'strong_migrations'
 
 group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console
   gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
+  gem 'meta_request'
+  gem 'dotenv-rails'
 end
 
 group :development do
@@ -20,6 +29,8 @@ group :development do
 end
 
 group :test do
+  gem 'rspec-rails', '~> 3.8'
+  gem 'rails-controller-testing'
 end
 
 # Windows does not include zoneinfo files, so bundle the tzinfo-data gem

Gemfile.lock

@@ -33,6 +33,8 @@ GEM
       activemodel (= 5.2.3)
       activesupport (= 5.2.3)
       arel (>= 9.0)
+    activerecord-import (1.0.1)
+      activerecord (>= 3.2)
     activestorage (5.2.3)
       actionpack (= 5.2.3)
       activerecord (= 5.2.3)
@@ -43,13 +45,19 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     arel (9.0.0)
-    bindex (0.6.0)
-    bootsnap (1.4.2)
+    bindex (0.7.0)
+    bootsnap (1.4.4)
       msgpack (~> 1.0)
     builder (3.2.3)
     byebug (11.0.1)
+    callsite (0.0.11)
     concurrent-ruby (1.1.5)
     crass (1.0.4)
+    diff-lcs (1.3)
+    dotenv (2.7.2)
+    dotenv-rails (2.7.2)
+      dotenv (= 2.7.2)
+      railties (>= 3.2, < 6.1)
     erubi (1.8.0)
     ffi (1.10.0)
     globalid (0.4.2)
@@ -67,18 +75,29 @@ GEM
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
+    meta_request (0.7.0)
+      callsite (~> 0.0, >= 0.0.11)
+      rack-contrib (>= 1.1, < 3)
+      railties (>= 3.0.0, < 7)
     method_source (0.9.2)
     mimemagic (0.3.3)
     mini_mime (1.0.1)
     mini_portile2 (2.4.0)
     minitest (5.11.3)
-    msgpack (1.2.9)
+    msgpack (1.2.10)
+    newrelic_rpm (6.3.0.355)
     nio4r (2.3.1)
-    nokogiri (1.10.2)
+    nokogiri (1.10.3)
       mini_portile2 (~> 2.4.0)
+    oj (3.7.12)
     pg (1.1.4)
+    pg_query (1.1.0)
+    pghero (2.2.0)
+      activerecord
     puma (3.12.1)
-    rack (2.0.6)
+    rack (2.0.7)
+    rack-contrib (2.1.0)
+      rack (~> 2.0)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails (5.2.3)
@@ -94,6 +113,10 @@ GEM
       bundler (>= 1.3.0)
       railties (= 5.2.3)
       sprockets-rails (>= 2.0.0)
+    rails-controller-testing (1.0.4)
+      actionpack (>= 5.0.1.x)
+      actionview (>= 5.0.1.x)
+      activesupport (>= 5.0.1.x)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
@@ -109,6 +132,24 @@ GEM
     rb-fsevent (0.10.3)
     rb-inotify (0.10.0)
       ffi (~> 1.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-rails (3.8.2)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    ruby-prof (0.17.0)
     ruby_dep (1.5.0)
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
@@ -117,6 +158,8 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
+    strong_migrations (0.3.1)
+      activerecord (>= 3.2.0)
     thor (0.20.3)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
@@ -134,12 +177,23 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activerecord-import
   bootsnap (>= 1.1.0)
   byebug
+  dotenv-rails
   listen (>= 3.0.5, < 3.2)
+  meta_request
+  newrelic_rpm
+  oj
   pg (>= 0.18, < 2.0)
+  pg_query (>= 0.9.0)
+  pghero
   puma (~> 3.11)
   rails (~> 5.2.3)
+  rails-controller-testing
+  rspec-rails (~> 3.8)
+  ruby-prof
+  strong_migrations
   tzinfo-data
   web-console (>= 3.3.0)
 

app/models/bus.rb

@@ -12,8 +12,10 @@ class Bus < ApplicationRecord
     'Газель',
   ].freeze
 
-  has_many :trips
-  has_and_belongs_to_many :services, join_table: :buses_services
+  has_many :trips, dependent: :destroy
+  # has_and_belongs_to_many :services, join_table: :buses_services
+  has_many :buses_services, class_name: 'BusesService'
+  has_many :services, through: :buses_services
 
   validates :number, presence: true, uniqueness: true
   validates :model, inclusion: { in: MODELS }

app/models/buses_service.rb

@@ -0,0 +1,4 @@
+class BusesService < ApplicationRecord
+  belongs_to :bus
+  belongs_to :service
+end

app/models/service.rb

@@ -12,7 +12,9 @@ class Service < ApplicationRecord
     'Можно не печатать билет',
   ].freeze
 
-  has_and_belongs_to_many :buses, join_table: :buses_services
+  # has_and_belongs_to_many :buses, join_table: :buses_services
+  has_many :buses_services, class_name: 'BusesService'
+  has_many :buses, through: :buses_services
 
   validates :name, presence: true
   validates :name, inclusion: { in: SERVICES }

app/models/trip.rb

@@ -1,9 +1,9 @@
 class Trip < ApplicationRecord
   HHMM_REGEXP = /([0-1][0-9]|[2][0-3]):[0-5][0-9]/
 
-  belongs_to :from, class_name: 'City'
-  belongs_to :to, class_name: 'City'
-  belongs_to :bus
+  belongs_to :from, class_name: 'City', foreign_key: :from_id, optional: true
+  belongs_to :to, class_name: 'City', foreign_key: :to_id, optional: true
+  belongs_to :bus, foreign_key: :bus_id, optional: true
 
   validates :from, presence: true
   validates :to, presence: true

app/scripts/ruby_prof.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'ruby-prof'
+require_relative '../services/json_importer'
+
+RubyProf.measure_mode = RubyProf::WALL_TIME
+
+INITIAL_DATA_FILE = 'fixtures/small.json'
+OUTPUT_DIR = 'tmp/data'
+
+
+def flat_profile
+  run_profiler do |result|
+    printer = RubyProf::FlatPrinterWithLineNumbers.new(result)
+    printer.print(File.open("#{OUTPUT_DIR}/ruby_prof_flat_demo.txt", 'w+'))   
+  end
+end
+
+def graph_profile
+  run_profiler do |result|
+    printer = RubyProf::GraphHtmlPrinter.new(result)
+    printer.print(File.open("#{OUTPUT_DIR}/ruby_prof_graph_demo.html", "w+"))
+  end
+end
+
+def callstack_profile
+  run_profiler do |result|
+    printer = RubyProf::CallStackPrinter.new(result)
+    printer.print(File.open("#{OUTPUT_DIR}/ruby_prof_callstack_demo.html", "w+"))
+  end
+end
+
+def calltree_profile
+  run_profiler do |result|
+    printer = RubyProf::CallTreePrinter.new(result)
+    printer.print(path: OUTPUT_DIR, profile: 'profile')
+  end
+end
+
+
+def run_profiler
+  RubyProf.measure_mode = RubyProf::WALL_TIME
+  result = RubyProf.profile { JsonImporter.new.import_json_to_db(file_path: INITIAL_DATA_FILE) }
+  yield result
+end
+
+# flat_profile
+# graph_profile
+# callstack_profile
+# calltree_profile

app/services/json_importer.rb

@@ -0,0 +1,79 @@
+  require 'oj'
+
+class JsonImporter
+  attr_accessor :services_names_hash, :cities_names_hash, :buses_hash, :trips
+  def initialize
+    @services_names_hash = {}
+    @cities_names_hash = {}
+    @buses_hash = {}
+    @trips = []
+  end
+
+  def import_json_to_db(file_path:)
+    # byebug
+    json = Oj.load_file(file_path)
+
+    ActiveRecord::Base.transaction do
+      delete_existing_records 
+      create_cities_and_services(json)
+      create_buses_with_services(json)  
+      create_trips(json)
+    end
+  end
+
+  def create_cities_and_services(json)
+    json.each do |trip|
+      cities_names_hash[trip['from']] ||= City.new(name: trip['from'])
+      cities_names_hash[trip['to']] ||= City.new(name: trip['to'])
+      trip['bus']['services'].each do |service|
+        services_names_hash[service] = Service.new(name: service)
+      end
+    end 
+    # byebug
+    City.import cities_names_hash.values, syncronize: true, raise_error: true
+    # byebug
+    Service.import services_names_hash.values, syncronize: true, raise_error: true   
+  end
+
+  def create_buses_with_services(json)
+    buses_numbers = []
+    json.each do |trip|
+      next if buses_numbers.include?(trip['bus']['number'])
+        bus = Bus.new(
+          number: trip['bus']['number'],
+          model: trip['bus']['model']
+        )
+        bus.services = services_names_hash.values_at(*trip['bus']['services'])
+
+        buses_hash[trip['bus']['number']] = bus
+        buses_numbers << trip['bus']['number']
+    end
+    # byebug
+    Bus.import buses_hash.values, recursive: true, syncronize: true,raise_error: true
+  end
+
+  def create_trips(json)
+    json.each do |trip|
+      from = cities_names_hash[trip['from']]
+      to = cities_names_hash[trip['to']]
+      bus = buses_hash[trip['bus']['number']]
+      trips << Trip.new(
+        from: from,
+        to: to,
+        bus: bus,
+        start_time: trip['start_time'],
+        duration_minutes: trip['duration_minutes'],
+        price_cents: trip['price_cents']
+      )
+    end
+    Trip.import trips, raise_error: true
+  end
+
+  def delete_existing_records
+    City.delete_all
+    Bus.delete_all
+    Service.delete_all
+    Trip.delete_all
+    ActiveRecord::Base.connection.execute('delete from buses_services;')
+  end
+end

app/views/trips/_trip.html.erb

@@ -1,5 +1,16 @@
-<li><%= "Отправление: #{trip.start_time}" %></li>
-<li><%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %></li>
-<li><%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %></li>
-<li><%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %></li>
-<li><%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %></li>
+<ul>
+  <li><%= "Отправление: #{trip.start_time}" %></li>
+  <li><%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %></li>
+  <li><%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %></li>
+  <li><%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %></li>
+  <li><%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %></li>
+    <% if trip.bus.services.present? %>
+      <li>Сервисы в автобусе:</li>
+      <ul>
+        <% trip.bus.services.each do |service| %>
+          <li><%= "#{service.name}" %></li>
+        <% end %>
+      </ul>
+    <% end %>
+</ul>
+====================================================

app/views/trips/index.html.erb

@@ -2,15 +2,7 @@
   <%= "Автобусы #{@from.name} – #{@to.name}" %>
 </h1>
 <h2>
-  <%= "В расписании #{@trips.count} рейсов" %>
+  <%= "В расписании #{@trips.size} рейсов" %>
 </h2>
 
-<% @trips.each do |trip| %>
-  <ul>
-    <%= render "trip", trip: trip %>
-    <% if trip.bus.services.present? %>
-      <%= render "services", services: trip.bus.services %>
-    <% end %>
-  </ul>
-  <%= render "delimiter" %>
-<% end %>
+<%= render partial: 'trip', collection: @trips %>