Ускорение загрузки данных и отображения расписания

.rspec

@@ -0,0 +1 @@
+--require spec_helper

Gemfile

@@ -8,15 +8,25 @@ gem 'pg', '>= 0.18', '< 2.0'
 gem 'puma', '~> 3.11'
 gem 'bootsnap', '>= 1.1.0', require: false
 
+gem 'pghero'
+gem 'activerecord-import'
+
 group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console
   gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
+  gem 'rspec-rails', '~> 3.8'
 end
 
 group :development do
   # Access an interactive console on exception pages or by calling 'console' anywhere in the code.
   gem 'web-console', '>= 3.3.0'
   gem 'listen', '>= 3.0.5', '< 3.2'
+
+  gem 'rack-mini-profiler'
+
+  gem 'flamegraph'
+  gem "stackprof", "~> 0.2", require: false, platforms: :ruby
+  gem "memory_profiler", "~> 0.9"
 end
 
 group :test do

Gemfile.lock

@@ -33,6 +33,8 @@ GEM
       activemodel (= 5.2.3)
       activesupport (= 5.2.3)
       arel (>= 9.0)
+    activerecord-import (1.0.1)
+      activerecord (>= 3.2)
     activestorage (5.2.3)
       actionpack (= 5.2.3)
       activerecord (= 5.2.3)
@@ -50,8 +52,10 @@ GEM
     byebug (11.0.1)
     concurrent-ruby (1.1.5)
     crass (1.0.4)
+    diff-lcs (1.3)
     erubi (1.8.0)
     ffi (1.10.0)
+    flamegraph (0.9.5)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
     i18n (1.6.0)
@@ -67,6 +71,7 @@ GEM
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
+    memory_profiler (0.9.13)
     method_source (0.9.2)
     mimemagic (0.3.3)
     mini_mime (1.0.1)
@@ -77,8 +82,12 @@ GEM
     nokogiri (1.10.2)
       mini_portile2 (~> 2.4.0)
     pg (1.1.4)
+    pghero (2.2.0)
+      activerecord
     puma (3.12.1)
     rack (2.0.6)
+    rack-mini-profiler (1.0.2)
+      rack (>= 1.2.0)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails (5.2.3)
@@ -109,6 +118,23 @@ GEM
     rb-fsevent (0.10.3)
     rb-inotify (0.10.0)
       ffi (~> 1.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-rails (3.8.2)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
     ruby_dep (1.5.0)
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
@@ -117,6 +143,7 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
+    stackprof (0.2.12)
     thor (0.20.3)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
@@ -134,12 +161,19 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activerecord-import
   bootsnap (>= 1.1.0)
   byebug
+  flamegraph
   listen (>= 3.0.5, < 3.2)
+  memory_profiler (~> 0.9)
   pg (>= 0.18, < 2.0)
+  pghero
   puma (~> 3.11)
+  rack-mini-profiler
   rails (~> 5.2.3)
+  rspec-rails (~> 3.8)
+  stackprof (~> 0.2)
   tzinfo-data
   web-console (>= 3.3.0)
 

app/controllers/trips_controller.rb

@@ -2,6 +2,8 @@ class TripsController < ApplicationController
   def index
     @from = City.find_by_name!(params[:from])
     @to = City.find_by_name!(params[:to])
-    @trips = Trip.where(from: @from, to: @to).order(:start_time)
+    @trips = Trip.where(from: @from, to: @to)
+                 .eager_load(bus: :services)
+                 .order(:start_time)
   end
 end

app/models/bus.rb

@@ -13,7 +13,8 @@ class Bus < ApplicationRecord
   ].freeze
 
   has_many :trips
-  has_and_belongs_to_many :services, join_table: :buses_services
+  has_many :buses_services
+  has_many :services, through: :buses_services
 
   validates :number, presence: true, uniqueness: true
   validates :model, inclusion: { in: MODELS }

app/models/buses_service.rb

@@ -0,0 +1,4 @@
+class BusesService < ApplicationRecord
+  belongs_to :bus
+  belongs_to :service
+end

app/models/service.rb

@@ -12,7 +12,8 @@ class Service < ApplicationRecord
     'Можно не печатать билет',
   ].freeze
 
-  has_and_belongs_to_many :buses, join_table: :buses_services
+  has_many :buses_services
+  has_many :buses, through: :buses_services
 
   validates :name, presence: true
   validates :name, inclusion: { in: SERVICES }

app/views/trips/_services.html.erb

@@ -1,6 +1,4 @@
 <li>Сервисы в автобусе:</li>
 <ul>
-  <% services.each do |service| %>
-    <%= render "service", service: service %>
-  <% end %>
+  <%= render partial: "trips/service", collection: services, cached: true %>
 </ul>

app/views/trips/_trip.html.erb

@@ -1,5 +1,12 @@
-<li><%= "Отправление: #{trip.start_time}" %></li>
-<li><%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %></li>
-<li><%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %></li>
-<li><%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %></li>
-<li><%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %></li>
+<ul>
+  <li><%= "Отправление: #{trip.start_time}" %></li>
+  <li><%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %></li>
+  <li><%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %></li>
+  <li><%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %></li>
+  <li><%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %></li>
+
+  <% if trip.bus.services.present? %>
+    <%= render "services", services: trip.bus.services %>
+  <% end %>
+</ul>
+====================================================

app/views/trips/index.html.erb

@@ -2,15 +2,7 @@
   <%= "Автобусы #{@from.name} – #{@to.name}" %>
 </h1>
 <h2>
-  <%= "В расписании #{@trips.count} рейсов" %>
+  <%= "В расписании #{@trips.length} рейсов" %>
 </h2>
 
-<% @trips.each do |trip| %>
-  <ul>
-    <%= render "trip", trip: trip %>
-    <% if trip.bus.services.present? %>
-      <%= render "services", services: trip.bus.services %>
-    <% end %>
-  </ul>
-  <%= render "delimiter" %>
-<% end %>
+<%= render partial: @trips %>

bin/benchmark-import.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
+require 'benchmark'
+
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+
+
+chdir APP_ROOT do
+  result = Benchmark.measure do
+    puts "\n== Loading data from fixtures/small.json =="
+    system! 'bin/rake reload_json[fixtures/small.json]'
+  end
+
+  puts result
+end

config/routes.rb

@@ -1,4 +1,6 @@
 Rails.application.routes.draw do
+  mount PgHero::Engine, at: "pghero"
+
   # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html
   get "/" => "statistics#index"
   get "автобусы/:from/:to" => "trips#index"

db/migrate/20190405213225_add_indexes.rb

@@ -0,0 +1,16 @@
+class AddIndexes < ActiveRecord::Migration[5.2]
+  def change
+    add_index :cities, :name, unique: true
+    add_index :buses, :number, unique: true
+    add_index :services, :name, unique: true
+    add_index :buses_services, [:bus_id, :service_id], unique: true
+
+    change_column_null :trips, :from_id, false
+    change_column_null :trips, :to_id, false
+    change_column_null :trips, :bus_id, false
+    change_column_null :trips, :duration_minutes, false
+    change_column_null :trips, :price_cents, false
+
+    add_index :trips, [:from_id, :to_id]
+  end
+end

db/schema.rb

@@ -10,36 +10,42 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_03_30_193044) do
+ActiveRecord::Schema.define(version: 2019_04_05_213225) do
 
   # These are extensions that must be enabled in order to support this database
+  enable_extension "pg_stat_statements"
   enable_extension "plpgsql"
 
   create_table "buses", force: :cascade do |t|
     t.string "number"
     t.string "model"
+    t.index ["number"], name: "index_buses_on_number", unique: true
   end
 
   create_table "buses_services", force: :cascade do |t|
     t.integer "bus_id"
     t.integer "service_id"
+    t.index ["bus_id", "service_id"], name: "index_buses_services_on_bus_id_and_service_id", unique: true
   end
 
   create_table "cities", force: :cascade do |t|
     t.string "name"
+    t.index ["name"], name: "index_cities_on_name", unique: true
   end
 
   create_table "services", force: :cascade do |t|
     t.string "name"
+    t.index ["name"], name: "index_services_on_name", unique: true
   end
 
   create_table "trips", force: :cascade do |t|
-    t.integer "from_id"
-    t.integer "to_id"
+    t.integer "from_id", null: false
+    t.integer "to_id", null: false
     t.string "start_time"
-    t.integer "duration_minutes"
-    t.integer "price_cents"
-    t.integer "bus_id"
+    t.integer "duration_minutes", null: false
+    t.integer "price_cents", null: false
+    t.integer "bus_id", null: false
+    t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id"
   end
 
 end

lib/tasks/utils.rake

@@ -10,25 +10,47 @@ task :reload_json, [:file_name] => :environment do |_task, args|
     Trip.delete_all
     ActiveRecord::Base.connection.execute('delete from buses_services;')
 
+    Service::SERVICES.each do |name|
+      Service.create!(name: name)
+    end
+    services_list = Service.pluck(:name, :id).to_h
+
+    cities_names = []
+    buses_hash = {}
+    json.each do |trip|
+      cities_names << trip['from'] << trip['to']
+      buses_hash[trip['bus']['number']] = { model: trip['bus']['model'], services: trip['bus']['services'] }
+    end
+
+    cities_names.uniq!.map! { |a| "('#{a}')" }
+    sql = "INSERT INTO cities (name) VALUES #{cities_names.join(', ')}"
+    ActiveRecord::Base.connection.execute(sql)
+    cities_list = City.pluck(:name, :id).to_h
+
+    buses = []
+    buses_hash.each do |number, attrs|
+      buses << Bus.new(number: number, model: attrs[:model])
+    end
+    Bus.import buses
+    buses_list = Bus.pluck(:number, :id).to_h
+
+    trips = []
+    buses_services = {}
     json.each do |trip|
-      from = City.find_or_create_by(name: trip['from'])
-      to = City.find_or_create_by(name: trip['to'])
-      services = []
-      trip['bus']['services'].each do |service|
-        s = Service.find_or_create_by(name: service)
-        services << s
-      end
-      bus = Bus.find_or_create_by(number: trip['bus']['number'])
-      bus.update(model: trip['bus']['model'], services: services)
-
-      Trip.create!(
-        from: from,
-        to: to,
-        bus: bus,
-        start_time: trip['start_time'],
-        duration_minutes: trip['duration_minutes'],
-        price_cents: trip['price_cents'],
-      )
+      bus_id = buses_list[trip['bus']['number']]
+      services_ids = services_list.slice(*trip['bus']['services']).values
+      buses_services[bus_id] = services_ids.map { |s_id| { bus_id: bus_id, service_id: s_id } } unless buses_services.key?(bus_id)
+
+      trip = Trip.new(from_id: cities_list[trip['from']],
+                      to_id: cities_list[trip['to']],
+                      bus_id: buses_list[trip['bus']['number']],
+                      start_time: trip['start_time'],
+                      duration_minutes: trip['duration_minutes'],
+                      price_cents: trip['price_cents'])
+      trips << trip
     end
+    Trip.import trips
+
+    BusesService.import buses_services.values.flatten
   end
 end

lib/tasks/utils_old.rake

@@ -0,0 +1,34 @@
+# Наивная загрузка данных из json-файла в БД
+# rake reload_json[fixtures/small.json]
+task :reload_json_old, [:file_name] => :environment do |_task, args|
+  json = JSON.parse(File.read(args.file_name))
+
+  ActiveRecord::Base.transaction do
+    City.delete_all
+    Bus.delete_all
+    # Service.delete_all
+    Trip.delete_all
+    ActiveRecord::Base.connection.execute('delete from buses_services;')
+
+    json.each do |trip|
+      from = City.find_or_create_by(name: trip['from'])
+      to = City.find_or_create_by(name: trip['to'])
+      services = []
+      trip['bus']['services'].each do |service|
+        s = Service.find_or_create_by(name: service)
+        services << s
+      end
+      bus = Bus.find_or_create_by(number: trip['bus']['number'])
+      bus.update(model: trip['bus']['model'], services: services)
+
+      Trip.create!(
+        from: from,
+        to: to,
+        bus: bus,
+        start_time: trip['start_time'],
+        duration_minutes: trip['duration_minutes'],
+        price_cents: trip['price_cents'],
+      )
+    end
+  end
+end