Data import and trips#index optimizations

.rspec

@@ -0,0 +1,4 @@
+--require rails_helper
+--profile 10
+--color
+

.ruby-version

@@ -1 +1 @@
-2.6.1
+2.6.1-railsexpress

Gemfile

@@ -4,19 +4,31 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 ruby '2.6.1'
 
 gem 'rails', '~> 5.2.3'
+
 gem 'pg', '>= 0.18', '< 2.0'
+gem 'bulk_insert', '~> 1.7'
+
 gem 'puma', '~> 3.11'
 gem 'bootsnap', '>= 1.1.0', require: false
 
 group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console
   gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
+  gem 'rspec-rails', '~> 3.8'
+
+  # Profiling
+  gem 'benchmark-ips'
+  gem 'kalibera'
+  gem 'memory_profiler'
+  gem 'stackprof'
+  gem 'ruby-prof'
 end
 
 group :development do
   # Access an interactive console on exception pages or by calling 'console' anywhere in the code.
   gem 'web-console', '>= 3.3.0'
   gem 'listen', '>= 3.0.5', '< 3.2'
+  # gem 'meta_request'
 end
 
 group :test do

Gemfile.lock

@@ -43,19 +43,27 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     arel (9.0.0)
+    benchmark-ips (2.7.2)
     bindex (0.6.0)
     bootsnap (1.4.2)
       msgpack (~> 1.0)
     builder (3.2.3)
+    bulk_insert (1.7.0)
+      activerecord (>= 3.2.0)
     byebug (11.0.1)
+    callsite (0.0.11)
     concurrent-ruby (1.1.5)
     crass (1.0.4)
+    diff-lcs (1.3)
     erubi (1.8.0)
     ffi (1.10.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
     i18n (1.6.0)
       concurrent-ruby (~> 1.0)
+    kalibera (0.1)
+      memoist (~> 0.11.0)
+      rbzip2 (~> 0.2.0)
     listen (3.1.5)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
@@ -67,6 +75,12 @@ GEM
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
+    memoist (0.11.0)
+    memory_profiler (0.9.13)
+    meta_request (0.6.0)
+      callsite (~> 0.0, >= 0.0.11)
+      rack-contrib (>= 1.1, < 3)
+      railties (>= 3.0.0, < 6)
     method_source (0.9.2)
     mimemagic (0.3.3)
     mini_mime (1.0.1)
@@ -79,6 +93,8 @@ GEM
     pg (1.1.4)
     puma (3.12.1)
     rack (2.0.6)
+    rack-contrib (2.1.0)
+      rack (~> 2.0)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails (5.2.3)
@@ -109,6 +125,25 @@ GEM
     rb-fsevent (0.10.3)
     rb-inotify (0.10.0)
       ffi (~> 1.0)
+    rbzip2 (0.2.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-rails (3.8.2)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    ruby-prof (0.17.0)
     ruby_dep (1.5.0)
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
@@ -117,6 +152,7 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
+    stackprof (0.2.12)
     thor (0.20.3)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
@@ -134,12 +170,20 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  benchmark-ips
   bootsnap (>= 1.1.0)
+  bulk_insert (~> 1.7)
   byebug
+  kalibera
   listen (>= 3.0.5, < 3.2)
+  memory_profiler
+  meta_request
   pg (>= 0.18, < 2.0)
   puma (~> 3.11)
   rails (~> 5.2.3)
+  rspec-rails (~> 3.8)
+  ruby-prof
+  stackprof
   tzinfo-data
   web-console (>= 3.3.0)
 

app/controllers/trips_controller.rb

@@ -2,6 +2,10 @@ class TripsController < ApplicationController
   def index
     @from = City.find_by_name!(params[:from])
     @to = City.find_by_name!(params[:to])
-    @trips = Trip.where(from: @from, to: @to).order(:start_time)
+
+    @trips = Trip
+      .eager_load(bus: :services)
+      .where(from: @from, to: @to)
+      .order(:start_time)
   end
 end

app/helpers/application_helper.rb

@@ -1,2 +1,5 @@
 module ApplicationHelper
+  def render_delimiter
+    "===================================================="
+  end
 end

app/helpers/trips_helper.rb

@@ -0,0 +1,16 @@
+module TripsHelper
+  def render_service(service)
+    "<li>#{service.name}</li>".html_safe
+  end
+
+  def render_services(services)
+    services.map do |service|
+      <<~HTML
+        <li>Сервисы в автобусе:</li>
+        <ul>
+          #{render_service(service)}
+        </ul>
+      HTML
+    end.join('').html_safe
+  end
+end

app/models/bus.rb

@@ -13,7 +13,8 @@ class Bus < ApplicationRecord
   ].freeze
 
   has_many :trips
-  has_and_belongs_to_many :services, join_table: :buses_services
+  has_many :bus_services
+  has_many :services, through: :bus_services
 
   validates :number, presence: true, uniqueness: true
   validates :model, inclusion: { in: MODELS }

app/models/bus_service.rb

@@ -0,0 +1,6 @@
+class BusService < ApplicationRecord
+  self.table_name = 'buses_services'
+
+  belongs_to :bus
+  belongs_to :service
+end

app/models/service.rb

@@ -12,7 +12,8 @@ class Service < ApplicationRecord
     'Можно не печатать билет',
   ].freeze
 
-  has_and_belongs_to_many :buses, join_table: :buses_services
+  has_many :bus_services
+  has_many :buses, through: :bus_services
 
   validates :name, presence: true
   validates :name, inclusion: { in: SERVICES }

app/services/db_populator.rb

@@ -0,0 +1,52 @@
+class DbPopulator
+  class << self
+    def populate(file_name)
+      json = JSON.parse(File.read(file_name))
+
+      all_services = {}
+      all_buses = {}
+      all_cities = {}
+
+      ActiveRecord::Base.transaction do
+        City.delete_all
+        BusService.delete_all
+        Bus.delete_all
+        Service.delete_all
+        Trip.delete_all
+
+        Trip.bulk_insert do |trip_worker|
+          BusService.bulk_insert do |bus_service_worker|
+            json.each do |trip|
+              from = (all_cities[trip['from']] ||= City.create!(name: trip['from']))
+              to = (all_cities[trip['to']] ||= City.create!(name: trip['to']))
+
+              services = []
+              trip['bus']['services'].each do |service|
+                s = (all_services[service] ||= Service.create!(name: service))
+                services << s
+              end
+
+              number = trip['bus']['number']
+              bus = (all_buses[number] ||= begin
+                Bus.create!(number: number, model: trip['bus']['model']).tap do |bus|
+                  services.each do |service|
+                    bus_service_worker.add(bus_id: bus.id, service_id: service.id)
+                  end
+                end
+              end)
+
+              trip_worker.add(
+                from_id: from.id,
+                to_id: to.id,
+                bus_id: bus.id,
+                start_time: trip['start_time'],
+                duration_minutes: trip['duration_minutes'],
+                price_cents: trip['price_cents'],
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

app/views/trips/_trip.html.erb

@@ -3,3 +3,9 @@
 <li><%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %></li>
 <li><%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %></li>
 <li><%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %></li>
+
+<% if trip.bus.services.present? %>
+  <%= render_services(trip.bus.services) %>
+<% end %>
+
+<%= render_delimiter %>

app/views/trips/index.html.erb

@@ -2,15 +2,10 @@
   <%= "Автобусы #{@from.name} – #{@to.name}" %>
 </h1>
 <h2>
-  <%= "В расписании #{@trips.count} рейсов" %>
+  <%= "В расписании #{@trips.length} рейсов" %>
 </h2>
 
-<% @trips.each do |trip| %>
-  <ul>
-    <%= render "trip", trip: trip %>
-    <% if trip.bus.services.present? %>
-      <%= render "services", services: trip.bus.services %>
-    <% end %>
-  </ul>
-  <%= render "delimiter" %>
-<% end %>
+
+<ul>
+  <%= render partial: "trip", collection: @trips, cached: true %>
+</ul>

config/environments/test.rb

@@ -12,6 +12,11 @@
   # preloads Rails for running tests, you may have to set it to true.
   config.eager_load = false
 
+  if ENV['ENABLE_CACHE'] == 'true'
+    puts 'Caching enabled!'
+    config.cache_store = :memory_store
+  end
+
   # Configure public file server for tests with Cache-Control for performance.
   config.public_file_server.enabled = true
   config.public_file_server.headers = {

db/migrate/20190405172042_add_indexes.rb

@@ -0,0 +1,9 @@
+class AddIndexes < ActiveRecord::Migration[5.2]
+  def change
+    add_index :buses_services, :bus_id
+    add_index :buses_services, :service_id
+    add_index :cities, :name
+    add_index :trips, :bus_id
+    add_index :trips, [:from_id, :to_id]
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_03_30_193044) do
+ActiveRecord::Schema.define(version: 2019_04_05_172042) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -23,10 +23,13 @@
   create_table "buses_services", force: :cascade do |t|
     t.integer "bus_id"
     t.integer "service_id"
+    t.index ["bus_id"], name: "index_buses_services_on_bus_id"
+    t.index ["service_id"], name: "index_buses_services_on_service_id"
   end
 
   create_table "cities", force: :cascade do |t|
     t.string "name"
+    t.index ["name"], name: "index_cities_on_name"
   end
 
   create_table "services", force: :cascade do |t|
@@ -40,6 +43,8 @@
     t.integer "duration_minutes"
     t.integer "price_cents"
     t.integer "bus_id"
+    t.index ["bus_id"], name: "index_trips_on_bus_id"
+    t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id"
   end
 
 end