From 35a845e427616d8d9d6626b860d2145f080c9686 Mon Sep 17 00:00:00 2001 From: Alexey Kirkov Date: Tue, 2 Apr 2019 18:13:16 +0400 Subject: [PATCH 1/2] Added optimization for rake task --- Gemfile | 3 ++ Gemfile.lock | 5 +++ bin/feedback-loop.rb | 28 +++++++++++++++ config/application.rb | 2 ++ config/routes.rb | 1 + config/skylight.yml | 3 ++ db/schema.rb | 1 + lib/tasks/utils.rake | 80 +++++++++++++++++++++++++++++++++---------- 8 files changed, 104 insertions(+), 19 deletions(-) create mode 100755 bin/feedback-loop.rb create mode 100644 config/skylight.yml diff --git a/Gemfile b/Gemfile index 33017fd..7ad5d95 100644 --- a/Gemfile +++ b/Gemfile @@ -24,3 +24,6 @@ end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] +# gem "skylight", "~> 3.1" +gem 'pghero' +gem 'oj' diff --git a/Gemfile.lock b/Gemfile.lock index eb22e16..f04b21a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -76,7 +76,10 @@ GEM nio4r (2.3.1) nokogiri (1.10.2) mini_portile2 (~> 2.4.0) + oj (3.7.11) pg (1.1.4) + pghero (2.2.0) + activerecord puma (3.12.1) rack (2.0.6) rack-test (1.1.0) @@ -137,7 +140,9 @@ DEPENDENCIES bootsnap (>= 1.1.0) byebug listen (>= 3.0.5, < 3.2) + oj pg (>= 0.18, < 2.0) + pghero puma (~> 3.11) rails (~> 5.2.3) tzinfo-data diff --git a/bin/feedback-loop.rb b/bin/feedback-loop.rb new file mode 100755 index 0000000..117d88d --- /dev/null +++ b/bin/feedback-loop.rb @@ -0,0 +1,28 @@ +#!/usr/bin/env ruby +require 'benchmark' +require 'fileutils' +include FileUtils + +FILES = %w[ + example.json + small.json + medium.json + large.json +].freeze + +APP_ROOT = File.expand_path('..', __dir__) + +def system!(*args) + system(*args) || abort("\n== Command #{args} failed ==") +end + +chdir APP_ROOT do + FILES.each do |file| + result = Benchmark.measure do + puts "\n== Loading data from fixtures/#{file} ==" + system! "bin/rake reload_json[fixtures/#{file}]" + end + + puts result + end +end diff --git a/config/application.rb b/config/application.rb index 9c33109..683f75c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -15,5 +15,7 @@ class Application < Rails::Application # Application configuration can go into files in config/initializers # -- all .rb files in that directory are automatically loaded after loading # the framework and any gems in your application. + + # config.skylight.environments += ["development"] end end diff --git a/config/routes.rb b/config/routes.rb index a2da6a7..cc2d640 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,6 @@ Rails.application.routes.draw do # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html + mount PgHero::Engine, at: "pghero" get "/" => "statistics#index" get "автобусы/:from/:to" => "trips#index" end diff --git a/config/skylight.yml b/config/skylight.yml new file mode 100644 index 0000000..26f1097 --- /dev/null +++ b/config/skylight.yml @@ -0,0 +1,3 @@ +--- +# The authentication token for the application. +authentication: DcUn7LxoVTsGHsGvnPXXA8yjoujr8NC-P0IW3m5LruA diff --git a/db/schema.rb b/db/schema.rb index f6921e4..6b32c39 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -13,6 +13,7 @@ ActiveRecord::Schema.define(version: 2019_03_30_193044) do # These are extensions that must be enabled in order to support this database + enable_extension "pg_stat_statements" enable_extension "plpgsql" create_table "buses", force: :cascade do |t| diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe87..01cff1c 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,7 +1,7 @@ -# Наивная загрузка данных из json-файла в БД -# rake reload_json[fixtures/small.json] +require 'oj' + task :reload_json, [:file_name] => :environment do |_task, args| - json = JSON.parse(File.read(args.file_name)) + json = Oj.load(File.read(args.file_name)) ActiveRecord::Base.transaction do City.delete_all @@ -10,25 +10,67 @@ task :reload_json, [:file_name] => :environment do |_task, args| Trip.delete_all ActiveRecord::Base.connection.execute('delete from buses_services;') + ActiveRecord::Base.connection.tables.each do |t| + ActiveRecord::Base.connection.reset_pk_sequence!(t) + end + + cities = [] + bus_array = [] + json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s + cities.push("('#{trip['from']}')") + cities.push("('#{trip['to']}')") + bus_array.push(trip['bus']) + end + + sql = "INSERT INTO cities (name) VALUES #{cities.uniq.join(', ')}" + ActiveRecord::Base.connection.execute(sql) + cities = City.pluck(:name, :id).to_h + + buses = [] + services = [] + + bus_array.each do |obj| + buses.push("(#{obj['number']}, '#{obj['model']}')") + obj['services'].each { |service| services.push("('#{service}')") } + end + + sql = "INSERT INTO buses (number, model) VALUES #{buses.uniq.join(', ')}" + ActiveRecord::Base.connection.execute(sql) + buses = Bus.pluck(:id, :number, :model) + + sql = "INSERT INTO services (name) VALUES #{services.uniq.join(', ')}" + ActiveRecord::Base.connection.execute(sql) + + list_services = Service.pluck(:name, :id).to_h + buses_services = [] + trips = [] + + json.each do |trip| + bus = buses.detect do |obj| + obj[1] == trip['bus']['number'] && + obj[2] == trip['bus']['model'] + end + list_services.slice(*trip['bus']['services']) + .each_value do |service_id| + buses_services.push("(#{bus[0]}, #{service_id})") end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], + trips.push( + "( + #{cities[trip['from']]}, + #{cities[trip['to']]}, + #{bus[0]}, + '#{trip['start_time']}', + #{trip['duration_minutes']}, + #{trip['price_cents']} + )" ) end + + sql = "INSERT INTO buses_services (bus_id, service_id) VALUES #{buses_services.uniq.join(', ')}" + ActiveRecord::Base.connection.execute(sql) + + sql = "INSERT INTO trips (from_id, to_id, bus_id, start_time, duration_minutes, price_cents) VALUES #{trips.join(', ')}" + ActiveRecord::Base.connection.execute(sql) end end From 1504f9364a784c02952f7c81c7081d4504ba39d0 Mon Sep 17 00:00:00 2001 From: Alexey Kirkov Date: Tue, 2 Apr 2019 19:37:14 +0400 Subject: [PATCH 2/2] Added optimization for view --- app/controllers/trips_controller.rb | 4 +++- app/views/trips/_services.html.erb | 4 +++- app/views/trips/index.html.erb | 18 +++++++++++++----- config/skylight.yml | 2 +- db/migrate/20190402073803_add_indexes.rb | 10 ++++++++++ db/schema.rb | 7 ++++++- 6 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 db/migrate/20190402073803_add_indexes.rb diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be..4b8bb7a 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,8 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + @trips = Trip.where(from: @from, to: @to) + .eager_load(bus: :services) + .order(:start_time) end end diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb index 2de639f..ad26420 100644 --- a/app/views/trips/_services.html.erb +++ b/app/views/trips/_services.html.erb @@ -1,6 +1,8 @@
  • Сервисы в автобусе:
    • diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce4..9db8729 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -7,10 +7,18 @@ <% @trips.each do |trip| %> - <%= render "delimiter" %> + ==================================================== <% end %> diff --git a/config/skylight.yml b/config/skylight.yml index 26f1097..4f91d05 100644 --- a/config/skylight.yml +++ b/config/skylight.yml @@ -1,3 +1,3 @@ --- # The authentication token for the application. -authentication: DcUn7LxoVTsGHsGvnPXXA8yjoujr8NC-P0IW3m5LruA +# authentication: DcUn7LxoVTsGHsGvnPXXA8yjoujr8NC-P0IW3m5LruA diff --git a/db/migrate/20190402073803_add_indexes.rb b/db/migrate/20190402073803_add_indexes.rb new file mode 100644 index 0000000..4fc2173 --- /dev/null +++ b/db/migrate/20190402073803_add_indexes.rb @@ -0,0 +1,10 @@ +class AddIndexes < ActiveRecord::Migration[5.2] + def change + + add_index :cities, %i[id name] + add_index :trips, %i[id from_id to_id] + add_index :buses, %i[id number model] + add_index :buses_services, %i[id bus_id service_id] + add_index :services, %i[id name] + end +end diff --git a/db/schema.rb b/db/schema.rb index 6b32c39..01db706 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2019_04_02_073803) do # These are extensions that must be enabled in order to support this database enable_extension "pg_stat_statements" @@ -19,19 +19,23 @@ create_table "buses", force: :cascade do |t| t.string "number" t.string "model" + t.index ["id", "number", "model"], name: "index_buses_on_id_and_number_and_model" end create_table "buses_services", force: :cascade do |t| t.integer "bus_id" t.integer "service_id" + t.index ["id", "bus_id", "service_id"], name: "index_buses_services_on_id_and_bus_id_and_service_id" end create_table "cities", force: :cascade do |t| t.string "name" + t.index ["id", "name"], name: "index_cities_on_id_and_name" end create_table "services", force: :cascade do |t| t.string "name" + t.index ["id", "name"], name: "index_services_on_id_and_name" end create_table "trips", force: :cascade do |t| @@ -41,6 +45,7 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.index ["id", "from_id", "to_id"], name: "index_trips_on_id_and_from_id_and_to_id" end end