From c30c09d89b2780dd2be0dbe7fd57ed55c9013119 Mon Sep 17 00:00:00 2001 From: Artsiom Musin Date: Fri, 12 Apr 2019 21:56:05 +0300 Subject: [PATCH 1/3] Prepare pghero and skylight --- .gitignore | 1 + Gemfile | 3 +++ Gemfile.lock | 10 ++++++++++ config/application.rb | 5 +---- config/routes.rb | 1 + 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 18b43c9..39bfdbb 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ # Ignore master key for decrypting credentials and more. /config/master.key +/config/skylight.yml diff --git a/Gemfile b/Gemfile index 33017fd..ebbb4c7 100644 --- a/Gemfile +++ b/Gemfile @@ -7,6 +7,9 @@ gem 'rails', '~> 5.2.3' gem 'pg', '>= 0.18', '< 2.0' gem 'puma', '~> 3.11' gem 'bootsnap', '>= 1.1.0', require: false +gem 'pghero' +gem 'pg_query', '>= 0.9.0' +gem 'skylight' group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console diff --git a/Gemfile.lock b/Gemfile.lock index eb22e16..322483d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -77,6 +77,9 @@ GEM nokogiri (1.10.2) mini_portile2 (~> 2.4.0) pg (1.1.4) + pg_query (1.1.0) + pghero (2.2.0) + activerecord puma (3.12.1) rack (2.0.6) rack-test (1.1.0) @@ -110,6 +113,10 @@ GEM rb-inotify (0.10.0) ffi (~> 1.0) ruby_dep (1.5.0) + skylight (3.1.5) + skylight-core (= 3.1.5) + skylight-core (3.1.5) + activesupport (>= 4.2.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) rack (> 1, < 3) @@ -138,8 +145,11 @@ DEPENDENCIES byebug listen (>= 3.0.5, < 3.2) pg (>= 0.18, < 2.0) + pg_query (>= 0.9.0) + pghero puma (~> 3.11) rails (~> 5.2.3) + skylight tzinfo-data web-console (>= 3.3.0) diff --git a/config/application.rb b/config/application.rb index 9c33109..e08aeef 100644 --- a/config/application.rb +++ b/config/application.rb @@ -11,9 +11,6 @@ class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. config.load_defaults 5.2 - # Settings in config/environments/* take precedence over those specified here. - # Application configuration can go into files in config/initializers - # -- all .rb files in that directory are automatically loaded after loading - # the framework and any gems in your application. + config.skylight.environments += ["development"] end end diff --git a/config/routes.rb b/config/routes.rb index a2da6a7..ab7cca7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,4 +1,5 @@ Rails.application.routes.draw do + #mount PgHero::Engine, at: 'pghero' # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html get "/" => "statistics#index" get "автобусы/:from/:to" => "trips#index" From b4b6589a671a743659e443e4da25a3914e5836ab Mon Sep 17 00:00:00 2001 From: Artsiom Musin Date: Fri, 12 Apr 2019 22:23:32 +0300 Subject: [PATCH 2/3] Speed up views --- app/controllers/trips_controller.rb | 2 +- app/views/trips/_delimiter.html.erb | 1 - app/views/trips/_services.html.erb | 6 ------ app/views/trips/_trip.html.erb | 21 ++++++++++++++++----- app/views/trips/index.html.erb | 11 +---------- 5 files changed, 18 insertions(+), 23 deletions(-) delete mode 100644 app/views/trips/_delimiter.html.erb delete mode 100644 app/views/trips/_services.html.erb diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be..ad823f6 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,6 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + @trips = Trip.includes(:bus, bus: :services).where(from: @from, to: @to).order(:start_time) end end diff --git a/app/views/trips/_delimiter.html.erb b/app/views/trips/_delimiter.html.erb deleted file mode 100644 index 3f845ad..0000000 --- a/app/views/trips/_delimiter.html.erb +++ /dev/null @@ -1 +0,0 @@ -==================================================== diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb deleted file mode 100644 index 2de639f..0000000 --- a/app/views/trips/_services.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -
  • Сервисы в автобусе:
    • - diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb index fa1de9a..7df6e98 100644 --- a/app/views/trips/_trip.html.erb +++ b/app/views/trips/_trip.html.erb @@ -1,5 +1,16 @@ -
  • <%= "Отправление: #{trip.start_time}" %>
    • -
  • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
    • -
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
    • -
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
    • -
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • + +==================================================== diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce4..f342d06 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -4,13 +4,4 @@

    <%= "В расписании #{@trips.count} рейсов" %>

    - -<% @trips.each do |trip| %> - - <%= render "delimiter" %> -<% end %> +<%= render partial: "trip", collection: @trips, as: :trip %> From a5816ca8244839e8cbc4cc3dfbc1fdfe8793cc99 Mon Sep 17 00:00:00 2001 From: Artsiom Musin Date: Sat, 13 Apr 2019 23:22:58 +0300 Subject: [PATCH 3/3] Speed up json loading rake task... reinvented the wheel? --- Gemfile | 2 + Gemfile.lock | 12 ++- app/models/bus.rb | 2 +- app/models/service.rb | 2 +- app/views/trips/_service.html.erb | 1 - app/views/trips/_trip.html.erb | 16 --- app/views/trips/index.html.erb | 21 +++- config/routes.rb | 2 +- db/migrate/20190412192616_add_trip_indices.rb | 7 ++ db/migrate/20190412195139_add_bus_index.rb | 7 ++ .../20190412203601_add_bus_service_indices.rb | 7 ++ db/schema.rb | 6 +- lib/tasks/utils.rake | 97 +++++++++++++------ 13 files changed, 130 insertions(+), 52 deletions(-) delete mode 100644 app/views/trips/_service.html.erb delete mode 100644 app/views/trips/_trip.html.erb create mode 100644 db/migrate/20190412192616_add_trip_indices.rb create mode 100644 db/migrate/20190412195139_add_bus_index.rb create mode 100644 db/migrate/20190412203601_add_bus_service_indices.rb diff --git a/Gemfile b/Gemfile index ebbb4c7..1dd1990 100644 --- a/Gemfile +++ b/Gemfile @@ -10,6 +10,8 @@ gem 'bootsnap', '>= 1.1.0', require: false gem 'pghero' gem 'pg_query', '>= 0.9.0' gem 'skylight' +gem 'activerecord-import' +gem 'strong_migrations' group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console diff --git a/Gemfile.lock b/Gemfile.lock index 322483d..aaeb2b0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,6 +33,8 @@ GEM activemodel (= 5.2.3) activesupport (= 5.2.3) arel (>= 9.0) + activerecord-import (1.0.1) + activerecord (>= 3.2) activestorage (5.2.3) actionpack (= 5.2.3) activerecord (= 5.2.3) @@ -43,8 +45,8 @@ GEM minitest (~> 5.1) tzinfo (~> 1.1) arel (9.0.0) - bindex (0.6.0) - bootsnap (1.4.2) + bindex (0.7.0) + bootsnap (1.4.3) msgpack (~> 1.0) builder (3.2.3) byebug (11.0.1) @@ -81,7 +83,7 @@ GEM pghero (2.2.0) activerecord puma (3.12.1) - rack (2.0.6) + rack (2.0.7) rack-test (1.1.0) rack (>= 1.0, < 3) rails (5.2.3) @@ -124,6 +126,8 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) + strong_migrations (0.3.1) + activerecord (>= 3.2.0) thor (0.20.3) thread_safe (0.3.6) tzinfo (1.2.5) @@ -141,6 +145,7 @@ PLATFORMS ruby DEPENDENCIES + activerecord-import bootsnap (>= 1.1.0) byebug listen (>= 3.0.5, < 3.2) @@ -150,6 +155,7 @@ DEPENDENCIES puma (~> 3.11) rails (~> 5.2.3) skylight + strong_migrations tzinfo-data web-console (>= 3.3.0) diff --git a/app/models/bus.rb b/app/models/bus.rb index 1dcc54c..956cfa6 100644 --- a/app/models/bus.rb +++ b/app/models/bus.rb @@ -13,7 +13,7 @@ class Bus < ApplicationRecord ].freeze has_many :trips - has_and_belongs_to_many :services, join_table: :buses_services + has_and_belongs_to_many :services, join_table: :buses_services, autosave: true validates :number, presence: true, uniqueness: true validates :model, inclusion: { in: MODELS } diff --git a/app/models/service.rb b/app/models/service.rb index 9cbb2a3..bb44974 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -12,7 +12,7 @@ class Service < ApplicationRecord 'Можно не печатать билет', ].freeze - has_and_belongs_to_many :buses, join_table: :buses_services + has_and_belongs_to_many :buses, join_table: :buses_services, autosave: true validates :name, presence: true validates :name, inclusion: { in: SERVICES } diff --git a/app/views/trips/_service.html.erb b/app/views/trips/_service.html.erb deleted file mode 100644 index 178ea8c..0000000 --- a/app/views/trips/_service.html.erb +++ /dev/null @@ -1 +0,0 @@ -
  • <%= "#{service.name}" %>
    • diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb deleted file mode 100644 index 7df6e98..0000000 --- a/app/views/trips/_trip.html.erb +++ /dev/null @@ -1,16 +0,0 @@ -
      -
    • <%= "Отправление: #{trip.start_time}" %>
      • -
    • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
      • -
    • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
      • -
    • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
      • -
    • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
      • - - <% services = trip.bus.services %> - <% if services.any? %> -
    • Сервисы в автобусе:
      • -
        - <%= render partial: "service", collection: services, as: :service %> -
      - <% end %> -
    -==================================================== diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index f342d06..7b18873 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -4,4 +4,23 @@

    <%= "В расписании #{@trips.count} рейсов" %>

    -<%= render partial: "trip", collection: @trips, as: :trip %> +<% @trips.each do |trip|%> +
      +
    • <%= "Отправление: #{trip.start_time}" %>
      • +
    • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
      • +
    • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
      • +
    • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
      • +
    • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
      • + + <% services = trip.bus.services %> + <% if services.any? %> +
    • Сервисы в автобусе:
      • +
        + <% services.each do |service|%> +
      • <%= "#{service.name}" %>
        • + <%end%> +
      + <% end %> +
    + ==================================================== +<% end %> diff --git a/config/routes.rb b/config/routes.rb index ab7cca7..581873d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,5 @@ Rails.application.routes.draw do - #mount PgHero::Engine, at: 'pghero' + # mount PgHero::Engine, at: 'pghero' # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html get "/" => "statistics#index" get "автобусы/:from/:to" => "trips#index" diff --git a/db/migrate/20190412192616_add_trip_indices.rb b/db/migrate/20190412192616_add_trip_indices.rb new file mode 100644 index 0000000..1d00e6a --- /dev/null +++ b/db/migrate/20190412192616_add_trip_indices.rb @@ -0,0 +1,7 @@ +class AddTripIndices < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + add_index :trips, [:from_id, :to_id], algorithm: :concurrently + end +end diff --git a/db/migrate/20190412195139_add_bus_index.rb b/db/migrate/20190412195139_add_bus_index.rb new file mode 100644 index 0000000..ef6021e --- /dev/null +++ b/db/migrate/20190412195139_add_bus_index.rb @@ -0,0 +1,7 @@ +class AddBusIndex < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + add_index :buses, :number, algorithm: :concurrently + end +end diff --git a/db/migrate/20190412203601_add_bus_service_indices.rb b/db/migrate/20190412203601_add_bus_service_indices.rb new file mode 100644 index 0000000..34dcdd8 --- /dev/null +++ b/db/migrate/20190412203601_add_bus_service_indices.rb @@ -0,0 +1,7 @@ +class AddBusServiceIndices < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + add_index :buses_services, [:bus_id, :service_id], algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index f6921e4..4c45580 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2019_04_12_211708) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -18,15 +18,18 @@ create_table "buses", force: :cascade do |t| t.string "number" t.string "model" + t.index ["number"], name: "index_buses_on_number" end create_table "buses_services", force: :cascade do |t| t.integer "bus_id" t.integer "service_id" + t.index ["bus_id", "service_id"], name: "index_buses_services_on_bus_id_and_service_id" end create_table "cities", force: :cascade do |t| t.string "name" + t.index ["name"], name: "index_cities_on_name" end create_table "services", force: :cascade do |t| @@ -40,6 +43,7 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id" end end diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe87..f76c2df 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,34 +1,77 @@ # Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] +require 'benchmark' + +def create_services + new_services = [] + all_services = Service.const_get('SERVICES').each_with_object({}) do |name, result| + new_services << result[name] = Service.new(name: name) + end + Service.import(new_services) + all_services +end + +def create_cities(data) + uniq_cities = data.each_with_object([]) do |trip, result| + result << trip['from'] + result << trip['to'] + end.uniq + new_cities = [] + all_cities = uniq_cities.each_with_object({}) do |name, result| + new_cities << result[name] = City.new(name: name) + end + City.import(new_cities) + all_cities +end + +def create_buses(data, all_services) + new_buses = [] + all_buses = data.each_with_object({}) do |trip, result| + next if result[trip['bus']['number']] + + services = trip['bus']['services'].map {|service| all_services[service] } + new_buses << result[trip['bus']['number']] = Bus.new( + number: trip['bus']['number'], + model: trip['bus']['model'], + services: services + ) + end + #Bus.import(new_buses) + new_buses.each(&:save!) + all_buses +end + task :reload_json, [:file_name] => :environment do |_task, args| - json = JSON.parse(File.read(args.file_name)) - - ActiveRecord::Base.transaction do - City.delete_all - Bus.delete_all - Service.delete_all - Trip.delete_all - ActiveRecord::Base.connection.execute('delete from buses_services;') - - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s + time = Benchmark.realtime do + json = JSON.parse(File.read(args.file_name)) + + ActiveRecord::Base.transaction do + City.delete_all + Bus.delete_all + Service.delete_all + Trip.delete_all + ActiveRecord::Base.connection.execute('delete from buses_services;') + + all_services = create_services + all_cities = create_cities(json) + all_buses = create_buses(json, all_services) + + new_records = json.map do |trip| + from = all_cities[trip['from']] + to = all_cities[trip['to']] + bus = all_buses[trip['bus']['number']] + + Trip.new( + from: from, + to: to, + bus: bus, + start_time: trip['start_time'], + duration_minutes: trip['duration_minutes'], + price_cents: trip['price_cents'], + ) end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) + Trip.import(new_records) end end + puts "Finish in #{time.round(2)}" end