From 28a55cfe5517a8a0a9cd162722010cedb2a44ef8 Mon Sep 17 00:00:00 2001 From: Alex Emelyanov Date: Wed, 3 Apr 2019 10:19:57 +0300 Subject: [PATCH 1/7] Add spec for populate task --- .rspec | 4 ++ Gemfile | 1 + Gemfile.lock | 19 ++++++ app/services/db_populator.rb | 36 +++++++++++ lib/tasks/utils.rake | 31 +-------- spec/fixtures/micro.json | 53 +++++++++++++++ spec/rails_helper.rb | 61 ++++++++++++++++++ spec/services/db_populator_spec.rb | 30 +++++++++ spec/spec_helper.rb | 96 ++++++++++++++++++++++++++++ test/application_system_test_case.rb | 5 -- test/controllers/.keep | 0 test/fixtures/.keep | 0 test/fixtures/files/.keep | 0 test/helpers/.keep | 0 test/integration/.keep | 0 test/mailers/.keep | 0 test/models/.keep | 0 test/system/.keep | 0 test/test_helper.rb | 10 --- 19 files changed, 301 insertions(+), 45 deletions(-) create mode 100644 .rspec create mode 100644 app/services/db_populator.rb create mode 100644 spec/fixtures/micro.json create mode 100644 spec/rails_helper.rb create mode 100644 spec/services/db_populator_spec.rb create mode 100644 spec/spec_helper.rb delete mode 100644 test/application_system_test_case.rb delete mode 100644 test/controllers/.keep delete mode 100644 test/fixtures/.keep delete mode 100644 test/fixtures/files/.keep delete mode 100644 test/helpers/.keep delete mode 100644 test/integration/.keep delete mode 100644 test/mailers/.keep delete mode 100644 test/models/.keep delete mode 100644 test/system/.keep delete mode 100644 test/test_helper.rb diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..93f21ce --- /dev/null +++ b/.rspec @@ -0,0 +1,4 @@ +--require rails_helper +--profile 10 +--color + diff --git a/Gemfile b/Gemfile index 33017fd..1534826 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,7 @@ gem 'bootsnap', '>= 1.1.0', require: false group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] + gem 'rspec-rails', '~> 3.8' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index eb22e16..3f111f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,6 +50,7 @@ GEM byebug (11.0.1) concurrent-ruby (1.1.5) crass (1.0.4) + diff-lcs (1.3) erubi (1.8.0) ffi (1.10.0) globalid (0.4.2) @@ -109,6 +110,23 @@ GEM rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-mocks (3.8.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-rails (3.8.2) + actionpack (>= 3.0) + activesupport (>= 3.0) + railties (>= 3.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.0) ruby_dep (1.5.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) @@ -140,6 +158,7 @@ DEPENDENCIES pg (>= 0.18, < 2.0) puma (~> 3.11) rails (~> 5.2.3) + rspec-rails (~> 3.8) tzinfo-data web-console (>= 3.3.0) diff --git a/app/services/db_populator.rb b/app/services/db_populator.rb new file mode 100644 index 0000000..9f584c9 --- /dev/null +++ b/app/services/db_populator.rb @@ -0,0 +1,36 @@ +class DbPopulator + class << self + def populate(file_name) + json = JSON.parse(File.read(file_name)) + + ActiveRecord::Base.transaction do + City.delete_all + Bus.delete_all + Service.delete_all + Trip.delete_all + ActiveRecord::Base.connection.execute('delete from buses_services;') + + json.each do |trip| + from = City.find_or_create_by(name: trip['from']) + to = City.find_or_create_by(name: trip['to']) + services = [] + trip['bus']['services'].each do |service| + s = Service.find_or_create_by(name: service) + services << s + end + bus = Bus.find_or_create_by(number: trip['bus']['number']) + bus.update(model: trip['bus']['model'], services: services) + + Trip.create!( + from: from, + to: to, + bus: bus, + start_time: trip['start_time'], + duration_minutes: trip['duration_minutes'], + price_cents: trip['price_cents'], + ) + end + end + end + end +end diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe87..60a1aac 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,34 +1,5 @@ # Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] task :reload_json, [:file_name] => :environment do |_task, args| - json = JSON.parse(File.read(args.file_name)) - - ActiveRecord::Base.transaction do - City.delete_all - Bus.delete_all - Service.delete_all - Trip.delete_all - ActiveRecord::Base.connection.execute('delete from buses_services;') - - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s - end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) - end - end + DbPopulator.populate(args.file_name) end diff --git a/spec/fixtures/micro.json b/spec/fixtures/micro.json new file mode 100644 index 0000000..18427be --- /dev/null +++ b/spec/fixtures/micro.json @@ -0,0 +1,53 @@ +[ + { + "bus": { + "model": "Икарус", + "number": "229", + "services": [ + "Ремни безопасности", + "Кондиционер общий", + "Кондиционер Индивидуальный", + "Телевизор индивидуальный", + "Стюардесса", + "Можно не печатать билет" + ] + }, + "duration_minutes": 83, + "from": "Сочи", + "price_cents": 23354, + "start_time": "16:11", + "to": "Тула" + }, + { + "bus": { + "model": "Вольво", + "number": "912", + "services": [ + "WiFi", + "Работающий туалет", + "Телевизор индивидуальный", + "Стюардесса", + "Можно не печатать билет" + ] + }, + "duration_minutes": 572, + "from": "Самара", + "price_cents": 83861, + "start_time": "13:13", + "to": "Самара" + }, + { + "bus": { + "model": "ГАЗ", + "number": "584", + "services": [ + "Кондиционер общий" + ] + }, + "duration_minutes": 186, + "from": "Красноярск", + "price_cents": 80288, + "start_time": "13:48", + "to": "Волгоград" + } +] diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb new file mode 100644 index 0000000..d73d80b --- /dev/null +++ b/spec/rails_helper.rb @@ -0,0 +1,61 @@ +# This file is copied to spec/ when you run 'rails generate rspec:install' +require 'spec_helper' +ENV['RAILS_ENV'] ||= 'test' +require File.expand_path('../../config/environment', __FILE__) +# Prevent database truncation if the environment is production +abort("The Rails environment is running in production mode!") if Rails.env.production? +require 'rspec/rails' +# Add additional requires below this line. Rails is not loaded until this point! + +# Requires supporting ruby files with custom matchers and macros, etc, in +# spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are +# run as spec files by default. This means that files in spec/support that end +# in _spec.rb will both be required and run as specs, causing the specs to be +# run twice. It is recommended that you do not name files matching this glob to +# end with _spec.rb. You can configure this pattern with the --pattern +# option on the command line or in ~/.rspec, .rspec or `.rspec-local`. +# +# The following line is provided for convenience purposes. It has the downside +# of increasing the boot-up time by auto-requiring all files in the support +# directory. Alternatively, in the individual `*_spec.rb` files, manually +# require only the support files necessary. +# +# Dir[Rails.root.join('spec', 'support', '**', '*.rb')].each { |f| require f } + +# Checks for pending migrations and applies them before tests are run. +# If you are not using ActiveRecord, you can remove these lines. +begin + ActiveRecord::Migration.maintain_test_schema! +rescue ActiveRecord::PendingMigrationError => e + puts e.to_s.strip + exit 1 +end +RSpec.configure do |config| + # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures + config.fixture_path = "#{::Rails.root}/spec/fixtures" + + # If you're not using ActiveRecord, or you'd prefer not to run each of your + # examples within a transaction, remove the following line or assign false + # instead of true. + config.use_transactional_fixtures = true + + # RSpec Rails can automatically mix in different behaviours to your tests + # based on their file location, for example enabling you to call `get` and + # `post` in specs under `spec/controllers`. + # + # You can disable this behaviour by removing the line below, and instead + # explicitly tag your specs with their type, e.g.: + # + # RSpec.describe UsersController, :type => :controller do + # # ... + # end + # + # The different available types are documented in the features, such as in + # https://relishapp.com/rspec/rspec-rails/docs + config.infer_spec_type_from_file_location! + + # Filter lines from Rails gems in backtraces. + config.filter_rails_from_backtrace! + # arbitrary gems may also be filtered via: + # config.filter_gems_from_backtrace("gem name") +end diff --git a/spec/services/db_populator_spec.rb b/spec/services/db_populator_spec.rb new file mode 100644 index 0000000..8ee765d --- /dev/null +++ b/spec/services/db_populator_spec.rb @@ -0,0 +1,30 @@ +describe DbPopulator do + describe '.populate' do + subject { described_class.populate(file_name) } + + let(:file_name) { 'spec/fixtures/micro.json' } + + it do + expect { subject }.to change { City.count }.by(5) + .and change { Bus.count }.by(3) + .and change { Service.count }.by(8) + .and change { Trip.count }.by(3) + + expect(City.pluck(:name).sort).to eq %w(Волгоград Красноярск Самара Сочи Тула) + expect(Bus.pluck(:number).sort).to eq %w(229 584 912) + + expect(Service.pluck(:name).sort).to eq [ + "WiFi", + "Кондиционер Индивидуальный", + "Кондиционер общий", + "Можно не печатать билет", + "Работающий туалет", + "Ремни безопасности", + "Стюардесса", + "Телевизор индивидуальный", + ] + + expect(Trip.pluck(:price_cents).sort).to eq [23354, 80288, 83861] + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..ce33d66 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,96 @@ +# This file was generated by the `rails generate rspec:install` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ + # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ + # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode + config.disable_monkey_patching! + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end diff --git a/test/application_system_test_case.rb b/test/application_system_test_case.rb deleted file mode 100644 index d19212a..0000000 --- a/test/application_system_test_case.rb +++ /dev/null @@ -1,5 +0,0 @@ -require "test_helper" - -class ApplicationSystemTestCase < ActionDispatch::SystemTestCase - driven_by :selenium, using: :chrome, screen_size: [1400, 1400] -end diff --git a/test/controllers/.keep b/test/controllers/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/fixtures/.keep b/test/fixtures/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/fixtures/files/.keep b/test/fixtures/files/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/helpers/.keep b/test/helpers/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/integration/.keep b/test/integration/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/mailers/.keep b/test/mailers/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/models/.keep b/test/models/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/system/.keep b/test/system/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/test/test_helper.rb b/test/test_helper.rb deleted file mode 100644 index 3ab84e3..0000000 --- a/test/test_helper.rb +++ /dev/null @@ -1,10 +0,0 @@ -ENV['RAILS_ENV'] ||= 'test' -require_relative '../config/environment' -require 'rails/test_help' - -class ActiveSupport::TestCase - # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. - fixtures :all - - # Add more helper methods to be used by all tests here... -end From a388cf39ce80d377ed3acf2aeeae2dddb59d84e4 Mon Sep 17 00:00:00 2001 From: Alex Emelyanov Date: Wed, 3 Apr 2019 10:36:16 +0300 Subject: [PATCH 2/7] Add optimization loop for DB populator --- Gemfile | 7 +++++++ Gemfile.lock | 14 ++++++++++++++ app/services/db_populator.rb | 2 ++ lib/tasks/optimization.rake | 28 ++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 lib/tasks/optimization.rake diff --git a/Gemfile b/Gemfile index 1534826..8ac96e7 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,13 @@ group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] gem 'rspec-rails', '~> 3.8' + + # Profiling + gem 'benchmark-ips' + gem 'kalibera' + gem 'memory_profiler' + gem 'stackprof' + gem 'ruby-prof' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 3f111f5..bb30229 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -43,6 +43,7 @@ GEM minitest (~> 5.1) tzinfo (~> 1.1) arel (9.0.0) + benchmark-ips (2.7.2) bindex (0.6.0) bootsnap (1.4.2) msgpack (~> 1.0) @@ -57,6 +58,9 @@ GEM activesupport (>= 4.2.0) i18n (1.6.0) concurrent-ruby (~> 1.0) + kalibera (0.1) + memoist (~> 0.11.0) + rbzip2 (~> 0.2.0) listen (3.1.5) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) @@ -68,6 +72,8 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) + memoist (0.11.0) + memory_profiler (0.9.13) method_source (0.9.2) mimemagic (0.3.3) mini_mime (1.0.1) @@ -110,6 +116,7 @@ GEM rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) + rbzip2 (0.2.0) rspec-core (3.8.0) rspec-support (~> 3.8.0) rspec-expectations (3.8.2) @@ -127,6 +134,7 @@ GEM rspec-mocks (~> 3.8.0) rspec-support (~> 3.8.0) rspec-support (3.8.0) + ruby-prof (0.17.0) ruby_dep (1.5.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) @@ -135,6 +143,7 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) + stackprof (0.2.12) thor (0.20.3) thread_safe (0.3.6) tzinfo (1.2.5) @@ -152,13 +161,18 @@ PLATFORMS ruby DEPENDENCIES + benchmark-ips bootsnap (>= 1.1.0) byebug + kalibera listen (>= 3.0.5, < 3.2) + memory_profiler pg (>= 0.18, < 2.0) puma (~> 3.11) rails (~> 5.2.3) rspec-rails (~> 3.8) + ruby-prof + stackprof tzinfo-data web-console (>= 3.3.0) diff --git a/app/services/db_populator.rb b/app/services/db_populator.rb index 9f584c9..df3fd10 100644 --- a/app/services/db_populator.rb +++ b/app/services/db_populator.rb @@ -13,11 +13,13 @@ def populate(file_name) json.each do |trip| from = City.find_or_create_by(name: trip['from']) to = City.find_or_create_by(name: trip['to']) + services = [] trip['bus']['services'].each do |service| s = Service.find_or_create_by(name: service) services << s end + bus = Bus.find_or_create_by(number: trip['bus']['number']) bus.update(model: trip['bus']['model'], services: services) diff --git a/lib/tasks/optimization.rake b/lib/tasks/optimization.rake new file mode 100644 index 0000000..368271b --- /dev/null +++ b/lib/tasks/optimization.rake @@ -0,0 +1,28 @@ +def test_correctness(spec_file) + puts `bundle exec rspec #{spec_file}` + + if !$?.success? + raise 'code broken!' + end +end + +def evaluate_metric + GC.disable + + Benchmark.ips do |b| + b.stats = :bootstrap + b.confidence = 99 + + yield(b) + end +end + +namespace :optimization do + task db_populator: :environment do |_task, args| + test_correctness('spec/services/db_populator_spec.rb') + + evaluate_metric do |b| + b.report('small.json') { DbPopulator.populate('fixtures/small.json') } + end + end +end From adc775e6e47dfd0157c6ec7815d1825981ac7e73 Mon Sep 17 00:00:00 2001 From: Alex Emelyanov Date: Wed, 3 Apr 2019 11:50:38 +0300 Subject: [PATCH 3/7] Optimize db population --- .ruby-version | 2 +- Gemfile | 3 ++ Gemfile.lock | 3 ++ app/models/bus.rb | 3 +- app/models/bus_service.rb | 6 ++++ app/models/service.rb | 3 +- app/services/db_populator.rb | 56 ++++++++++++++++++++++++------------ lib/tasks/optimization.rake | 22 ++++++++++++++ lib/tasks/profiling.rake | 46 +++++++++++++++++++++++++++++ 9 files changed, 122 insertions(+), 22 deletions(-) create mode 100644 app/models/bus_service.rb create mode 100644 lib/tasks/profiling.rake diff --git a/.ruby-version b/.ruby-version index 6a6a3d8..473d5b8 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.6.1 +2.6.1-railsexpress diff --git a/Gemfile b/Gemfile index 8ac96e7..bc01d04 100644 --- a/Gemfile +++ b/Gemfile @@ -4,7 +4,10 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" } ruby '2.6.1' gem 'rails', '~> 5.2.3' + gem 'pg', '>= 0.18', '< 2.0' +gem 'bulk_insert', '~> 1.7' + gem 'puma', '~> 3.11' gem 'bootsnap', '>= 1.1.0', require: false diff --git a/Gemfile.lock b/Gemfile.lock index bb30229..244f7dc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -48,6 +48,8 @@ GEM bootsnap (1.4.2) msgpack (~> 1.0) builder (3.2.3) + bulk_insert (1.7.0) + activerecord (>= 3.2.0) byebug (11.0.1) concurrent-ruby (1.1.5) crass (1.0.4) @@ -163,6 +165,7 @@ PLATFORMS DEPENDENCIES benchmark-ips bootsnap (>= 1.1.0) + bulk_insert (~> 1.7) byebug kalibera listen (>= 3.0.5, < 3.2) diff --git a/app/models/bus.rb b/app/models/bus.rb index 1dcc54c..ae56e99 100644 --- a/app/models/bus.rb +++ b/app/models/bus.rb @@ -13,7 +13,8 @@ class Bus < ApplicationRecord ].freeze has_many :trips - has_and_belongs_to_many :services, join_table: :buses_services + has_many :bus_services + has_many :services, through: :bus_services validates :number, presence: true, uniqueness: true validates :model, inclusion: { in: MODELS } diff --git a/app/models/bus_service.rb b/app/models/bus_service.rb new file mode 100644 index 0000000..c842d9f --- /dev/null +++ b/app/models/bus_service.rb @@ -0,0 +1,6 @@ +class BusService < ApplicationRecord + self.table_name = 'buses_services' + + belongs_to :bus + belongs_to :service +end diff --git a/app/models/service.rb b/app/models/service.rb index 9cbb2a3..c991963 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -12,7 +12,8 @@ class Service < ApplicationRecord 'Можно не печатать билет', ].freeze - has_and_belongs_to_many :buses, join_table: :buses_services + has_many :bus_services + has_many :buses, through: :bus_services validates :name, presence: true validates :name, inclusion: { in: SERVICES } diff --git a/app/services/db_populator.rb b/app/services/db_populator.rb index df3fd10..6fe43d2 100644 --- a/app/services/db_populator.rb +++ b/app/services/db_populator.rb @@ -3,34 +3,52 @@ class << self def populate(file_name) json = JSON.parse(File.read(file_name)) + all_services = {} + all_buses = {} + all_cities = {} + ActiveRecord::Base.transaction do City.delete_all + BusService.delete_all Bus.delete_all Service.delete_all Trip.delete_all - ActiveRecord::Base.connection.execute('delete from buses_services;') - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) + Trip.bulk_insert do |trip_worker| + BusService.bulk_insert do |bus_service_worker| + json.each do |trip| + from = (all_cities[trip['from']] ||= City.create!(name: trip['from'])) + to = (all_cities[trip['to']] ||= City.create!(name: trip['to'])) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s - end + services = [] + trip['bus']['services'].each do |service| + s = (all_services[service] ||= Service.create!(name: service)) + services << s + end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) + number = trip['bus']['number'] + bus = (all_buses[number] ||= begin + Bus.create!(number: number, model: trip['bus']['model']).tap do |bus| + services.each do |service| + bus_service_worker.add(bus: bus, service: service) + end + end + end) - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) + # if bus.model != trip['bus']['model'] || bus.services != services + # bus.update(model: trip['bus']['model'], services: services) + # end + + trip_worker.add( + from: from, + to: to, + bus: bus, + start_time: trip['start_time'], + duration_minutes: trip['duration_minutes'], + price_cents: trip['price_cents'], + ) + end + end end end end diff --git a/lib/tasks/optimization.rake b/lib/tasks/optimization.rake index 368271b..d7fe2a5 100644 --- a/lib/tasks/optimization.rake +++ b/lib/tasks/optimization.rake @@ -25,4 +25,26 @@ namespace :optimization do b.report('small.json') { DbPopulator.populate('fixtures/small.json') } end end + + <<~VALUES + Initial value: + 0.105 + + Cache services creating: + 0.125 + + Cache buses creating: + 0.152 + + Cache cities creating: + 0.232 + + Bulk inserting trips: + 0.301 + + Bulk inserting bus services: + 1.156 + + bundle exec rake "reload_json[fixtures/large.json]" 8.88s user 0.89s system 86% cpu 11.274 total + VALUES end diff --git a/lib/tasks/profiling.rake b/lib/tasks/profiling.rake new file mode 100644 index 0000000..5004f9d --- /dev/null +++ b/lib/tasks/profiling.rake @@ -0,0 +1,46 @@ +def ruby_prof_wall(*args) + # GC.disable + + # require 'memory_profiler' + # report = MemoryProfiler.report do + # work('data_medium.txt') + # end + # report.pretty_print(color_output: true, scale_bytes: true) + + # require 'ruby-prof' + # RubyProf.measure_mode = RubyProf::MEMORY + RubyProf.measure_mode = RubyProf::WALL_TIME + result = RubyProf.profile do + yield + end + + printer = RubyProf::FlatPrinter.new(result) + printer.print(STDOUT) + + # printer = RubyProf::GraphPrinter.new(result) + # printer.print(STDOUT, {}) + + # printer = RubyProf::CallStackPrinter.new(result) + # File.open('ruby-prof-call-stack.html', "w") do |f| + # printer.print(f, threshold: 0, min_percent: 0, title: "ruby_prof WALL_TIME") + # end + + # printer = RubyProf::CallTreePrinter.new(result) + # printer.print() +end + +def stack_prof_wall(method) + StackProf.run(mode: :wall, out: 'tmp/stackprof.dump', raw: true) do + yield + end + + puts `stackprof tmp/stackprof.dump --method #{method}` +end + +namespace :profiling do + task :db_populator, [:profiler] => :environment do |_task, args| + send(args.profiler, 'DbPopulator.populate') do + DbPopulator.populate('fixtures/small.json') + end + end +end From 3479b6fade51c39e9f96ccbdf403efc8df5746ab Mon Sep 17 00:00:00 2001 From: Alex Emelyanov Date: Fri, 5 Apr 2019 01:08:41 +0300 Subject: [PATCH 4/7] Fix DB populator + add trips spec --- app/services/db_populator.rb | 12 ++++-------- spec/controllers/trips_controller_spec.rb | 24 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 spec/controllers/trips_controller_spec.rb diff --git a/app/services/db_populator.rb b/app/services/db_populator.rb index 6fe43d2..79c4202 100644 --- a/app/services/db_populator.rb +++ b/app/services/db_populator.rb @@ -30,19 +30,15 @@ def populate(file_name) bus = (all_buses[number] ||= begin Bus.create!(number: number, model: trip['bus']['model']).tap do |bus| services.each do |service| - bus_service_worker.add(bus: bus, service: service) + bus_service_worker.add(bus_id: bus.id, service_id: service.id) end end end) - # if bus.model != trip['bus']['model'] || bus.services != services - # bus.update(model: trip['bus']['model'], services: services) - # end - trip_worker.add( - from: from, - to: to, - bus: bus, + from_id: from.id, + to_id: to.id, + bus_id: bus.id, start_time: trip['start_time'], duration_minutes: trip['duration_minutes'], price_cents: trip['price_cents'], diff --git a/spec/controllers/trips_controller_spec.rb b/spec/controllers/trips_controller_spec.rb new file mode 100644 index 0000000..545719f --- /dev/null +++ b/spec/controllers/trips_controller_spec.rb @@ -0,0 +1,24 @@ +describe TripsController do + include Rack::Test::Methods + + def app + Rails.application + end + + render_views + + describe '#index' do + subject { get URI.escape('/автобусы/Самара/Москва') } + + before do + DbPopulator.populate('fixtures/example.json') + end + + it do + subject + expect(last_response.status).to eq 200 + expect(last_response.body).to include 'В расписании 5 рейсов' + expect(last_response.body).to include '
  • Автобус: Икарус №123
    • ' + end + end +end From 1f0b884736f772c78c9cf9b94bd53eae1724121e Mon Sep 17 00:00:00 2001 From: Alex Emelyanov Date: Fri, 5 Apr 2019 01:17:59 +0300 Subject: [PATCH 5/7] Add optimization feedback loop for trials controller --- lib/tasks/optimization.rake | 66 ++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/lib/tasks/optimization.rake b/lib/tasks/optimization.rake index d7fe2a5..f67a494 100644 --- a/lib/tasks/optimization.rake +++ b/lib/tasks/optimization.rake @@ -17,34 +17,68 @@ def evaluate_metric end end + namespace :optimization do + task trips_controller: :environment do |_task, args| + test_correctness('spec/controllers/trips_controller_spec.rb') + + class TripsControllerExecutor + include Rack::Test::Methods + + def app + Rails.application + end + + def run + get URI.escape('/автобусы/Самара/Москва') + end + end + + evaluate_metric do |b| + DbPopulator.populate('fixtures/small.json') + executor = TripsControllerExecutor.new + + b.report('large.json') { executor.run } + end + + <<~RESULTS + bundle exec rake optimization:trips_controller + + Initial value: + 13.137 ips + + RESULTS + end + task db_populator: :environment do |_task, args| test_correctness('spec/services/db_populator_spec.rb') evaluate_metric do |b| b.report('small.json') { DbPopulator.populate('fixtures/small.json') } end - end - <<~VALUES - Initial value: - 0.105 + <<~RESULTS + Initial value: + 0.105 ips - Cache services creating: - 0.125 + Cache services creating: + 0.125 - Cache buses creating: - 0.152 + Cache buses creating: + 0.152 - Cache cities creating: - 0.232 + Cache cities creating: + 0.232 - Bulk inserting trips: - 0.301 + Bulk inserting trips: + 0.301 - Bulk inserting bus services: - 1.156 + Bulk inserting bus services: + 1.156 + + Final: + bundle exec rake "reload_json[fixtures/large.json]" 8.88s user 0.89s system 86% cpu 11.274 total + RESULTS + end - bundle exec rake "reload_json[fixtures/large.json]" 8.88s user 0.89s system 86% cpu 11.274 total - VALUES end From 20fc703622e1cf88dc803f4d5b595865edcfa07f Mon Sep 17 00:00:00 2001 From: Alex Emelyanov Date: Fri, 5 Apr 2019 13:51:13 +0300 Subject: [PATCH 6/7] Optimize trips#index --- Gemfile | 1 + Gemfile.lock | 8 ++++ app/controllers/trips_controller.rb | 6 ++- app/helpers/application_helper.rb | 3 ++ app/helpers/trips_helper.rb | 16 ++++++++ app/views/trips/_delimiter.html.erb | 1 - app/views/trips/_service.html.erb | 1 - app/views/trips/_services.html.erb | 6 --- app/views/trips/_trip.html.erb | 6 +++ app/views/trips/index.html.erb | 13 ++---- lib/tasks/optimization.rake | 62 +++++++++++++++++++++++++++-- 11 files changed, 101 insertions(+), 22 deletions(-) create mode 100644 app/helpers/trips_helper.rb delete mode 100644 app/views/trips/_delimiter.html.erb delete mode 100644 app/views/trips/_service.html.erb delete mode 100644 app/views/trips/_services.html.erb diff --git a/Gemfile b/Gemfile index bc01d04..9efc701 100644 --- a/Gemfile +++ b/Gemfile @@ -28,6 +28,7 @@ group :development do # Access an interactive console on exception pages or by calling 'console' anywhere in the code. gem 'web-console', '>= 3.3.0' gem 'listen', '>= 3.0.5', '< 3.2' + # gem 'meta_request' end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index 244f7dc..a306a39 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -51,6 +51,7 @@ GEM bulk_insert (1.7.0) activerecord (>= 3.2.0) byebug (11.0.1) + callsite (0.0.11) concurrent-ruby (1.1.5) crass (1.0.4) diff-lcs (1.3) @@ -76,6 +77,10 @@ GEM mimemagic (~> 0.3.2) memoist (0.11.0) memory_profiler (0.9.13) + meta_request (0.6.0) + callsite (~> 0.0, >= 0.0.11) + rack-contrib (>= 1.1, < 3) + railties (>= 3.0.0, < 6) method_source (0.9.2) mimemagic (0.3.3) mini_mime (1.0.1) @@ -88,6 +93,8 @@ GEM pg (1.1.4) puma (3.12.1) rack (2.0.6) + rack-contrib (2.1.0) + rack (~> 2.0) rack-test (1.1.0) rack (>= 1.0, < 3) rails (5.2.3) @@ -170,6 +177,7 @@ DEPENDENCIES kalibera listen (>= 3.0.5, < 3.2) memory_profiler + meta_request pg (>= 0.18, < 2.0) puma (~> 3.11) rails (~> 5.2.3) diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be..e4f4ee1 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -2,6 +2,10 @@ class TripsController < ApplicationController def index @from = City.find_by_name!(params[:from]) @to = City.find_by_name!(params[:to]) - @trips = Trip.where(from: @from, to: @to).order(:start_time) + + @trips = Trip + .includes(bus: :services) + .where(from: @from, to: @to) + .order(:start_time) end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index de6be79..82298b5 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1,2 +1,5 @@ module ApplicationHelper + def render_delimiter + "====================================================" + end end diff --git a/app/helpers/trips_helper.rb b/app/helpers/trips_helper.rb new file mode 100644 index 0000000..dd24e01 --- /dev/null +++ b/app/helpers/trips_helper.rb @@ -0,0 +1,16 @@ +module TripsHelper + def render_service(service) + "
  • #{service.name}
    • ".html_safe + end + + def render_services(services) + services.map do |service| + <<~HTML +
  • Сервисы в автобусе:
    • +
      + #{render_service(service)} +
    + HTML + end.join('').html_safe + end +end diff --git a/app/views/trips/_delimiter.html.erb b/app/views/trips/_delimiter.html.erb deleted file mode 100644 index 3f845ad..0000000 --- a/app/views/trips/_delimiter.html.erb +++ /dev/null @@ -1 +0,0 @@ -==================================================== diff --git a/app/views/trips/_service.html.erb b/app/views/trips/_service.html.erb deleted file mode 100644 index 178ea8c..0000000 --- a/app/views/trips/_service.html.erb +++ /dev/null @@ -1 +0,0 @@ -
  • <%= "#{service.name}" %>
    • diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb deleted file mode 100644 index 2de639f..0000000 --- a/app/views/trips/_services.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -
  • Сервисы в автобусе:
    • -
      - <% services.each do |service| %> - <%= render "service", service: service %> - <% end %> -
    diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb index fa1de9a..49c70b6 100644 --- a/app/views/trips/_trip.html.erb +++ b/app/views/trips/_trip.html.erb @@ -3,3 +3,9 @@
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • + +<% if trip.bus.services.present? %> + <%= render_services(trip.bus.services) %> +<% end %> + +<%= render_delimiter %> diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce4..8a2f952 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -5,12 +5,7 @@ <%= "В расписании #{@trips.count} рейсов" %> -<% @trips.each do |trip| %> -
      - <%= render "trip", trip: trip %> - <% if trip.bus.services.present? %> - <%= render "services", services: trip.bus.services %> - <% end %> -
    - <%= render "delimiter" %> -<% end %> + +
      + <%= render partial: "trip", collection: @trips %> +
    diff --git a/lib/tasks/optimization.rake b/lib/tasks/optimization.rake index f67a494..fbaddc3 100644 --- a/lib/tasks/optimization.rake +++ b/lib/tasks/optimization.rake @@ -17,7 +17,6 @@ def evaluate_metric end end - namespace :optimization do task trips_controller: :environment do |_task, args| test_correctness('spec/controllers/trips_controller_spec.rb') @@ -35,17 +34,72 @@ namespace :optimization do end evaluate_metric do |b| - DbPopulator.populate('fixtures/small.json') + file = 'small.json' + DbPopulator.populate("fixtures/#{file}") executor = TripsControllerExecutor.new - b.report('large.json') { executor.run } + b.report(file) { executor.run } end <<~RESULTS bundle exec rake optimization:trips_controller Initial value: - 13.137 ips + 7.187 ips + + Render trips as a collection: + 7.583 + + Render services as a collection: + 9.880 + + Replace delimiter with view helper: + 11.110 + + Fix n+1 + 18.567 + + Replace rendering of services with view helper: + 25.831 + + ab -n 100 -c 1 http://localhost:3000/автобусы/Самара/Москва + With enabled meta_request: + Concurrency Level: 1 + Time taken for tests: 4.692 seconds + Complete requests: 100 + Failed requests: 0 + Total transferred: 1087890 bytes + HTML transferred: 1014900 bytes + Requests per second: 21.31 [#/sec] (mean) + Time per request: 46.917 [ms] (mean) + Time per request: 46.917 [ms] (mean, across all concurrent requests) + Transfer rate: 226.44 [Kbytes/sec] received + + Connection Times (ms) + min mean[+/-sd] median max + Connect: 0 0 0.0 0 0 + Processing: 38 47 11.8 41 100 + Waiting: 38 47 11.8 41 100 + Total: 38 47 11.8 41 101 + + With disabled meta_request: + Concurrency Level: 1 + Time taken for tests: 3.201 seconds + Complete requests: 100 + Failed requests: 0 + Total transferred: 1084760 bytes + HTML transferred: 1014900 bytes + Requests per second: 31.24 [#/sec] (mean) + Time per request: 32.010 [ms] (mean) + Time per request: 32.010 [ms] (mean, across all concurrent requests) + Transfer rate: 330.94 [Kbytes/sec] received + + Connection Times (ms) + min mean[+/-sd] median max + Connect: 0 0 0.0 0 0 + Processing: 27 32 6.7 29 60 + Waiting: 27 32 6.7 29 60 + Total: 27 32 6.7 29 60 RESULTS end From 0209a2831838ed6c365a59730c5b8ac8bc8d6336 Mon Sep 17 00:00:00 2001 From: Alex Emelyanov Date: Fri, 5 Apr 2019 20:42:29 +0300 Subject: [PATCH 7/7] Optimize trips#controller with large.json --- app/controllers/trips_controller.rb | 2 +- app/views/trips/index.html.erb | 4 +- config/environments/test.rb | 5 +++ db/migrate/20190405172042_add_indexes.rb | 9 +++++ db/schema.rb | 7 +++- lib/tasks/optimization.rake | 49 +++++++++++++++++++++++- 6 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 db/migrate/20190405172042_add_indexes.rb diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index e4f4ee1..67d2aed 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -4,7 +4,7 @@ def index @to = City.find_by_name!(params[:to]) @trips = Trip - .includes(bus: :services) + .eager_load(bus: :services) .where(from: @from, to: @to) .order(:start_time) end diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index 8a2f952..fc1ce5c 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -2,10 +2,10 @@ <%= "Автобусы #{@from.name} – #{@to.name}" %>

    - <%= "В расписании #{@trips.count} рейсов" %> + <%= "В расписании #{@trips.length} рейсов" %>

      - <%= render partial: "trip", collection: @trips %> + <%= render partial: "trip", collection: @trips, cached: true %>
    diff --git a/config/environments/test.rb b/config/environments/test.rb index 0a38fd3..f3d6178 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -12,6 +12,11 @@ # preloads Rails for running tests, you may have to set it to true. config.eager_load = false + if ENV['ENABLE_CACHE'] == 'true' + puts 'Caching enabled!' + config.cache_store = :memory_store + end + # Configure public file server for tests with Cache-Control for performance. config.public_file_server.enabled = true config.public_file_server.headers = { diff --git a/db/migrate/20190405172042_add_indexes.rb b/db/migrate/20190405172042_add_indexes.rb new file mode 100644 index 0000000..7915642 --- /dev/null +++ b/db/migrate/20190405172042_add_indexes.rb @@ -0,0 +1,9 @@ +class AddIndexes < ActiveRecord::Migration[5.2] + def change + add_index :buses_services, :bus_id + add_index :buses_services, :service_id + add_index :cities, :name + add_index :trips, :bus_id + add_index :trips, [:from_id, :to_id] + end +end diff --git a/db/schema.rb b/db/schema.rb index f6921e4..1a6c722 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2019_04_05_172042) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -23,10 +23,13 @@ create_table "buses_services", force: :cascade do |t| t.integer "bus_id" t.integer "service_id" + t.index ["bus_id"], name: "index_buses_services_on_bus_id" + t.index ["service_id"], name: "index_buses_services_on_service_id" end create_table "cities", force: :cascade do |t| t.string "name" + t.index ["name"], name: "index_cities_on_name" end create_table "services", force: :cascade do |t| @@ -40,6 +43,8 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.index ["bus_id"], name: "index_trips_on_bus_id" + t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id" end end diff --git a/lib/tasks/optimization.rake b/lib/tasks/optimization.rake index fbaddc3..027d1a9 100644 --- a/lib/tasks/optimization.rake +++ b/lib/tasks/optimization.rake @@ -34,13 +34,60 @@ namespace :optimization do end evaluate_metric do |b| - file = 'small.json' + file = 'large.json' DbPopulator.populate("fixtures/#{file}") executor = TripsControllerExecutor.new b.report(file) { executor.run } end + # Session 2 (for large.json) + + <<~RESULTS + Original version: + ab -n 10 -c 1 http://localhost:3000/автобусы/Самара/Москва + In dev mode mean = 20504 ms + + + Optimized after session 1: + ab -n 20 -c 1 http://localhost:3000/автобусы/Самара/Москва + In dev mode mean = 388 ms + + RAILS_ENV=test ENABLE_CACHE=true bundle exec rake optimization:trips_controller + + Initial value: + 3.373 IPS + + SQL optimization: eager_load instead preload + all required indexes + 4.928 + + Enable caching: + 6.404 + + ab -n 100 -c 1 http://localhost:3000/автобусы/Самара/Москва + Concurrency Level: 1 + Time taken for tests: 17.733 seconds + Complete requests: 100 + Failed requests: 0 + Total transferred: 70281278 bytes + HTML transferred: 70211500 bytes + Requests per second: 5.64 [#/sec] (mean) + Time per request: 177.326 [ms] (mean) + Time per request: 177.326 [ms] (mean, across all concurrent requests) + Transfer rate: 3870.50 [Kbytes/sec] received + + Connection Times (ms) + min mean[+/-sd] median max + Connect: 0 0 0.0 0 0 + Processing: 145 177 48.0 172 545 + Waiting: 144 177 48.0 172 545 + Total: 145 177 48.0 172 545 + + 20504/177 = 115.84x + RESULTS + + # Session 1 (for small.json) + <<~RESULTS bundle exec rake optimization:trips_controller