diff --git a/.gitignore b/.gitignore index 18b43c9..be342df 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ # Ignore master key for decrypting credentials and more. /config/master.key +.env diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..2626e15 --- /dev/null +++ b/.rspec @@ -0,0 +1,3 @@ +--color +--require spec_helper +--require rails_helper \ No newline at end of file diff --git a/Gemfile b/Gemfile index 33017fd..c62f8b4 100644 --- a/Gemfile +++ b/Gemfile @@ -7,10 +7,19 @@ gem 'rails', '~> 5.2.3' gem 'pg', '>= 0.18', '< 2.0' gem 'puma', '~> 3.11' gem 'bootsnap', '>= 1.1.0', require: false +gem 'oj' +gem 'activerecord-import' +gem 'newrelic_rpm' +gem 'pghero' +gem 'pg_query', '>= 0.9.0' +gem 'ruby-prof' +gem 'strong_migrations' group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] + gem 'meta_request' + gem 'dotenv-rails' end group :development do @@ -20,6 +29,8 @@ group :development do end group :test do + gem 'rspec-rails', '~> 3.8' + gem 'rails-controller-testing' end # Windows does not include zoneinfo files, so bundle the tzinfo-data gem diff --git a/Gemfile.lock b/Gemfile.lock index eb22e16..5187268 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -33,6 +33,8 @@ GEM activemodel (= 5.2.3) activesupport (= 5.2.3) arel (>= 9.0) + activerecord-import (1.0.1) + activerecord (>= 3.2) activestorage (5.2.3) actionpack (= 5.2.3) activerecord (= 5.2.3) @@ -43,13 +45,19 @@ GEM minitest (~> 5.1) tzinfo (~> 1.1) arel (9.0.0) - bindex (0.6.0) - bootsnap (1.4.2) + bindex (0.7.0) + bootsnap (1.4.4) msgpack (~> 1.0) builder (3.2.3) byebug (11.0.1) + callsite (0.0.11) concurrent-ruby (1.1.5) crass (1.0.4) + diff-lcs (1.3) + dotenv (2.7.2) + dotenv-rails (2.7.2) + dotenv (= 2.7.2) + railties (>= 3.2, < 6.1) erubi (1.8.0) ffi (1.10.0) globalid (0.4.2) @@ -67,18 +75,29 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) + meta_request (0.7.0) + callsite (~> 0.0, >= 0.0.11) + rack-contrib (>= 1.1, < 3) + railties (>= 3.0.0, < 7) method_source (0.9.2) mimemagic (0.3.3) mini_mime (1.0.1) mini_portile2 (2.4.0) minitest (5.11.3) - msgpack (1.2.9) + msgpack (1.2.10) + newrelic_rpm (6.3.0.355) nio4r (2.3.1) - nokogiri (1.10.2) + nokogiri (1.10.3) mini_portile2 (~> 2.4.0) + oj (3.7.12) pg (1.1.4) + pg_query (1.1.0) + pghero (2.2.0) + activerecord puma (3.12.1) - rack (2.0.6) + rack (2.0.7) + rack-contrib (2.1.0) + rack (~> 2.0) rack-test (1.1.0) rack (>= 1.0, < 3) rails (5.2.3) @@ -94,6 +113,10 @@ GEM bundler (>= 1.3.0) railties (= 5.2.3) sprockets-rails (>= 2.0.0) + rails-controller-testing (1.0.4) + actionpack (>= 5.0.1.x) + actionview (>= 5.0.1.x) + activesupport (>= 5.0.1.x) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) @@ -109,6 +132,24 @@ GEM rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-mocks (3.8.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.8.0) + rspec-rails (3.8.2) + actionpack (>= 3.0) + activesupport (>= 3.0) + railties (>= 3.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.0) + ruby-prof (0.17.0) ruby_dep (1.5.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) @@ -117,6 +158,8 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) + strong_migrations (0.3.1) + activerecord (>= 3.2.0) thor (0.20.3) thread_safe (0.3.6) tzinfo (1.2.5) @@ -134,12 +177,23 @@ PLATFORMS ruby DEPENDENCIES + activerecord-import bootsnap (>= 1.1.0) byebug + dotenv-rails listen (>= 3.0.5, < 3.2) + meta_request + newrelic_rpm + oj pg (>= 0.18, < 2.0) + pg_query (>= 0.9.0) + pghero puma (~> 3.11) rails (~> 5.2.3) + rails-controller-testing + rspec-rails (~> 3.8) + ruby-prof + strong_migrations tzinfo-data web-console (>= 3.3.0) diff --git a/app/models/bus.rb b/app/models/bus.rb index 1dcc54c..1ee02a7 100644 --- a/app/models/bus.rb +++ b/app/models/bus.rb @@ -12,8 +12,10 @@ class Bus < ApplicationRecord 'Газель', ].freeze - has_many :trips - has_and_belongs_to_many :services, join_table: :buses_services + has_many :trips, dependent: :destroy + # has_and_belongs_to_many :services, join_table: :buses_services + has_many :buses_services, class_name: 'BusesService' + has_many :services, through: :buses_services validates :number, presence: true, uniqueness: true validates :model, inclusion: { in: MODELS } diff --git a/app/models/buses_service.rb b/app/models/buses_service.rb new file mode 100644 index 0000000..6ec494e --- /dev/null +++ b/app/models/buses_service.rb @@ -0,0 +1,4 @@ +class BusesService < ApplicationRecord + belongs_to :bus + belongs_to :service +end \ No newline at end of file diff --git a/app/models/service.rb b/app/models/service.rb index 9cbb2a3..d1da573 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -12,7 +12,9 @@ class Service < ApplicationRecord 'Можно не печатать билет', ].freeze - has_and_belongs_to_many :buses, join_table: :buses_services + # has_and_belongs_to_many :buses, join_table: :buses_services + has_many :buses_services, class_name: 'BusesService' + has_many :buses, through: :buses_services validates :name, presence: true validates :name, inclusion: { in: SERVICES } diff --git a/app/models/trip.rb b/app/models/trip.rb index 9d63dff..66b45fb 100644 --- a/app/models/trip.rb +++ b/app/models/trip.rb @@ -1,9 +1,9 @@ class Trip < ApplicationRecord HHMM_REGEXP = /([0-1][0-9]|[2][0-3]):[0-5][0-9]/ - belongs_to :from, class_name: 'City' - belongs_to :to, class_name: 'City' - belongs_to :bus + belongs_to :from, class_name: 'City', foreign_key: :from_id, optional: true + belongs_to :to, class_name: 'City', foreign_key: :to_id, optional: true + belongs_to :bus, foreign_key: :bus_id, optional: true validates :from, presence: true validates :to, presence: true diff --git a/app/scripts/ruby_prof.rb b/app/scripts/ruby_prof.rb new file mode 100644 index 0000000..ab77cb2 --- /dev/null +++ b/app/scripts/ruby_prof.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'ruby-prof' +require_relative '../services/json_importer' + +RubyProf.measure_mode = RubyProf::WALL_TIME + +INITIAL_DATA_FILE = 'fixtures/small.json' +OUTPUT_DIR = 'tmp/data' + + +def flat_profile + run_profiler do |result| + printer = RubyProf::FlatPrinterWithLineNumbers.new(result) + printer.print(File.open("#{OUTPUT_DIR}/ruby_prof_flat_demo.txt", 'w+')) + end +end + +def graph_profile + run_profiler do |result| + printer = RubyProf::GraphHtmlPrinter.new(result) + printer.print(File.open("#{OUTPUT_DIR}/ruby_prof_graph_demo.html", "w+")) + end +end + +def callstack_profile + run_profiler do |result| + printer = RubyProf::CallStackPrinter.new(result) + printer.print(File.open("#{OUTPUT_DIR}/ruby_prof_callstack_demo.html", "w+")) + end +end + +def calltree_profile + run_profiler do |result| + printer = RubyProf::CallTreePrinter.new(result) + printer.print(path: OUTPUT_DIR, profile: 'profile') + end +end + + +def run_profiler + RubyProf.measure_mode = RubyProf::WALL_TIME + result = RubyProf.profile { JsonImporter.new.import_json_to_db(file_path: INITIAL_DATA_FILE) } + yield result +end + +# flat_profile +# graph_profile +# callstack_profile +# calltree_profile \ No newline at end of file diff --git a/app/services/json_importer.rb b/app/services/json_importer.rb new file mode 100644 index 0000000..557b50d --- /dev/null +++ b/app/services/json_importer.rb @@ -0,0 +1,79 @@ + require 'oj' + +class JsonImporter + attr_accessor :services_names_hash, :cities_names_hash, :buses_hash, :trips + def initialize + @services_names_hash = {} + @cities_names_hash = {} + @buses_hash = {} + @trips = [] + end + + def import_json_to_db(file_path:) + # byebug + json = Oj.load_file(file_path) + + ActiveRecord::Base.transaction do + delete_existing_records + create_cities_and_services(json) + create_buses_with_services(json) + create_trips(json) + end + end + + def create_cities_and_services(json) + json.each do |trip| + cities_names_hash[trip['from']] ||= City.new(name: trip['from']) + cities_names_hash[trip['to']] ||= City.new(name: trip['to']) + trip['bus']['services'].each do |service| + services_names_hash[service] = Service.new(name: service) + end + end + # byebug + City.import cities_names_hash.values, syncronize: true, raise_error: true + # byebug + Service.import services_names_hash.values, syncronize: true, raise_error: true + end + + def create_buses_with_services(json) + buses_numbers = [] + json.each do |trip| + next if buses_numbers.include?(trip['bus']['number']) + bus = Bus.new( + number: trip['bus']['number'], + model: trip['bus']['model'] + ) + bus.services = services_names_hash.values_at(*trip['bus']['services']) + + buses_hash[trip['bus']['number']] = bus + buses_numbers << trip['bus']['number'] + end + # byebug + Bus.import buses_hash.values, recursive: true, syncronize: true,raise_error: true + end + + def create_trips(json) + json.each do |trip| + from = cities_names_hash[trip['from']] + to = cities_names_hash[trip['to']] + bus = buses_hash[trip['bus']['number']] + trips << Trip.new( + from: from, + to: to, + bus: bus, + start_time: trip['start_time'], + duration_minutes: trip['duration_minutes'], + price_cents: trip['price_cents'] + ) + end + Trip.import trips, raise_error: true + end + + def delete_existing_records + City.delete_all + Bus.delete_all + Service.delete_all + Trip.delete_all + ActiveRecord::Base.connection.execute('delete from buses_services;') + end +end diff --git a/app/views/trips/_delimiter.html.erb b/app/views/trips/_delimiter.html.erb deleted file mode 100644 index 3f845ad..0000000 --- a/app/views/trips/_delimiter.html.erb +++ /dev/null @@ -1 +0,0 @@ -==================================================== diff --git a/app/views/trips/_service.html.erb b/app/views/trips/_service.html.erb deleted file mode 100644 index 178ea8c..0000000 --- a/app/views/trips/_service.html.erb +++ /dev/null @@ -1 +0,0 @@ -
  • <%= "#{service.name}" %>
    • diff --git a/app/views/trips/_services.html.erb b/app/views/trips/_services.html.erb deleted file mode 100644 index 2de639f..0000000 --- a/app/views/trips/_services.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -
  • Сервисы в автобусе:
    • - diff --git a/app/views/trips/_trip.html.erb b/app/views/trips/_trip.html.erb index fa1de9a..c04dcf0 100644 --- a/app/views/trips/_trip.html.erb +++ b/app/views/trips/_trip.html.erb @@ -1,5 +1,16 @@ -
  • <%= "Отправление: #{trip.start_time}" %>
    • -
  • <%= "Прибытие: #{(Time.parse(trip.start_time) + trip.duration_minutes.minutes).strftime('%H:%M')}" %>
    • -
  • <%= "В пути: #{trip.duration_minutes / 60}ч. #{trip.duration_minutes % 60}мин." %>
    • -
  • <%= "Цена: #{trip.price_cents / 100}р. #{trip.price_cents % 100}коп." %>
    • -
  • <%= "Автобус: #{trip.bus.model} №#{trip.bus.number}" %>
    • + +==================================================== diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce4..0df45dc 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -2,15 +2,7 @@ <%= "Автобусы #{@from.name} – #{@to.name}" %>

    - <%= "В расписании #{@trips.count} рейсов" %> + <%= "В расписании #{@trips.size} рейсов" %>

    -<% @trips.each do |trip| %> - - <%= render "delimiter" %> -<% end %> +<%= render partial: 'trip', collection: @trips %> diff --git a/config/application.rb b/config/application.rb index 9c33109..e094ea9 100644 --- a/config/application.rb +++ b/config/application.rb @@ -15,5 +15,10 @@ class Application < Rails::Application # Application configuration can go into files in config/initializers # -- all .rb files in that directory are automatically loaded after loading # the framework and any gems in your application. + config.autoload_paths << Rails.root.join('services') + + config.after_initialize do |app| + app.config.paths.add 'app/services', :eager_load => true + end end end diff --git a/config/newrelic.yml b/config/newrelic.yml new file mode 100644 index 0000000..f724f14 --- /dev/null +++ b/config/newrelic.yml @@ -0,0 +1,47 @@ +# +# This file configures the New Relic Agent. New Relic monitors Ruby, Java, +# .NET, PHP, Python, Node, and Go applications with deep visibility and low +# overhead. For more information, visit www.newrelic.com. +# +# Generated May 06, 2019 +# +# This configuration file is custom generated for Selfemployed_57 +# +# For full documentation of agent configuration options, please refer to +# https://docs.newrelic.com/docs/agents/ruby-agent/installation-configuration/ruby-agent-configuration + +common: &default_settings + # Required license key associated with your New Relic account. + license_key: ENV["NEWRELIC_LICENSE_KEY"] + # c4e0921d5f0dc7a300da9c9c5d58f6e94f26b8f1 + + # Your application name. Renaming here affects where data displays in New + # Relic. For more details, see https://docs.newrelic.com/docs/apm/new-relic-apm/maintenance/renaming-applications + app_name: Bus Tours + + # To disable the agent regardless of other settings, uncomment the following: + # agent_enabled: false + + # Logging level for log/newrelic_agent.log + log_level: info + + +# Environment-specific settings are in this section. +# RAILS_ENV or RACK_ENV (as appropriate) is used to determine the environment. +# If your application has other named environments, configure them here. +development: + <<: *default_settings + app_name: Bus Tours (Development) + monitor_mode: true + +test: + <<: *default_settings + # It doesn't make sense to report to New Relic from automated test runs. + monitor_mode: false + +staging: + <<: *default_settings + app_name: Bus Tours (Staging) + +production: + <<: *default_settings diff --git a/config/routes.rb b/config/routes.rb index a2da6a7..4cc70a9 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,4 +1,6 @@ Rails.application.routes.draw do + mount PgHero::Engine, at: "pghero" + # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html get "/" => "statistics#index" get "автобусы/:from/:to" => "trips#index" diff --git a/db/migrate/20190506035213_add_indicies_to_trips.rb b/db/migrate/20190506035213_add_indicies_to_trips.rb new file mode 100644 index 0000000..e91caeb --- /dev/null +++ b/db/migrate/20190506035213_add_indicies_to_trips.rb @@ -0,0 +1,7 @@ +class AddIndiciesToTrips < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + def change + add_index :trips, %i[from_id to_id], algorithm: :concurrently + add_index :trips, :start_time, algorithm: :concurrently + end +end diff --git a/db/migrate/20190506044704_add_foreign_key_to_trips.rb b/db/migrate/20190506044704_add_foreign_key_to_trips.rb new file mode 100644 index 0000000..6fc40b8 --- /dev/null +++ b/db/migrate/20190506044704_add_foreign_key_to_trips.rb @@ -0,0 +1,6 @@ +class AddForeignKeyToTrips < ActiveRecord::Migration[5.2] + def change + add_foreign_key :trips, :cities, column: :from_id, on_delete: :cascade + add_foreign_key :trips, :cities, column: :to_id, on_delete: :cascade + end +end diff --git a/db/migrate/20190506051147_add_more_indicies.rb b/db/migrate/20190506051147_add_more_indicies.rb new file mode 100644 index 0000000..f6514d4 --- /dev/null +++ b/db/migrate/20190506051147_add_more_indicies.rb @@ -0,0 +1,8 @@ +class AddMoreIndicies < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + def change + add_index :cities, :name, unique: true, algorithm: :concurrently + add_index :buses, :number, unique: true, algorithm: :concurrently + add_index :services, :name, unique: true, algorithm: :concurrently + end +end diff --git a/db/migrate/20190512223139_add_foreign_key_to_buses_services.rb b/db/migrate/20190512223139_add_foreign_key_to_buses_services.rb new file mode 100644 index 0000000..57225a1 --- /dev/null +++ b/db/migrate/20190512223139_add_foreign_key_to_buses_services.rb @@ -0,0 +1,6 @@ +class AddForeignKeyToBusesServices < ActiveRecord::Migration[5.2] + def change + add_foreign_key :buses_services, :buses, column: :bus_id, on_delete: :cascade + add_foreign_key :buses_services, :services, column: :service_id, on_delete: :cascade + end +end diff --git a/db/migrate/20190513023411_remove_fk_from_buses_services.rb b/db/migrate/20190513023411_remove_fk_from_buses_services.rb new file mode 100644 index 0000000..78927ce --- /dev/null +++ b/db/migrate/20190513023411_remove_fk_from_buses_services.rb @@ -0,0 +1,6 @@ +class RemoveFkFromBusesServices < ActiveRecord::Migration[5.2] + def change + remove_foreign_key :buses_services, column: :bus_id + remove_foreign_key :buses_services, column: :service_id + end +end diff --git a/db/schema.rb b/db/schema.rb index f6921e4..dbf98ae 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,14 +10,17 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_03_30_193044) do +ActiveRecord::Schema.define(version: 2019_05_13_023411) do # These are extensions that must be enabled in order to support this database + enable_extension "pg_stat_statements" + enable_extension "pgcrypto" enable_extension "plpgsql" create_table "buses", force: :cascade do |t| t.string "number" t.string "model" + t.index ["number"], name: "index_buses_on_number", unique: true end create_table "buses_services", force: :cascade do |t| @@ -27,10 +30,23 @@ create_table "cities", force: :cascade do |t| t.string "name" + t.index ["name"], name: "index_cities_on_name", unique: true + end + + create_table "pghero_query_stats", force: :cascade do |t| + t.text "database" + t.text "user" + t.text "query" + t.bigint "query_hash" + t.float "total_time" + t.bigint "calls" + t.datetime "captured_at" + t.index ["database", "captured_at"], name: "index_pghero_query_stats_on_database_and_captured_at" end create_table "services", force: :cascade do |t| t.string "name" + t.index ["name"], name: "index_services_on_name", unique: true end create_table "trips", force: :cascade do |t| @@ -40,6 +56,10 @@ t.integer "duration_minutes" t.integer "price_cents" t.integer "bus_id" + t.index ["from_id", "to_id"], name: "index_trips_on_from_id_and_to_id" + t.index ["start_time"], name: "index_trips_on_start_time" end + add_foreign_key "trips", "cities", column: "from_id", on_delete: :cascade + add_foreign_key "trips", "cities", column: "to_id", on_delete: :cascade end diff --git a/lib/tasks/metric_measurement.rake b/lib/tasks/metric_measurement.rake new file mode 100644 index 0000000..b612532 --- /dev/null +++ b/lib/tasks/metric_measurement.rake @@ -0,0 +1,36 @@ +require 'benchmark' + +desc 'measure performance for optimization rake import_json_benchmark[fixtures/small.json]' +task :import_json_benchmark, [:file_name] => :environment do |_task, args| + result = Benchmark.measure do + puts "\n== Loading data from #{args.file_name} ==" + Benchmark.measure { Rake::Task['reload_json'].invoke(*args) } + end + + puts result +end + +# CPU time, system CPU time, the sum of the user and system CPU times, and the elapsed real time. The unit of time is seconds. + +# initial results +# 2.654352 0.956313 13.612620 ( 14.781368) + +# add indicies to trips +# 12.471791 0.888674 13.360465 ( 14.467896) + +# add indicies to cities, services and buses +# 11.874181 0.867256 12.743072 ( 13.819131) + +# == Loading data from fixtures/small.json == +# Reload complete! +# 14.654087 0.681433 15.337232 ( 16.270607) + +# add Oj and activerecord-import +# == Loading data from fixtures/small.json == +# Reload complete! +# 1.585816 0.067304 1.655319 ( 1.651922) + + +# Loading data from fixtures/large.json == +# Reload complete! +# 33.660827 0.582401 34.245360 (40.213537) \ No newline at end of file diff --git a/lib/tasks/utils.rake b/lib/tasks/utils.rake index 540fe87..e8616b5 100644 --- a/lib/tasks/utils.rake +++ b/lib/tasks/utils.rake @@ -1,34 +1,8 @@ + # Наивная загрузка данных из json-файла в БД # rake reload_json[fixtures/small.json] +desc 'Import data from json file into database rake reload_json[fixtures/small.json]' task :reload_json, [:file_name] => :environment do |_task, args| - json = JSON.parse(File.read(args.file_name)) - - ActiveRecord::Base.transaction do - City.delete_all - Bus.delete_all - Service.delete_all - Trip.delete_all - ActiveRecord::Base.connection.execute('delete from buses_services;') - - json.each do |trip| - from = City.find_or_create_by(name: trip['from']) - to = City.find_or_create_by(name: trip['to']) - services = [] - trip['bus']['services'].each do |service| - s = Service.find_or_create_by(name: service) - services << s - end - bus = Bus.find_or_create_by(number: trip['bus']['number']) - bus.update(model: trip['bus']['model'], services: services) - - Trip.create!( - from: from, - to: to, - bus: bus, - start_time: trip['start_time'], - duration_minutes: trip['duration_minutes'], - price_cents: trip['price_cents'], - ) - end - end -end + JsonImporter.new.import_json_to_db(file_path: args.file_name) + puts 'Reload complete!' +end \ No newline at end of file diff --git a/metrics/after/slow_queries_after.png b/metrics/after/slow_queries_after.png new file mode 100644 index 0000000..5c3a0ef Binary files /dev/null and b/metrics/after/slow_queries_after.png differ diff --git a/metrics/after/slowest_queries_after.png b/metrics/after/slowest_queries_after.png new file mode 100644 index 0000000..10263cf Binary files /dev/null and b/metrics/after/slowest_queries_after.png differ diff --git a/metrics/after/transactions_after.png b/metrics/after/transactions_after.png new file mode 100644 index 0000000..f4827d5 Binary files /dev/null and b/metrics/after/transactions_after.png differ diff --git a/metrics/before/Screen Shot 2019-05-05 at 10.07.13 PM.png b/metrics/before/Screen Shot 2019-05-05 at 10.07.13 PM.png new file mode 100644 index 0000000..046122f Binary files /dev/null and b/metrics/before/Screen Shot 2019-05-05 at 10.07.13 PM.png differ diff --git a/metrics/before/Screen Shot 2019-05-05 at 10.07.22 PM.png b/metrics/before/Screen Shot 2019-05-05 at 10.07.22 PM.png new file mode 100644 index 0000000..e75827d Binary files /dev/null and b/metrics/before/Screen Shot 2019-05-05 at 10.07.22 PM.png differ diff --git a/metrics/before/front_end_services_before.png b/metrics/before/front_end_services_before.png new file mode 100644 index 0000000..8c97933 Binary files /dev/null and b/metrics/before/front_end_services_before.png differ diff --git a/metrics/before/pg_hero_advice.png b/metrics/before/pg_hero_advice.png new file mode 100644 index 0000000..eda590c Binary files /dev/null and b/metrics/before/pg_hero_advice.png differ diff --git a/metrics/before/slow_queries.png b/metrics/before/slow_queries.png new file mode 100644 index 0000000..8f048df Binary files /dev/null and b/metrics/before/slow_queries.png differ diff --git a/metrics/before/slowest_queries.png b/metrics/before/slowest_queries.png new file mode 100644 index 0000000..7789dbb Binary files /dev/null and b/metrics/before/slowest_queries.png differ diff --git a/metrics/case_study_english.md b/metrics/case_study_english.md new file mode 100644 index 0000000..ffe035f --- /dev/null +++ b/metrics/case_study_english.md @@ -0,0 +1,149 @@ +# We had a problem of loading large json files as well as rendering web pages fast with the given amount of data. + +## Main goal is: + - Optimize program to load large.json file within 1(one) minute. + - Find out how we ca n render schedule pages faster + +## Tools I used to solve the problem: + - gem pghero + - newrelic + - gem activerecord-import + - gem strong_migrations + - benchmark + - apache benchmark testing + - siege + +![pg_hero](before/pg_hero_advice.png) + +## For given feedback loop on database level I did the following: + - desided to start with `small.json` in order to increase feedback loop + - extracted logic from rake tasks in it's own class so that it's easier to test. + - wrote test for loadeing `.json` files + - by implementing any changes on database level I made sure tests are passing and my metrics from new relic and benchmark metrics show better results + +## Let\'s look at `Benchmark.measure` numbers showing the following: +- CPU time +- system CPU time +- the sum of the user and system CPU time +- the elapsed real time. The unit of time is seconds. + +#### Initial results of running `be rake 'import_json_benchmark[fixtures/small.json]'`: +``` + 2.654352 0.956313 13.612620 (14.781368) +``` + +![slow_queries](before/slow_queries.png) +![slowest_queries](before/slowest_queries.png) + + +#### Results of running `be rake 'import_json_benchmark[fixtures/small.json]'` after optimization and using `gem oj`, `gem activerecord-import`, adding necessary indicies: +``` +1.585816 0.067304 1.655319 (1.651922) +``` + +#### This improvement allowed me to load `fixtures/large.json` file with the following metrics: +``` + 33.660827 0.582401 34.245360 (40.213537) +``` + + +### Page load metrics +### Newrelic +![slowest_queries](before/front_end_services_before.png) +#### Apache benchmark +1. With indicies but without front end optimization + +``` +ab -n 10 -c 1 http://localhost:3000/автобусы/Самара/Москва + + +Server Software: +Server Hostname: localhost +Server Port: 3000 + +Document Path: /%D0%B0%D0%B2%D1%82%D0%BE%D0%B1%D1%83%D1%81%D1%8B/%D0%A1%D0%B0%D0%BC%D0%B0%D1%80%D0%B0/%D0%9C%D0%BE%D1%81%D0%BA%D0%B2%D0%B0 +Document Length: 538780 bytes + +Concurrency Level: 1 +Time taken for tests: 247.233 seconds +Complete requests: 10 +Failed requests: 0 +Total transferred: 5395342 bytes +HTML transferred: 5387800 bytes +Requests per second: 0.04 [#/sec] (mean) +Time per request: 24723.295 [ms] (mean) +Time per request: 24723.295 [ms] (mean, across all concurrent requests) +Transfer rate: 21.31 [Kbytes/sec] received + +Connection Times (ms) + min mean[+/-sd] median max +Connect: 0 0 0.0 0 0 +Processing: 22301 24723 948.8 25043 25566 +Waiting: 22300 24723 948.9 25043 25566 +Total: 22301 24723 948.8 25043 25566 + +Percentage of the requests served within a certain time (ms) + 50% 25043 + 66% 25186 + 75% 25199 + 80% 25265 + 90% 25566 + 95% 25566 + 98% 25566 + 99% 25566 + 100% 25566 (longest request) + ```` + + +2. After removing partials and adding collections +``` +ab -n 10 -c 1 http://localhost:3000/автобусы/Самара/Москва +Server Software: +Server Hostname: localhost +Server Port: 3000 + +Document Path: /%D0%B0%D0%B2%D1%82%D0%BE%D0%B1%D1%83%D1%81%D1%8B/%D0%A1%D0%B0%D0%BC%D0%B0%D1%80%D0%B0/%D0%9C%D0%BE%D1%81%D0%BA%D0%B2%D0%B0 +Document Length: 569667 bytes + +Concurrency Level: 1 +Time taken for tests: 45.169 seconds +Complete requests: 10 +Failed requests: 0 +Total transferred: 5704204 bytes +HTML transferred: 5696670 bytes +Requests per second: 0.22 [#/sec] (mean) +Time per request: 4516.931 [ms] (mean) +Time per request: 4516.931 [ms] (mean, across all concurrent requests) +Transfer rate: 123.33 [Kbytes/sec] received + +Connection Times (ms) + min mean[+/-sd] median max +Connect: 0 0 0.0 0 0 +Processing: 3838 4517 448.1 4523 5575 +Waiting: 3838 4516 448.1 4522 5575 +Total: 3838 4517 448.1 4523 5575 + +Percentage of the requests served within a certain time (ms) + 50% 4523 + 66% 4573 + 75% 4583 + 80% 4725 + 90% 5575 + 95% 5575 + 98% 5575 + 99% 5575 + 100% 5575 (longest request) + ```` + +As we can see from the results above after getting rid of partials and rendering partial with collection +- Time taken for tests reduced from 247.233 seconds to 45.169 seconds +- Time per request reducded from 24723.295 [ms] (mean) to 4516.931 [ms] (mean) +- Requests per second increased from 0.04 [#/sec] to 0.22 [#/sec] (mean) + + + +On the screenshots below we can observe that slowest queries and transactions metrics were improved as well: + +![slowest_queries](after/slowest_queries_after.png) +![slow_queries](after/slow_queries_after.png) +![transactions](after/transactions_after.png) \ No newline at end of file diff --git a/spec/controllers/trips_controller_spec.rb b/spec/controllers/trips_controller_spec.rb new file mode 100644 index 0000000..2f89d36 --- /dev/null +++ b/spec/controllers/trips_controller_spec.rb @@ -0,0 +1,21 @@ +require 'rails_helper' + +RSpec.describe TripsController, type: :controller do + describe 'GET #index' do + before do + JsonImporter.new().import_json_to_db(file_path: 'spec/fixtures/data.json') + get :index, params: { from: 'Сочи', to: 'Тула'} + end + + it 'populates correct data' do + Trip.where(from: 'Сочи', to: 'Тула').find_each do |trip| + expect(response.body).to include?("Отправление: #{trip.start_time}") + expect(response.body).to include?("Автобус: #{trip.bus.model} №#{trip.bus.number}") + end + end + + it 'renders index view' do + expect(response).to render_template :index + end + end +end diff --git a/spec/fixtures/data.json b/spec/fixtures/data.json new file mode 100644 index 0000000..ac21537 --- /dev/null +++ b/spec/fixtures/data.json @@ -0,0 +1,89 @@ +[{ + "from": "Сочи", + "to": "Тула", + "start_time": "16:11", + "duration_minutes": 83, + "price_cents": 23354, + "bus": { + "number": "229", + "model": "Икарус", + "services": ["Ремни безопасности", "Кондиционер общий", "Кондиционер Индивидуальный", "Телевизор индивидуальный", "Стюардесса", "Можно не печатать билет"] + } +}, { + "from": "Самара", + "to": "Самара", + "start_time": "13:13", + "duration_minutes": 572, + "price_cents": 83861, + "bus": { + "number": "912", + "model": "Вольво", + "services": ["WiFi", "Работающий туалет", "Телевизор индивидуальный", "Стюардесса", "Можно не печатать билет"] + } +}, { + "from": "Красноярск", + "to": "Волгоград", + "start_time": "13:48", + "duration_minutes": 186, + "price_cents": 80288, + "bus": { + "number": "584", + "model": "ГАЗ", + "services": ["Кондиционер общий"] + } +}, { + "from": "Рыбинск", + "to": "Саратов", + "start_time": "15:13", + "duration_minutes": 271, + "price_cents": 6803, + "bus": { + "number": "739", + "model": "УАЗ", + "services": ["WiFi", "Работающий туалет", "Ремни безопасности", "Кондиционер общий", "Кондиционер Индивидуальный", "Телевизор общий", "Телевизор индивидуальный", "Стюардесса", "Можно не печатать билет"] + } +}, { + "from": "Тула", + "to": "Саратов", + "start_time": "18:24", + "duration_minutes": 67, + "price_cents": 87845, + "bus": { + "number": "811", + "model": "Вольво", + "services": ["Работающий туалет"] + } +}, { + "from": "Москва", + "to": "Самара", + "start_time": "16:15", + "duration_minutes": 214, + "price_cents": 62185, + "bus": { + "number": "387", + "model": "Буханка", + "services": ["Туалет", "Ремни безопасности"] + } +}, { + "from": "Тула", + "to": "Саратов", + "start_time": "18:27", + "duration_minutes": 269, + "price_cents": 95089, + "bus": { + "number": "974", + "model": "ГАЗ", + "services": ["WiFi"] + } +}, { + "from": "Сочи", + "to": "Самара", + "start_time": "21:10", + "duration_minutes": 479, + "price_cents": 26945, + "bus": { + "number": "379", + "model": "Газель", + "services": ["WiFi", "Туалет", "Работающий туалет", "Кондиционер общий", "Кондиционер Индивидуальный", "Телевизор индивидуальный", "Стюардесса", "Можно не печатать билет"] + } +}] \ No newline at end of file diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb new file mode 100644 index 0000000..b5fed83 --- /dev/null +++ b/spec/rails_helper.rb @@ -0,0 +1,61 @@ +# This file is copied to spec/ when you run 'rails generate rspec:install' +require 'rails_helper' +ENV['RAILS_ENV'] ||= 'test' +require File.expand_path('../../config/environment', __FILE__) +# Prevent database truncation if the environment is production +abort("The Rails environment is running in production mode!") if Rails.env.production? +require 'rspec/rails' +# Add additional requires below this line. Rails is not loaded until this point! + +# Requires supporting ruby files with custom matchers and macros, etc, in +# spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are +# run as spec files by default. This means that files in spec/support that end +# in _spec.rb will both be required and run as specs, causing the specs to be +# run twice. It is recommended that you do not name files matching this glob to +# end with _spec.rb. You can configure this pattern with the --pattern +# option on the command line or in ~/.rspec, .rspec or `.rspec-local`. +# +# The following line is provided for convenience purposes. It has the downside +# of increasing the boot-up time by auto-requiring all files in the support +# directory. Alternatively, in the individual `*_spec.rb` files, manually +# require only the support files necessary. +# +# Dir[Rails.root.join('spec', 'support', '**', '*.rb')].each { |f| require f } + +# Checks for pending migrations and applies them before tests are run. +# If you are not using ActiveRecord, you can remove these lines. +begin + ActiveRecord::Migration.maintain_test_schema! +rescue ActiveRecord::PendingMigrationError => e + puts e.to_s.strip + exit 1 +end +RSpec.configure do |config| + # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures + config.fixture_path = "#{::Rails.root}/spec/fixtures" + + # If you're not using ActiveRecord, or you'd prefer not to run each of your + # examples within a transaction, remove the following line or assign false + # instead of true. + config.use_transactional_fixtures = true + + # RSpec Rails can automatically mix in different behaviours to your tests + # based on their file location, for example enabling you to call `get` and + # `post` in specs under `spec/controllers`. + # + # You can disable this behaviour by removing the line below, and instead + # explicitly tag your specs with their type, e.g.: + # + # RSpec.describe UsersController, :type => :controller do + # # ... + # end + # + # The different available types are documented in the features, such as in + # https://relishapp.com/rspec/rspec-rails/docs + config.infer_spec_type_from_file_location! + + # Filter lines from Rails gems in backtraces. + config.filter_rails_from_backtrace! + # arbitrary gems may also be filtered via: + # config.filter_gems_from_backtrace("gem name") +end diff --git a/spec/services/json_importer_spec.rb b/spec/services/json_importer_spec.rb new file mode 100644 index 0000000..a01ea25 --- /dev/null +++ b/spec/services/json_importer_spec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' +describe 'Services::JsonImporter' do + describe '#import_json_to_db', focus: true do + it 'imports eight records of Trip into db', focus: true do + expect { import_file }.to change(Trip, :count).by(8) + end + + it 'has correct data' do + import_file + actual_result = Trip.first.to_h + expected_result = { + from: "Сочи", + to: "Тула", + start_time: "16:11", + duration_minutes: 83, + price_cents: 23354, + bus: { + number: "229", + model: "Икарус", + services: ["Ремни безопасности", "Кондиционер общий", "Кондиционер Индивидуальный", "Телевизор индивидуальный", "Стюардесса", "Можно не печатать билет"] + } + } + end + + # expect(actual_result).to eq(expected_result) + end + + def import_file + JsonImporter.new().import_json_to_db(file_path: 'spec/fixtures/data.json') + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..7954854 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,97 @@ +# This file was generated by the `rails generate rspec:install` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + config.warnings = false + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ + # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ + # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode + config.disable_monkey_patching! + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end