diff --git a/Gemfile b/Gemfile index b704305..b7246c0 100644 --- a/Gemfile +++ b/Gemfile @@ -10,7 +10,12 @@ if defined?(JRUBY_VERSION) else gem 'sqlite3' gem 'mysql2' - gem 'pg' + if RUBY_VERSION == '1.8.7' + gem 'pg', '0.17.1' + gem 'i18n', '0.6.11' + else + gem 'pg' + end if rails_version =~ /(^|[^.\d])(2|3\.0)\.\d+/ gem 'activerecord-mysql2-adapter' end diff --git a/README.markdown b/README.markdown index b8face9..aa52889 100644 --- a/README.markdown +++ b/README.markdown @@ -145,6 +145,8 @@ reset database: rake dump:restore MIGRATE_DOWN=reset +`REBUILD_INDEXES` — remove indexes for each table before restore, and create them after restore if you pass "1", "true" or "yes". `REBUILD_INDEXES` is useful to speed up restoring for large tables (see https://github.com/toy/dump/pull/12#issuecomment-69462275), but may affect index structure because of database adapters implementations. + `RESTORE_SCHEMA` — don't read/change schema if you pass "0", "no" or "false" (useful to just restore data for table; note that schema info tables are also not restored) don't restore schema: diff --git a/lib/dump/env.rb b/lib/dump/env.rb index 88b7baa..f6cf70b 100644 --- a/lib/dump/env.rb +++ b/lib/dump/env.rb @@ -16,6 +16,7 @@ module Env :backup => %w[BACKUP AUTOBACKUP AUTO_BACKUP], :transfer_via => %w[TRANSFER_VIA], :migrate_down => %w[MIGRATE_DOWN], + :rebuild_indexes => %w[REBUILD_INDEXES], :restore_schema => %w[RESTORE_SCHEMA], :restore_tables => %w[RESTORE_TABLES], :restore_assets => %w[RESTORE_ASSETS], @@ -33,6 +34,7 @@ module Env :backup => 'no autobackup if you pass "0", "no" or "false"', :transfer_via => 'transfer method (rsync, sftp or scp)', :migrate_down => 'don\'t run down for migrations not present in dump if you pass "0", "no" or "false"; pass "reset" to recreate (drop and create) db', + :rebuild_indexes => 'remove indexes before restore, and add them after if you pass "1", "true" or "yes"; speed up restoring for large tables, but may affect indexes structure (see README for details)', :restore_schema => 'don\'t read/change schema if you pass "0", "no" or "false" (useful to just restore data for table; note that schema info tables are also not restored)', :restore_tables => 'works as TABLES, but for restoring', :restore_assets => 'works as ASSETS, but for restoring', diff --git a/lib/dump/reader.rb b/lib/dump/reader.rb index 3fbf87a..16b821b 100644 --- a/lib/dump/reader.rb +++ b/lib/dump/reader.rb @@ -5,6 +5,8 @@ require 'rake' require 'zlib' require 'tempfile' +require 'dump/reader/summary' +require 'dump/reader/assets' module Dump # Reading dump @@ -19,35 +21,11 @@ def self.restore(path) dump.read_schema dump.read_tables - dump.read_assets + Assets.new(dump).read end end end - # Helper class for building summary of dump - class Summary - attr_reader :text - alias_method :to_s, :text - def initialize - @text = '' - end - - def header(header) - @text << " #{header}:\n" - end - - def data(entries) - entries.each do |entry| - @text << " #{entry}\n" - end - end - - # from ActionView::Helpers::TextHelper - def self.pluralize(count, singular) - "#{count} #{count == 1 ? singular : singular.pluralize}" - end - end - def self.summary(path, options = {}) new(path).open do |dump| dump.read_config @@ -98,7 +76,8 @@ def open def find_entry(matcher) stream.each do |entry| if entry.full_name.match(matcher) - # we can not return entry - after exiting stream.each the entry will be invalid and will read from tar start + # we can not return entry - after exiting stream.each + # the entry will be invalid and will read from tar start return yield(entry) end end @@ -186,109 +165,42 @@ def read_tables end end + def rebuild_indexes? + Dump::Env.yes?(:rebuild_indexes) + end + def read_table(table, rows_count) find_entry("#{table}.dump") do |entry| table_sql = quote_table_name(table) clear_table(table_sql) - columns = Marshal.load(entry) - columns_sql = columns_insert_sql(columns) - Progress.start(table, rows_count) do - until entry.eof? - rows_sql = [] - 1000.times do - rows_sql << values_insert_sql(Marshal.load(entry)) unless entry.eof? - end - - begin - insert_into_table(table_sql, columns_sql, rows_sql) - Progress.step(rows_sql.length) - rescue - rows_sql.each do |row_sql| - insert_into_table(table_sql, columns_sql, row_sql) - Progress.step - end - end + columns_sql = columns_insert_sql(Marshal.load(entry)) + if rebuild_indexes? + with_disabled_indexes table do + bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql) end + else + bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql) end fix_sequence!(table) end end - def read_assets - return if Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty? - return if config[:assets].blank? - - assets = config[:assets] - if assets.is_a?(Hash) - assets_count = assets.values.sum{ |value| value.is_a?(Hash) ? value[:total] : value } - assets_paths = assets.keys - else - assets_count, assets_paths = nil, assets - end - - if Dump::Env[:restore_assets] - assets_paths.each do |asset| - Dump::Assets.glob_asset_children(asset, '**/*').reverse.each do |child| - next unless read_asset?(child, Dump.rails_root) - case - when File.file?(child) - File.unlink(child) - when File.directory?(child) - begin - Dir.unlink(child) - rescue Errno::ENOTEMPTY - nil - end - end - end - end - else - Dump::Env.with_env(:assets => assets_paths.join(':')) do - Rake::Task['assets:delete'].invoke - end - end - - read_assets_entries(assets_paths, assets_count) do |stream, root, entry, prefix| - if !Dump::Env[:restore_assets] || read_asset?(entry.full_name, prefix) - stream.extract_entry(root, entry) - end - end - end - - def read_asset?(path, prefix) - Dump::Env.filter(:restore_assets, Dump::Assets::SPLITTER).custom_pass? do |value| - File.fnmatch(File.join(prefix, value), path) || - File.fnmatch(File.join(prefix, value, '**'), path) - end - end - - def read_assets_entries(_assets_paths, assets_count) - Progress.start('Assets', assets_count || 1) do - found_assets = false - # old style - in separate tar - find_entry('assets.tar') do |assets_tar| - def assets_tar.rewind - # rewind will fail - it must go to center of gzip - # also we don't need it - this is last step in dump restore + def bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql) + Progress.start(table, rows_count) do + until entry.eof? + rows_sql = [] + 1000.times do + rows_sql << values_insert_sql(Marshal.load(entry)) unless entry.eof? end - Archive::Tar::Minitar.open(assets_tar) do |inp| - inp.each do |entry| - yield inp, Dump.rails_root, entry, nil - Progress.step if assets_count - end - end - found_assets = true - end - unless found_assets - # new style - in same tar - assets_root_link do |tmpdir, prefix| - stream.each do |entry| - if entry.full_name.starts_with?("#{prefix}/") - yield stream, tmpdir, entry, prefix - Progress.step if assets_count - end + begin + insert_into_table(table_sql, columns_sql, rows_sql) + Progress.step(rows_sql.length) + rescue + rows_sql.each do |row_sql| + insert_into_table(table_sql, columns_sql, row_sql) + Progress.step end end end diff --git a/lib/dump/reader/assets.rb b/lib/dump/reader/assets.rb new file mode 100644 index 0000000..32de89f --- /dev/null +++ b/lib/dump/reader/assets.rb @@ -0,0 +1,93 @@ +module Dump + class Reader < Snapshot + # Helper class for reading assets + class Assets + attr_reader :dump + delegate :config, :to => :dump + + def initialize(dump) + @dump = dump + end + + def read + return if Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty? + return if config[:assets].blank? + + assets = config[:assets] + if assets.is_a?(Hash) + assets_count = assets.values.sum{ |value| value.is_a?(Hash) ? value[:total] : value } + assets_paths = assets.keys + else + assets_count, assets_paths = nil, assets + end + + if Dump::Env[:restore_assets] + assets_paths.each do |asset| + Dump::Assets.glob_asset_children(asset, '**/*').reverse.each do |child| + next unless read_asset?(child, Dump.rails_root) + case + when File.file?(child) + File.unlink(child) + when File.directory?(child) + begin + Dir.unlink(child) + rescue Errno::ENOTEMPTY + nil + end + end + end + end + else + Dump::Env.with_env(:assets => assets_paths.join(':')) do + Rake::Task['assets:delete'].invoke + end + end + + read_assets_entries(assets_paths, assets_count) do |stream, root, entry, prefix| + if !Dump::Env[:restore_assets] || read_asset?(entry.full_name, prefix) + stream.extract_entry(root, entry) + end + end + end + + def read_assets_entries(_assets_paths, assets_count) + Progress.start('Assets', assets_count || 1) do + found_assets = false + # old style - in separate tar + dump.find_entry('assets.tar') do |assets_tar| + def assets_tar.rewind + # rewind will fail - it must go to center of gzip + # also we don't need it - this is last step in dump restore + end + Archive::Tar::Minitar.open(assets_tar) do |inp| + inp.each do |entry| + yield inp, Dump.rails_root, entry, nil + Progress.step if assets_count + end + end + found_assets = true + end + + unless found_assets + # new style - in same tar + dump.send :assets_root_link do |tmpdir, prefix| + dump.stream.each do |entry| + if entry.full_name.starts_with?("#{prefix}/") + yield dump.stream, tmpdir, entry, prefix + Progress.step if assets_count + end + end + end + end + end + end + + def read_asset?(path, prefix) + Dump::Env.filter(:restore_assets, Dump::Assets::SPLITTER).custom_pass? do |value| + File.fnmatch(File.join(prefix, value), path) || + File.fnmatch(File.join(prefix, value, '**'), path) + end + end + end + end +end diff --git a/lib/dump/reader/summary.rb b/lib/dump/reader/summary.rb new file mode 100644 index 0000000..79e4063 --- /dev/null +++ b/lib/dump/reader/summary.rb @@ -0,0 +1,27 @@ +module Dump + class Reader < Snapshot + # Helper class for building summary of dump + class Summary + attr_reader :text + alias_method :to_s, :text + def initialize + @text = '' + end + + def header(header) + @text << " #{header}:\n" + end + + def data(entries) + entries.each do |entry| + @text << " #{entry}\n" + end + end + + # from ActionView::Helpers::TextHelper + def self.pluralize(count, singular) + "#{count} #{count == 1 ? singular : singular.pluralize}" + end + end + end +end diff --git a/lib/dump/table_manipulation.rb b/lib/dump/table_manipulation.rb index ba02b66..e44dc35 100644 --- a/lib/dump/table_manipulation.rb +++ b/lib/dump/table_manipulation.rb @@ -29,6 +29,35 @@ def clear_table(table_sql) connection.delete("DELETE FROM #{table_sql}", 'Clearing table') end + def with_disabled_indexes(table, &block) + table_indexes = ActiveRecord::Base.connection.indexes(table) + remove_indexes(table_indexes) + block.call + add_indexes(table_indexes) + end + + def remove_indexes(indexes) + indexes.each do |index| + ActiveRecord::Base.connection.remove_index index.table, :name => index.name + end + end + + VALID_INDEX_OPTIONS = [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type].freeze + + def index_options(index) + options = VALID_INDEX_OPTIONS.map{ |field| [field, index.members.include?(field) ? index.send(field) : nil] } + non_empty_options = options.select{ |pair| !pair[1].nil? } + non_empty_options << [:length, index.lengths] if index.try(:lengths).present? + + Hash[*non_empty_options.flatten] + end + + def add_indexes(indexes) + indexes.each do |index| + ActiveRecord::Base.connection.add_index index.table, index.columns, index_options(index) + end + end + def insert_into_table(table_sql, columns_sql, values_sql) values_sql = values_sql.join(',') if values_sql.is_a?(Array) sql = "INSERT INTO #{table_sql} #{columns_sql} VALUES #{values_sql}" diff --git a/spec/dump/reader_spec.rb b/spec/dump/reader_spec.rb index 42eeda9..33b543a 100644 --- a/spec/dump/reader_spec.rb +++ b/spec/dump/reader_spec.rb @@ -22,7 +22,10 @@ expect(@dump).to receive(:migrate_down).ordered expect(@dump).to receive(:read_schema).ordered expect(@dump).to receive(:read_tables).ordered - expect(@dump).to receive(:read_assets).ordered + + @assets = double('assets') + expect(Dump::Reader::Assets).to receive(:new).with(@dump).and_return(@assets).ordered + expect(@assets).to receive(:read).ordered described_class.restore('/abc/123.tmp') end @@ -478,190 +481,224 @@ def create_entry(rows_count) expect(@dump).to receive(:insert_into_table).with('`first`', '(`id`, `name`)', @rows.map(&:inspect)) @dump.read_table('first', 100) end - end - end - - describe 'read_assets' do - before do - @task = double('task') - allow(Rake::Task).to receive(:[]).with('assets:delete').and_return(@task) - allow(@task).to receive(:invoke) - allow(@dump).to receive(:assets_root_link).and_yield('/tmp', 'assets') - end - it 'does not read assets if config[:assets] is nil' do - allow(@dump).to receive(:config).and_return({}) - expect(@dump).not_to receive(:find_entry) - @dump.read_assets - end + it 'should remove indexes around reading/writing table in case of rebuild indexes' do + create_entry(100) + allow(@dump).to receive(:clear_table) + allow(@dump).to receive(:insert_into_table) + expect(@dump).to receive(:rebuild_indexes?).and_return(true) + expect(@dump).to receive(:with_disabled_indexes).with('first') + @dump.read_table('first', 100) + end - it 'does not read assets if config[:assets] is blank' do - allow(@dump).to receive(:config).and_return({:assets => []}) - expect(@dump).not_to receive(:find_entry) - @dump.read_assets + it 'should not remove indexes around reading/writing table in case of not rebuild indexes' do + create_entry(100) + allow(@dump).to receive(:clear_table) + allow(@dump).to receive(:insert_into_table) + expect(@dump).to receive(:rebuild_indexes?).and_return(false) + expect(@dump).not_to receive(:with_disabled_indexes) + @dump.read_table('first', 100) + end end + end - describe 'deleting existing assets' do + describe 'assets' do + Assets = Reader::Assets + describe Assets do before do - allow(@stream).to receive(:each) + @dump = Dump::Reader.new('123.tgz') + @assets_reader = described_class.new(@dump) + + @e1 = double('e1', :full_name => 'config', :read => 'config_data') + @e2 = double('e2', :full_name => 'first.dump', :read => 'first.dump_data') + @e3 = double('e3', :full_name => 'second.dump', :read => 'second.dump_data') + @stream = [@e1, @e2, @e3] + allow(@dump).to receive(:stream).and_return(@stream) end - it 'calls assets:delete' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) - allow(@dump).to receive(:find_entry) + describe 'read_assets' do + before do + @task = double('task') + allow(Rake::Task).to receive(:[]).with('assets:delete').and_return(@task) + allow(@task).to receive(:invoke) + allow(@dump).to receive(:assets_root_link).and_yield('/tmp', 'assets') + end - expect(@task).to receive(:invoke) + it 'does not read assets if config[:assets] is nil' do + allow(@dump).to receive(:config).and_return({}) + expect(@dump).not_to receive(:find_entry) + @assets_reader.read + end - @dump.read_assets - end + it 'does not read assets if config[:assets] is blank' do + allow(@dump).to receive(:config).and_return({:assets => []}) + expect(@dump).not_to receive(:find_entry) + @assets_reader.read + end - it 'calls assets:delete with ASSETS set to config[:assets] joined with :' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) - allow(@dump).to receive(:find_entry) + describe 'deleting existing assets' do + before do + allow(@stream).to receive(:each) + end - expect(@task).to receive(:invoke) do - expect(Dump::Env[:assets]).to eq('images:videos') - end + it 'calls assets:delete' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) + allow(@dump).to receive(:find_entry) - @dump.read_assets - end + expect(@task).to receive(:invoke) - describe 'when called with restore_assets' do - it 'deletes files and dirs only in requested paths' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) - - expect(Dump::Assets).to receive('glob_asset_children').with('images', '**/*').and_return(%w[images images/a.jpg images/b.jpg]) - expect(Dump::Assets).to receive('glob_asset_children').with('videos', '**/*').and_return(%w[videos videos/a.mov]) - - expect(@dump).to receive('read_asset?').with('images/b.jpg', Dump.rails_root).ordered.and_return(false) - expect(@dump).to receive('read_asset?').with('images/a.jpg', Dump.rails_root).ordered.and_return(true) - expect(@dump).to receive('read_asset?').with('images', Dump.rails_root).ordered.and_return(true) - expect(@dump).to receive('read_asset?').with('videos/a.mov', Dump.rails_root).ordered.and_return(false) - expect(@dump).to receive('read_asset?').with('videos', Dump.rails_root).ordered.and_return(false) - - expect(File).to receive('file?').with('images/a.jpg').and_return(true) - expect(File).to receive('unlink').with('images/a.jpg') - expect(File).not_to receive('file?').with('images/b.jpg') - expect(File).to receive('file?').with('images').and_return(false) - expect(File).to receive('directory?').with('images').and_return(true) - expect(Dir).to receive('unlink').with('images').and_raise(Errno::ENOTEMPTY) - - Dump::Env.with_env(:restore_assets => 'images/a.*:stylesheets') do - @dump.read_assets + @assets_reader.read + end + + it 'calls assets:delete with ASSETS set to config[:assets] joined with :' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) + allow(@dump).to receive(:find_entry) + + expect(@task).to receive(:invoke) do + expect(Dump::Env[:assets]).to eq('images:videos') + end + + @assets_reader.read end - end - it 'does not delete any files and dirs for empty list' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) + describe 'when called with restore_assets' do + it 'deletes files and dirs only in requested paths' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) + + expect(Dump::Assets).to receive('glob_asset_children').with('images', '**/*').and_return(%w[images images/a.jpg images/b.jpg]) + expect(Dump::Assets).to receive('glob_asset_children').with('videos', '**/*').and_return(%w[videos videos/a.mov]) + + expect(@assets_reader).to receive('read_asset?').with('images/b.jpg', Dump.rails_root).ordered.and_return(false) + expect(@assets_reader).to receive('read_asset?').with('images/a.jpg', Dump.rails_root).ordered.and_return(true) + expect(@assets_reader).to receive('read_asset?').with('images', Dump.rails_root).ordered.and_return(true) + expect(@assets_reader).to receive('read_asset?').with('videos/a.mov', Dump.rails_root).ordered.and_return(false) + expect(@assets_reader).to receive('read_asset?').with('videos', Dump.rails_root).ordered.and_return(false) + + expect(File).to receive('file?').with('images/a.jpg').and_return(true) + expect(File).to receive('unlink').with('images/a.jpg') + expect(File).not_to receive('file?').with('images/b.jpg') + expect(File).to receive('file?').with('images').and_return(false) + expect(File).to receive('directory?').with('images').and_return(true) + expect(Dir).to receive('unlink').with('images').and_raise(Errno::ENOTEMPTY) + + Dump::Env.with_env(:restore_assets => 'images/a.*:stylesheets') do + @assets_reader.read + end + end - expect(Dump::Assets).not_to receive('glob_asset_children') + it 'does not delete any files and dirs for empty list' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) - expect(@dump).not_to receive('read_asset?') + expect(Dump::Assets).not_to receive('glob_asset_children') - expect(File).not_to receive('directory?') - expect(File).not_to receive('file?') - expect(File).not_to receive('unlink') + expect(@dump).not_to receive('read_asset?') - Dump::Env.with_env(:restore_assets => '') do - @dump.read_assets + expect(File).not_to receive('directory?') + expect(File).not_to receive('file?') + expect(File).not_to receive('unlink') + + Dump::Env.with_env(:restore_assets => '') do + @assets_reader.read + end + end end end - end - end - describe 'old style' do - it 'finds assets.tar' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) - allow(Dir).to receive(:glob).and_return([]) - allow(FileUtils).to receive(:remove_entry) - allow(@stream).to receive(:each) + describe 'old style' do + it 'finds assets.tar' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) + allow(Dir).to receive(:glob).and_return([]) + allow(FileUtils).to receive(:remove_entry) + allow(@stream).to receive(:each) - expect(@dump).to receive(:find_entry).with('assets.tar') - @dump.read_assets - end - - [ - %w[images videos], - {'images' => 0, 'videos' => 0}, - {'images' => {:files => 0, :total => 0}, 'videos' => {:files => 0, :total => 0}}, - ].each do |assets| - it 'rewrites rewind method to empty method - to not raise exception, opens tar and extracts each entry' do - allow(@dump).to receive(:config).and_return({:assets => assets}) - allow(Dir).to receive(:glob).and_return([]) - allow(FileUtils).to receive(:remove_entry) - - @assets_tar = double('assets_tar') - allow(@assets_tar).to receive(:rewind).and_raise('hehe - we want to rewind to center of gzip') - allow(@dump).to receive(:find_entry).and_yield(@assets_tar) - - @inp = double('inp') - each_excpectation = expect(@inp).to receive(:each) - @entries = %w[a b c d].map do |s| - file = double("file_#{s}") - each_excpectation.and_yield(file) - expect(@inp).to receive(:extract_entry).with(Dump.rails_root, file) - file + expect(@dump).to receive(:find_entry).with('assets.tar') + @assets_reader.read end - expect(Archive::Tar::Minitar).to receive(:open).with(@assets_tar).and_yield(@inp) - @dump.read_assets + [ + %w[images videos], + {'images' => 0, 'videos' => 0}, + {'images' => {:files => 0, :total => 0}, 'videos' => {:files => 0, :total => 0}}, + ].each do |assets| + it 'rewrites rewind method to empty method - to not raise exception, opens tar and extracts each entry' do + allow(@dump).to receive(:config).and_return({:assets => assets}) + allow(Dir).to receive(:glob).and_return([]) + allow(FileUtils).to receive(:remove_entry) + + @assets_tar = double('assets_tar') + allow(@assets_tar).to receive(:rewind).and_raise('hehe - we want to rewind to center of gzip') + allow(@dump).to receive(:find_entry).and_yield(@assets_tar) + + @inp = double('inp') + each_excpectation = expect(@inp).to receive(:each) + @entries = %w[a b c d].map do |s| + file = double("file_#{s}") + each_excpectation.and_yield(file) + expect(@inp).to receive(:extract_entry).with(Dump.rails_root, file) + file + end + expect(Archive::Tar::Minitar).to receive(:open).with(@assets_tar).and_yield(@inp) + + @assets_reader.read + end + end end - end - end - describe 'new style' do - before do - expect(@dump).to receive(:find_entry).with('assets.tar') - end - - [ - %w[images videos], - {'images' => 0, 'videos' => 0}, - {'images' => {:files => 0, :total => 0}, 'videos' => {:files => 0, :total => 0}}, - ].each do |assets| - it 'extracts each entry' do - allow(@dump).to receive(:config).and_return({:assets => assets}) - allow(Dir).to receive(:glob).and_return([]) - allow(FileUtils).to receive(:remove_entry) - - expect(@dump).to receive(:assets_root_link).and_yield('/tmp/abc', 'assets') - each_excpectation = expect(@stream).to receive(:each) - @entries = %w[a b c d].map do |s| - file = double("file_#{s}", :full_name => "assets/#{s}") - each_excpectation.and_yield(file) - expect(@stream).to receive(:extract_entry).with('/tmp/abc', file) - file + describe 'new style' do + before do + expect(@dump).to receive(:find_entry).with('assets.tar') end - other_file = double('other_file', :full_name => 'other_file') - each_excpectation.and_yield(other_file) - expect(@stream).not_to receive(:extract_entry).with('/tmp/abc', other_file) - @dump.read_assets + [ + %w[images videos], + {'images' => 0, 'videos' => 0}, + {'images' => {:files => 0, :total => 0}, 'videos' => {:files => 0, :total => 0}}, + ].each do |assets| + it 'extracts each entry' do + allow(@dump).to receive(:config).and_return({:assets => assets}) + allow(Dir).to receive(:glob).and_return([]) + allow(FileUtils).to receive(:remove_entry) + + expect(@dump).to receive(:assets_root_link).and_yield('/tmp/abc', 'assets') + each_excpectation = expect(@stream).to receive(:each) + @entries = %w[a b c d].map do |s| + file = double("file_#{s}", :full_name => "assets/#{s}") + each_excpectation.and_yield(file) + expect(@stream).to receive(:extract_entry).with('/tmp/abc', file) + file + end + other_file = double('other_file', :full_name => 'other_file') + each_excpectation.and_yield(other_file) + expect(@stream).not_to receive(:extract_entry).with('/tmp/abc', other_file) + + @assets_reader.read + end + end end end - end - end - describe 'read_asset?' do - it 'creates filter and call custom_pass? on it' do - @filter = double('filter') - allow(@filter).to receive('custom_pass?') + describe 'read_asset?' do + it 'creates filter and call custom_pass? on it' do + @filter = double('filter') + allow(@filter).to receive('custom_pass?') - expect(Dump::Env).to receive('filter').with(:restore_assets, Dump::Assets::SPLITTER).and_return(@filter) + expect(Dump::Env).to receive('filter').with(:restore_assets, Dump::Assets::SPLITTER).and_return(@filter) - @dump.read_asset?('a', 'b') - end + @assets_reader.read_asset?('a', 'b') + end - it 'tests path usint fnmatch' do - Dump::Env.with_env(:restore_assets => '[a-b]') do - expect(@dump.read_asset?('x/a', 'x')).to be_truthy - expect(@dump.read_asset?('x/b/file', 'x')).to be_truthy - expect(@dump.read_asset?('x/c', 'x')).to be_falsey + it 'tests path usint fnmatch' do + Dump::Env.with_env(:restore_assets => '[a-b]') do + expect(@assets_reader.read_asset?('x/a', 'x')).to be_truthy + expect(@assets_reader.read_asset?('x/b/file', 'x')).to be_truthy + expect(@assets_reader.read_asset?('x/c', 'x')).to be_falsey + end + end end end end diff --git a/spec/dump/table_manipulation_spec.rb b/spec/dump/table_manipulation_spec.rb index 9fc247b..2af91f6 100644 --- a/spec/dump/table_manipulation_spec.rb +++ b/spec/dump/table_manipulation_spec.rb @@ -45,6 +45,69 @@ end end + describe 'with_disabled_indexes' do + it 'calls indexes, remove_indexes, block, add_indexes in order' do + block = proc{} + + expect(ActiveRecord::Base.connection).to receive(:indexes).with('table').and_return([]).ordered + expect(self).to receive(:remove_indexes).with([]).ordered + expect(block).to receive(:call).ordered + expect(self).to receive(:add_indexes).with([]).ordered + + with_disabled_indexes 'table' do + block.call + end + end + end + + describe 'remove_indexes' do + it 'calls remove_index for each passed index' do + indexes = [ + OpenStruct.new(:table => 'table', :name => 'table_index_1'), + OpenStruct.new(:table => 'table', :name => 'table_index_2'), + ] + + expect(ActiveRecord::Base.connection).to receive(:remove_index).with('table', :name => 'table_index_1').ordered + expect(ActiveRecord::Base.connection).to receive(:remove_index).with('table', :name => 'table_index_2').ordered + + remove_indexes indexes + end + end + + describe 'add_indexes' do + it 'calls add_index for each passed index' do + indexes = [ + OpenStruct.new(:table => 'table', :name => 'table_index_1', :columns => [:col1], :members => [:unique, :length], :unique => true, :test => 1), + OpenStruct.new(:table => 'table', :name => 'table_index_2', :columns => [:col2], :members => [], :lengths => 1), + ] + + expect(ActiveRecord::Base.connection).to receive(:add_index).with('table', [:col1], :unique => true).ordered + expect(ActiveRecord::Base.connection).to receive(:add_index).with('table', [:col2], :length => 1).ordered + + add_indexes indexes + end + end + + describe 'index_options' do + it 'returns only valid index options' do + index = OpenStruct.new( + :members => [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :test], + :unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8, :test => 10) + expect(index_options(index)).to eq(:unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8) + end + + it 'returns only non nil index options' do + index = OpenStruct.new(:members => [:unique, :where], :unique => nil, :where => '(a=1)') + expect(index_options(index)).to eq(:where => '(a=1)') + end + + # mysql adapter implementation detail + it 'returns length for lengths index options' do + index = OpenStruct.new(:members => [], :lengths => 1) + expect(index_options(index)).to eq(:length => 1) + end + end + describe 'insert_into_table' do it 'calls connection.insert with sql for insert if values is string' do expect(connection).to receive(:insert).with('INSERT INTO `table` (`c1`,`c2`) VALUES (`v1`,`v2`)', anything)