From 20b1c41ca30be2cb9c905432461a44307a410b5e Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Mon, 29 Sep 2014 23:07:00 +0400 Subject: [PATCH 01/17] added remove indexes before bulk insert and add indexes after --- lib/dump/reader.rb | 28 +++++++++++++++------------- lib/dump/table_manipulation.rb | 30 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/lib/dump/reader.rb b/lib/dump/reader.rb index 3fbf87a..7d53a83 100644 --- a/lib/dump/reader.rb +++ b/lib/dump/reader.rb @@ -193,20 +193,22 @@ def read_table(table, rows_count) columns = Marshal.load(entry) columns_sql = columns_insert_sql(columns) - Progress.start(table, rows_count) do - until entry.eof? - rows_sql = [] - 1000.times do - rows_sql << values_insert_sql(Marshal.load(entry)) unless entry.eof? - end + with_disabled_indexes table do + Progress.start(table, rows_count) do + until entry.eof? + rows_sql = [] + 1000.times do + rows_sql << values_insert_sql(Marshal.load(entry)) unless entry.eof? + end - begin - insert_into_table(table_sql, columns_sql, rows_sql) - Progress.step(rows_sql.length) - rescue - rows_sql.each do |row_sql| - insert_into_table(table_sql, columns_sql, row_sql) - Progress.step + begin + insert_into_table(table_sql, columns_sql, rows_sql) + Progress.step(rows_sql.length) + rescue + rows_sql.each do |row_sql| + insert_into_table(table_sql, columns_sql, row_sql) + Progress.step + end end end end diff --git a/lib/dump/table_manipulation.rb b/lib/dump/table_manipulation.rb index ba02b66..359ce2b 100644 --- a/lib/dump/table_manipulation.rb +++ b/lib/dump/table_manipulation.rb @@ -29,6 +29,36 @@ def clear_table(table_sql) connection.delete("DELETE FROM #{table_sql}", 'Clearing table') end + def with_disabled_indexes(table, &block) + all_indexes = ActiveRecord::Base.connection.indexes(table) + table_indexes = all_indexes.keep_if {|index| index.table == table} + + remove_indexes(table_indexes) + block.call + add_indexes(table_indexes) + end + + def remove_indexes(indexes) + indexes.each do |index| + ActiveRecord::Base.connection.remove_index index.table, :name => index.name + end + end + + def add_indexes(indexes) + indexes.each do |index| + options = { + :name => index.name, + :unique => index.unique, + :length => index.lengths, + :order => index.orders, + :where => index.where, + :type => index.type, + :using => index.using + } + ActiveRecord::Base.connection.add_index index.table, index.columns, options + end + end + def insert_into_table(table_sql, columns_sql, values_sql) values_sql = values_sql.join(',') if values_sql.is_a?(Array) sql = "INSERT INTO #{table_sql} #{columns_sql} VALUES #{values_sql}" From 278a1d0fa0c9351e0e2a4da243443bef700d1d91 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Sun, 7 Dec 2014 20:00:03 +0300 Subject: [PATCH 02/17] fixed index restore for rails 2.3 and rails 3.2 --- lib/dump/table_manipulation.rb | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/dump/table_manipulation.rb b/lib/dump/table_manipulation.rb index 359ce2b..f48e934 100644 --- a/lib/dump/table_manipulation.rb +++ b/lib/dump/table_manipulation.rb @@ -44,18 +44,15 @@ def remove_indexes(indexes) end end + INDEX_FIELDS = [:name, :unique, :length, :order, :where, :type, :using].freeze + + def index_options(index) + Hash[*INDEX_FIELDS.map{|field| [field, index.members.include?(field) ? index.send(field) : nil]}.flatten] + end + def add_indexes(indexes) indexes.each do |index| - options = { - :name => index.name, - :unique => index.unique, - :length => index.lengths, - :order => index.orders, - :where => index.where, - :type => index.type, - :using => index.using - } - ActiveRecord::Base.connection.add_index index.table, index.columns, options + ActiveRecord::Base.connection.add_index index.table, index.columns, index_options(index) end end From 9e18d999362499a441f3dbebfd7d93fead014e11 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Sun, 7 Dec 2014 21:10:37 +0300 Subject: [PATCH 03/17] fixed indexes recreate on restore for ruby 1.8.7 --- lib/dump/table_manipulation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dump/table_manipulation.rb b/lib/dump/table_manipulation.rb index f48e934..ee507d3 100644 --- a/lib/dump/table_manipulation.rb +++ b/lib/dump/table_manipulation.rb @@ -31,7 +31,7 @@ def clear_table(table_sql) def with_disabled_indexes(table, &block) all_indexes = ActiveRecord::Base.connection.indexes(table) - table_indexes = all_indexes.keep_if {|index| index.table == table} + table_indexes = all_indexes.select {|index| index.table == table} remove_indexes(table_indexes) block.call From 8ce66f69e78d1c25bbc6c3186ae641ca2aaba384 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Sun, 7 Dec 2014 21:28:14 +0300 Subject: [PATCH 04/17] fixed table indexes recreate for empty index name --- lib/dump/table_manipulation.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/dump/table_manipulation.rb b/lib/dump/table_manipulation.rb index ee507d3..be8bfe8 100644 --- a/lib/dump/table_manipulation.rb +++ b/lib/dump/table_manipulation.rb @@ -47,7 +47,10 @@ def remove_indexes(indexes) INDEX_FIELDS = [:name, :unique, :length, :order, :where, :type, :using].freeze def index_options(index) - Hash[*INDEX_FIELDS.map{|field| [field, index.members.include?(field) ? index.send(field) : nil]}.flatten] + options = INDEX_FIELDS.map{|field| [field, index.members.include?(field) ? index.send(field) : nil]} + non_empty_options = options.select{|pair| !pair[1].nil?} + + Hash[*non_empty_options.flatten] end def add_indexes(indexes) From 520b499832889e1527a93bd14336d1afc28d961f Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Sun, 11 Jan 2015 21:46:41 +0300 Subject: [PATCH 05/17] removed additional table indexes filtration from with_disabled_indexes --- lib/dump/table_manipulation.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/dump/table_manipulation.rb b/lib/dump/table_manipulation.rb index be8bfe8..95dbf89 100644 --- a/lib/dump/table_manipulation.rb +++ b/lib/dump/table_manipulation.rb @@ -30,8 +30,7 @@ def clear_table(table_sql) end def with_disabled_indexes(table, &block) - all_indexes = ActiveRecord::Base.connection.indexes(table) - table_indexes = all_indexes.select {|index| index.table == table} + table_indexes = ActiveRecord::Base.connection.indexes(table) remove_indexes(table_indexes) block.call From 4e78a0d20d8dce530783992ce5cad6b746e477ce Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Sun, 11 Jan 2015 21:47:18 +0300 Subject: [PATCH 06/17] updated valid index options list for index_options --- lib/dump/table_manipulation.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dump/table_manipulation.rb b/lib/dump/table_manipulation.rb index 95dbf89..4e905ad 100644 --- a/lib/dump/table_manipulation.rb +++ b/lib/dump/table_manipulation.rb @@ -43,10 +43,10 @@ def remove_indexes(indexes) end end - INDEX_FIELDS = [:name, :unique, :length, :order, :where, :type, :using].freeze + VALID_INDEX_OPTIONS = [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type].freeze def index_options(index) - options = INDEX_FIELDS.map{|field| [field, index.members.include?(field) ? index.send(field) : nil]} + options = VALID_INDEX_OPTIONS.map{|field| [field, index.members.include?(field) ? index.send(field) : nil]} non_empty_options = options.select{|pair| !pair[1].nil?} Hash[*non_empty_options.flatten] From 5df6df87ca681d294625059a0db10e7e9c11046c Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Sun, 11 Jan 2015 21:47:55 +0300 Subject: [PATCH 07/17] added disable indexes while restoring unit tests --- spec/dump/reader_spec.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spec/dump/reader_spec.rb b/spec/dump/reader_spec.rb index 42eeda9..8f49137 100644 --- a/spec/dump/reader_spec.rb +++ b/spec/dump/reader_spec.rb @@ -478,6 +478,14 @@ def create_entry(rows_count) expect(@dump).to receive(:insert_into_table).with('`first`', '(`id`, `name`)', @rows.map(&:inspect)) @dump.read_table('first', 100) end + + it "should remove indexes around reading/writing table" do + create_entry(100) + allow(@dump).to receive(:clear_table) + allow(@dump).to receive(:insert_into_table) + expect(@dump).to receive(:with_disabled_indexes).with('first') + @dump.read_table('first', 100) + end end end From b6e666f1119446cda7f5efe1971617fdff91df77 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Sun, 11 Jan 2015 22:13:23 +0300 Subject: [PATCH 08/17] fixed rubocop codestyle issues --- lib/dump/reader.rb | 6 ++---- lib/dump/table_manipulation.rb | 4 ++-- spec/dump/reader_spec.rb | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/dump/reader.rb b/lib/dump/reader.rb index 7d53a83..f0127f7 100644 --- a/lib/dump/reader.rb +++ b/lib/dump/reader.rb @@ -218,8 +218,7 @@ def read_table(table, rows_count) end def read_assets - return if Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty? - return if config[:assets].blank? + return if (Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty?) || config[:assets].blank? assets = config[:assets] if assets.is_a?(Hash) @@ -260,8 +259,7 @@ def read_assets def read_asset?(path, prefix) Dump::Env.filter(:restore_assets, Dump::Assets::SPLITTER).custom_pass? do |value| - File.fnmatch(File.join(prefix, value), path) || - File.fnmatch(File.join(prefix, value, '**'), path) + File.fnmatch(File.join(prefix, value), path) || File.fnmatch(File.join(prefix, value, '**'), path) end end diff --git a/lib/dump/table_manipulation.rb b/lib/dump/table_manipulation.rb index 4e905ad..f3b6954 100644 --- a/lib/dump/table_manipulation.rb +++ b/lib/dump/table_manipulation.rb @@ -46,8 +46,8 @@ def remove_indexes(indexes) VALID_INDEX_OPTIONS = [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type].freeze def index_options(index) - options = VALID_INDEX_OPTIONS.map{|field| [field, index.members.include?(field) ? index.send(field) : nil]} - non_empty_options = options.select{|pair| !pair[1].nil?} + options = VALID_INDEX_OPTIONS.map{ |field| [field, index.members.include?(field) ? index.send(field) : nil] } + non_empty_options = options.select{ |pair| !pair[1].nil? } Hash[*non_empty_options.flatten] end diff --git a/spec/dump/reader_spec.rb b/spec/dump/reader_spec.rb index 8f49137..caf1f12 100644 --- a/spec/dump/reader_spec.rb +++ b/spec/dump/reader_spec.rb @@ -479,7 +479,7 @@ def create_entry(rows_count) @dump.read_table('first', 100) end - it "should remove indexes around reading/writing table" do + it 'should remove indexes around reading/writing table' do create_entry(100) allow(@dump).to receive(:clear_table) allow(@dump).to receive(:insert_into_table) From 6adfbcf12b652d9743b7a2524916ff237872e9db Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Sun, 11 Jan 2015 22:25:00 +0300 Subject: [PATCH 09/17] fixed gem pg version to 0.17.1 for ruby 1.8.7 --- Gemfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index b704305..1666d93 100644 --- a/Gemfile +++ b/Gemfile @@ -10,7 +10,11 @@ if defined?(JRUBY_VERSION) else gem 'sqlite3' gem 'mysql2' - gem 'pg' + if RUBY_VERSION == '1.8.7' + gem 'pg', '0.17.1' + else + gem 'pg' + end if rails_version =~ /(^|[^.\d])(2|3\.0)\.\d+/ gem 'activerecord-mysql2-adapter' end From 93b6afa59ccf1fb05270a570c5f942b3b0ba1a45 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Sun, 11 Jan 2015 22:35:50 +0300 Subject: [PATCH 10/17] fixed gem i18n version to 0.6.11 for ruby 1.8.7 --- Gemfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile b/Gemfile index 1666d93..b7246c0 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,7 @@ else gem 'mysql2' if RUBY_VERSION == '1.8.7' gem 'pg', '0.17.1' + gem 'i18n', '0.6.11' else gem 'pg' end From 20540dd68ac87c36b3869f34df547ff17335d198 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Mon, 19 Jan 2015 14:10:11 +0300 Subject: [PATCH 11/17] fixed length index option for mysql --- lib/dump/table_manipulation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dump/table_manipulation.rb b/lib/dump/table_manipulation.rb index f3b6954..e44dc35 100644 --- a/lib/dump/table_manipulation.rb +++ b/lib/dump/table_manipulation.rb @@ -31,7 +31,6 @@ def clear_table(table_sql) def with_disabled_indexes(table, &block) table_indexes = ActiveRecord::Base.connection.indexes(table) - remove_indexes(table_indexes) block.call add_indexes(table_indexes) @@ -48,6 +47,7 @@ def remove_indexes(indexes) def index_options(index) options = VALID_INDEX_OPTIONS.map{ |field| [field, index.members.include?(field) ? index.send(field) : nil] } non_empty_options = options.select{ |pair| !pair[1].nil? } + non_empty_options << [:length, index.lengths] if index.try(:lengths).present? Hash[*non_empty_options.flatten] end From d9f4493176a487bde1c97100b5b9f5b869de4e59 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Tue, 20 Jan 2015 11:35:11 +0300 Subject: [PATCH 12/17] added with_disabled_indexes, remove_indexes, add_indexes, index_options unit tests --- spec/dump/table_manipulation_spec.rb | 65 ++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/spec/dump/table_manipulation_spec.rb b/spec/dump/table_manipulation_spec.rb index 9fc247b..24e2e71 100644 --- a/spec/dump/table_manipulation_spec.rb +++ b/spec/dump/table_manipulation_spec.rb @@ -45,6 +45,71 @@ end end + describe 'with_disabled_indexes' do + it 'calls indexes, remove_indexes, block, add_indexes in order' do + block = Proc.new {} + + expect(ActiveRecord::Base.connection).to receive(:indexes).with('table').and_return([]).ordered + expect(self).to receive(:remove_indexes).with([]).ordered + expect(block).to receive(:call).ordered + expect(self).to receive(:add_indexes).with([]).ordered + + with_disabled_indexes 'table' do + block.call + end + end + end + + describe 'remove_indexes' do + it 'calls remove_index for each passed index' do + indexes = [ + OpenStruct.new(table: 'table', name: 'table_index_1'), + OpenStruct.new(table: 'table', name: 'table_index_2') + ] + + expect(ActiveRecord::Base.connection).to receive(:remove_index).with('table', name: 'table_index_1').ordered + expect(ActiveRecord::Base.connection).to receive(:remove_index).with('table', name: 'table_index_2').ordered + + remove_indexes indexes + end + end + + describe 'add_indexes' do + it 'calls add_index for each passed index' do + indexes = [ + OpenStruct.new(table: 'table', name: 'table_index_1', columns: [:col1], members: [:unique, :length], unique: true, test: 1), + OpenStruct.new(table: 'table', name: 'table_index_2', columns: [:col2], members: [], lengths: 1) + ] + + expect(ActiveRecord::Base.connection).to receive(:add_index).with('table', [:col1], unique: true).ordered + expect(ActiveRecord::Base.connection).to receive(:add_index).with('table', [:col2], length: 1).ordered + + add_indexes indexes + end + end + + describe 'index_options' do + it 'returns only valid index options' do + index = OpenStruct.new( + members: [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type, :test], + unique: 1, order: 2, name: 3, where: 4, length: 5, internal: 6, using: 7, algorithm: 8, type: 9, + test: 10 + ) + expect(index_options(index)).to eq(unique: 1, order: 2, name: 3, where: 4, length: 5, internal: 6, using: 7, algorithm: 8, type: 9) + end + + it 'returns only non nil index options' do + index = OpenStruct.new(members: [:unique, :where], unique: nil, where: '(a=1)') + expect(index_options(index)).to eq(where: '(a=1)') + end + + # mysql adapter implementation detail + it 'returns length for lengths index options' do + index = OpenStruct.new(members: [], lengths: 1) + expect(index_options(index)).to eq(length: 1) + end + end + describe 'insert_into_table' do it 'calls connection.insert with sql for insert if values is string' do expect(connection).to receive(:insert).with('INSERT INTO `table` (`c1`,`c2`) VALUES (`v1`,`v2`)', anything) From 5956f765caf2249ee4fc72d9b716a0a0549f1320 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Tue, 20 Jan 2015 17:02:46 +0300 Subject: [PATCH 13/17] fixed codestyle issues --- lib/dump/reader.rb | 37 ++++++---------------------- lib/dump/reader/summary.rb | 27 ++++++++++++++++++++ spec/dump/table_manipulation_spec.rb | 34 ++++++++++++------------- 3 files changed, 52 insertions(+), 46 deletions(-) create mode 100644 lib/dump/reader/summary.rb diff --git a/lib/dump/reader.rb b/lib/dump/reader.rb index f0127f7..d6e0bd2 100644 --- a/lib/dump/reader.rb +++ b/lib/dump/reader.rb @@ -5,6 +5,7 @@ require 'rake' require 'zlib' require 'tempfile' +require 'dump/reader/summary' module Dump # Reading dump @@ -24,30 +25,6 @@ def self.restore(path) end end - # Helper class for building summary of dump - class Summary - attr_reader :text - alias_method :to_s, :text - def initialize - @text = '' - end - - def header(header) - @text << " #{header}:\n" - end - - def data(entries) - entries.each do |entry| - @text << " #{entry}\n" - end - end - - # from ActionView::Helpers::TextHelper - def self.pluralize(count, singular) - "#{count} #{count == 1 ? singular : singular.pluralize}" - end - end - def self.summary(path, options = {}) new(path).open do |dump| dump.read_config @@ -98,7 +75,8 @@ def open def find_entry(matcher) stream.each do |entry| if entry.full_name.match(matcher) - # we can not return entry - after exiting stream.each the entry will be invalid and will read from tar start + # we can not return entry - after exiting stream.each + # the entry will be invalid and will read from tar start return yield(entry) end end @@ -191,8 +169,7 @@ def read_table(table, rows_count) table_sql = quote_table_name(table) clear_table(table_sql) - columns = Marshal.load(entry) - columns_sql = columns_insert_sql(columns) + columns_sql = columns_insert_sql(Marshal.load(entry)) with_disabled_indexes table do Progress.start(table, rows_count) do until entry.eof? @@ -218,7 +195,8 @@ def read_table(table, rows_count) end def read_assets - return if (Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty?) || config[:assets].blank? + return if Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty? + return if config[:assets].blank? assets = config[:assets] if assets.is_a?(Hash) @@ -259,7 +237,8 @@ def read_assets def read_asset?(path, prefix) Dump::Env.filter(:restore_assets, Dump::Assets::SPLITTER).custom_pass? do |value| - File.fnmatch(File.join(prefix, value), path) || File.fnmatch(File.join(prefix, value, '**'), path) + File.fnmatch(File.join(prefix, value), path) || + File.fnmatch(File.join(prefix, value, '**'), path) end end diff --git a/lib/dump/reader/summary.rb b/lib/dump/reader/summary.rb new file mode 100644 index 0000000..79e4063 --- /dev/null +++ b/lib/dump/reader/summary.rb @@ -0,0 +1,27 @@ +module Dump + class Reader < Snapshot + # Helper class for building summary of dump + class Summary + attr_reader :text + alias_method :to_s, :text + def initialize + @text = '' + end + + def header(header) + @text << " #{header}:\n" + end + + def data(entries) + entries.each do |entry| + @text << " #{entry}\n" + end + end + + # from ActionView::Helpers::TextHelper + def self.pluralize(count, singular) + "#{count} #{count == 1 ? singular : singular.pluralize}" + end + end + end +end diff --git a/spec/dump/table_manipulation_spec.rb b/spec/dump/table_manipulation_spec.rb index 24e2e71..c67c644 100644 --- a/spec/dump/table_manipulation_spec.rb +++ b/spec/dump/table_manipulation_spec.rb @@ -47,7 +47,7 @@ describe 'with_disabled_indexes' do it 'calls indexes, remove_indexes, block, add_indexes in order' do - block = Proc.new {} + block = proc{} expect(ActiveRecord::Base.connection).to receive(:indexes).with('table').and_return([]).ordered expect(self).to receive(:remove_indexes).with([]).ordered @@ -63,12 +63,12 @@ describe 'remove_indexes' do it 'calls remove_index for each passed index' do indexes = [ - OpenStruct.new(table: 'table', name: 'table_index_1'), - OpenStruct.new(table: 'table', name: 'table_index_2') + OpenStruct.new(:table => 'table', :name => 'table_index_1'), + OpenStruct.new(:table => 'table', :name => 'table_index_2'), ] - expect(ActiveRecord::Base.connection).to receive(:remove_index).with('table', name: 'table_index_1').ordered - expect(ActiveRecord::Base.connection).to receive(:remove_index).with('table', name: 'table_index_2').ordered + expect(ActiveRecord::Base.connection).to receive(:remove_index).with('table', :name => 'table_index_1').ordered + expect(ActiveRecord::Base.connection).to receive(:remove_index).with('table', :name => 'table_index_2').ordered remove_indexes indexes end @@ -77,12 +77,12 @@ describe 'add_indexes' do it 'calls add_index for each passed index' do indexes = [ - OpenStruct.new(table: 'table', name: 'table_index_1', columns: [:col1], members: [:unique, :length], unique: true, test: 1), - OpenStruct.new(table: 'table', name: 'table_index_2', columns: [:col2], members: [], lengths: 1) + OpenStruct.new(:table => 'table', :name => 'table_index_1', :columns => [:col1], :members => [:unique, :length], :unique => true, :test => 1), + OpenStruct.new(:table => 'table', :name => 'table_index_2', :columns => [:col2], :members => [], :lengths => 1), ] - expect(ActiveRecord::Base.connection).to receive(:add_index).with('table', [:col1], unique: true).ordered - expect(ActiveRecord::Base.connection).to receive(:add_index).with('table', [:col2], length: 1).ordered + expect(ActiveRecord::Base.connection).to receive(:add_index).with('table', [:col1], :unique => true).ordered + expect(ActiveRecord::Base.connection).to receive(:add_index).with('table', [:col2], :length => 1).ordered add_indexes indexes end @@ -91,22 +91,22 @@ describe 'index_options' do it 'returns only valid index options' do index = OpenStruct.new( - members: [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type, :test], - unique: 1, order: 2, name: 3, where: 4, length: 5, internal: 6, using: 7, algorithm: 8, type: 9, - test: 10 + :members => [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type, :test], + :unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8, :type => 9, + :test => 10, ) - expect(index_options(index)).to eq(unique: 1, order: 2, name: 3, where: 4, length: 5, internal: 6, using: 7, algorithm: 8, type: 9) + expect(index_options(index)).to eq(:unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8, :type => 9) end it 'returns only non nil index options' do - index = OpenStruct.new(members: [:unique, :where], unique: nil, where: '(a=1)') - expect(index_options(index)).to eq(where: '(a=1)') + index = OpenStruct.new(:members => [:unique, :where], :unique => nil, :where => '(a=1)') + expect(index_options(index)).to eq(:where => '(a=1)') end # mysql adapter implementation detail it 'returns length for lengths index options' do - index = OpenStruct.new(members: [], lengths: 1) - expect(index_options(index)).to eq(length: 1) + index = OpenStruct.new(:members => [], :lengths => 1) + expect(index_options(index)).to eq(:length => 1) end end From 9aa91625789aa3eb34d55317a479290663508f24 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Wed, 21 Jan 2015 12:43:55 +0300 Subject: [PATCH 14/17] fixed specs for ruby 1.8.7 --- spec/dump/table_manipulation_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/dump/table_manipulation_spec.rb b/spec/dump/table_manipulation_spec.rb index c67c644..38940f3 100644 --- a/spec/dump/table_manipulation_spec.rb +++ b/spec/dump/table_manipulation_spec.rb @@ -91,11 +91,11 @@ describe 'index_options' do it 'returns only valid index options' do index = OpenStruct.new( - :members => [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type, :test], - :unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8, :type => 9, - :test => 10, + :members => [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :test], + :unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8, + :test => 10 ) - expect(index_options(index)).to eq(:unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8, :type => 9) + expect(index_options(index)).to eq(:unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8) end it 'returns only non nil index options' do From 670cb1e308ff4823e845b5c011811f6da4991882 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Wed, 21 Jan 2015 13:47:14 +0300 Subject: [PATCH 15/17] fixed spec/dump/table_manipulation_spec.rb rubocop issue --- spec/dump/table_manipulation_spec.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/spec/dump/table_manipulation_spec.rb b/spec/dump/table_manipulation_spec.rb index 38940f3..2af91f6 100644 --- a/spec/dump/table_manipulation_spec.rb +++ b/spec/dump/table_manipulation_spec.rb @@ -92,9 +92,7 @@ it 'returns only valid index options' do index = OpenStruct.new( :members => [:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :test], - :unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8, - :test => 10 - ) + :unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8, :test => 10) expect(index_options(index)).to eq(:unique => 1, :order => 2, :name => 3, :where => 4, :length => 5, :internal => 6, :using => 7, :algorithm => 8) end From 29d5d52546678ead5770b7d5f25307be9691fae0 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Wed, 21 Jan 2015 15:05:50 +0300 Subject: [PATCH 16/17] added REBUILD_INDEXES restore option with explanation in README and specs --- README.markdown | 2 ++ lib/dump/env.rb | 2 ++ lib/dump/reader.rb | 46 +++++++++++++++++++++++++--------------- spec/dump/reader_spec.rb | 12 ++++++++++- 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/README.markdown b/README.markdown index b8face9..aa52889 100644 --- a/README.markdown +++ b/README.markdown @@ -145,6 +145,8 @@ reset database: rake dump:restore MIGRATE_DOWN=reset +`REBUILD_INDEXES` — remove indexes for each table before restore, and create them after restore if you pass "1", "true" or "yes". `REBUILD_INDEXES` is useful to speed up restoring for large tables (see https://github.com/toy/dump/pull/12#issuecomment-69462275), but may affect index structure because of database adapters implementations. + `RESTORE_SCHEMA` — don't read/change schema if you pass "0", "no" or "false" (useful to just restore data for table; note that schema info tables are also not restored) don't restore schema: diff --git a/lib/dump/env.rb b/lib/dump/env.rb index 88b7baa..f6cf70b 100644 --- a/lib/dump/env.rb +++ b/lib/dump/env.rb @@ -16,6 +16,7 @@ module Env :backup => %w[BACKUP AUTOBACKUP AUTO_BACKUP], :transfer_via => %w[TRANSFER_VIA], :migrate_down => %w[MIGRATE_DOWN], + :rebuild_indexes => %w[REBUILD_INDEXES], :restore_schema => %w[RESTORE_SCHEMA], :restore_tables => %w[RESTORE_TABLES], :restore_assets => %w[RESTORE_ASSETS], @@ -33,6 +34,7 @@ module Env :backup => 'no autobackup if you pass "0", "no" or "false"', :transfer_via => 'transfer method (rsync, sftp or scp)', :migrate_down => 'don\'t run down for migrations not present in dump if you pass "0", "no" or "false"; pass "reset" to recreate (drop and create) db', + :rebuild_indexes => 'remove indexes before restore, and add them after if you pass "1", "true" or "yes"; speed up restoring for large tables, but may affect indexes structure (see README for details)', :restore_schema => 'don\'t read/change schema if you pass "0", "no" or "false" (useful to just restore data for table; note that schema info tables are also not restored)', :restore_tables => 'works as TABLES, but for restoring', :restore_assets => 'works as ASSETS, but for restoring', diff --git a/lib/dump/reader.rb b/lib/dump/reader.rb index d6e0bd2..ee316fb 100644 --- a/lib/dump/reader.rb +++ b/lib/dump/reader.rb @@ -164,33 +164,45 @@ def read_tables end end + def rebuild_indexes? + Dump::Env.yes?(:rebuild_indexes) + end + def read_table(table, rows_count) find_entry("#{table}.dump") do |entry| table_sql = quote_table_name(table) clear_table(table_sql) columns_sql = columns_insert_sql(Marshal.load(entry)) - with_disabled_indexes table do - Progress.start(table, rows_count) do - until entry.eof? - rows_sql = [] - 1000.times do - rows_sql << values_insert_sql(Marshal.load(entry)) unless entry.eof? - end + if rebuild_indexes? + with_disabled_indexes table do + bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql) + end + else + bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql) + end + fix_sequence!(table) + end + end - begin - insert_into_table(table_sql, columns_sql, rows_sql) - Progress.step(rows_sql.length) - rescue - rows_sql.each do |row_sql| - insert_into_table(table_sql, columns_sql, row_sql) - Progress.step - end - end + def bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql) + Progress.start(table, rows_count) do + until entry.eof? + rows_sql = [] + 1000.times do + rows_sql << values_insert_sql(Marshal.load(entry)) unless entry.eof? + end + + begin + insert_into_table(table_sql, columns_sql, rows_sql) + Progress.step(rows_sql.length) + rescue + rows_sql.each do |row_sql| + insert_into_table(table_sql, columns_sql, row_sql) + Progress.step end end end - fix_sequence!(table) end end diff --git a/spec/dump/reader_spec.rb b/spec/dump/reader_spec.rb index caf1f12..4877b80 100644 --- a/spec/dump/reader_spec.rb +++ b/spec/dump/reader_spec.rb @@ -479,13 +479,23 @@ def create_entry(rows_count) @dump.read_table('first', 100) end - it 'should remove indexes around reading/writing table' do + it 'should remove indexes around reading/writing table in case of rebuild indexes' do create_entry(100) allow(@dump).to receive(:clear_table) allow(@dump).to receive(:insert_into_table) + expect(@dump).to receive(:rebuild_indexes?).and_return(true) expect(@dump).to receive(:with_disabled_indexes).with('first') @dump.read_table('first', 100) end + + it 'should not remove indexes around reading/writing table in case of not rebuild indexes' do + create_entry(100) + allow(@dump).to receive(:clear_table) + allow(@dump).to receive(:insert_into_table) + expect(@dump).to receive(:rebuild_indexes?).and_return(false) + expect(@dump).not_to receive(:with_disabled_indexes) + @dump.read_table('first', 100) + end end end From 7966b031625bc5ac4232e2ad32cbda2aa618e6a3 Mon Sep 17 00:00:00 2001 From: Alexandr Borisov Date: Wed, 21 Jan 2015 15:08:07 +0300 Subject: [PATCH 17/17] extracted assets from Reader to separate class to reduce Reader class definition --- lib/dump/reader.rb | 83 +--------- lib/dump/reader/assets.rb | 93 +++++++++++ spec/dump/reader_spec.rb | 319 ++++++++++++++++++++------------------ 3 files changed, 264 insertions(+), 231 deletions(-) create mode 100644 lib/dump/reader/assets.rb diff --git a/lib/dump/reader.rb b/lib/dump/reader.rb index ee316fb..16b821b 100644 --- a/lib/dump/reader.rb +++ b/lib/dump/reader.rb @@ -6,6 +6,7 @@ require 'zlib' require 'tempfile' require 'dump/reader/summary' +require 'dump/reader/assets' module Dump # Reading dump @@ -20,7 +21,7 @@ def self.restore(path) dump.read_schema dump.read_tables - dump.read_assets + Assets.new(dump).read end end end @@ -205,85 +206,5 @@ def bulk_insert_into_table(table, rows_count, entry, table_sql, columns_sql) end end end - - def read_assets - return if Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty? - return if config[:assets].blank? - - assets = config[:assets] - if assets.is_a?(Hash) - assets_count = assets.values.sum{ |value| value.is_a?(Hash) ? value[:total] : value } - assets_paths = assets.keys - else - assets_count, assets_paths = nil, assets - end - - if Dump::Env[:restore_assets] - assets_paths.each do |asset| - Dump::Assets.glob_asset_children(asset, '**/*').reverse.each do |child| - next unless read_asset?(child, Dump.rails_root) - case - when File.file?(child) - File.unlink(child) - when File.directory?(child) - begin - Dir.unlink(child) - rescue Errno::ENOTEMPTY - nil - end - end - end - end - else - Dump::Env.with_env(:assets => assets_paths.join(':')) do - Rake::Task['assets:delete'].invoke - end - end - - read_assets_entries(assets_paths, assets_count) do |stream, root, entry, prefix| - if !Dump::Env[:restore_assets] || read_asset?(entry.full_name, prefix) - stream.extract_entry(root, entry) - end - end - end - - def read_asset?(path, prefix) - Dump::Env.filter(:restore_assets, Dump::Assets::SPLITTER).custom_pass? do |value| - File.fnmatch(File.join(prefix, value), path) || - File.fnmatch(File.join(prefix, value, '**'), path) - end - end - - def read_assets_entries(_assets_paths, assets_count) - Progress.start('Assets', assets_count || 1) do - found_assets = false - # old style - in separate tar - find_entry('assets.tar') do |assets_tar| - def assets_tar.rewind - # rewind will fail - it must go to center of gzip - # also we don't need it - this is last step in dump restore - end - Archive::Tar::Minitar.open(assets_tar) do |inp| - inp.each do |entry| - yield inp, Dump.rails_root, entry, nil - Progress.step if assets_count - end - end - found_assets = true - end - - unless found_assets - # new style - in same tar - assets_root_link do |tmpdir, prefix| - stream.each do |entry| - if entry.full_name.starts_with?("#{prefix}/") - yield stream, tmpdir, entry, prefix - Progress.step if assets_count - end - end - end - end - end - end end end diff --git a/lib/dump/reader/assets.rb b/lib/dump/reader/assets.rb new file mode 100644 index 0000000..32de89f --- /dev/null +++ b/lib/dump/reader/assets.rb @@ -0,0 +1,93 @@ +module Dump + class Reader < Snapshot + # Helper class for reading assets + class Assets + attr_reader :dump + delegate :config, :to => :dump + + def initialize(dump) + @dump = dump + end + + def read + return if Dump::Env[:restore_assets] && Dump::Env[:restore_assets].empty? + return if config[:assets].blank? + + assets = config[:assets] + if assets.is_a?(Hash) + assets_count = assets.values.sum{ |value| value.is_a?(Hash) ? value[:total] : value } + assets_paths = assets.keys + else + assets_count, assets_paths = nil, assets + end + + if Dump::Env[:restore_assets] + assets_paths.each do |asset| + Dump::Assets.glob_asset_children(asset, '**/*').reverse.each do |child| + next unless read_asset?(child, Dump.rails_root) + case + when File.file?(child) + File.unlink(child) + when File.directory?(child) + begin + Dir.unlink(child) + rescue Errno::ENOTEMPTY + nil + end + end + end + end + else + Dump::Env.with_env(:assets => assets_paths.join(':')) do + Rake::Task['assets:delete'].invoke + end + end + + read_assets_entries(assets_paths, assets_count) do |stream, root, entry, prefix| + if !Dump::Env[:restore_assets] || read_asset?(entry.full_name, prefix) + stream.extract_entry(root, entry) + end + end + end + + def read_assets_entries(_assets_paths, assets_count) + Progress.start('Assets', assets_count || 1) do + found_assets = false + # old style - in separate tar + dump.find_entry('assets.tar') do |assets_tar| + def assets_tar.rewind + # rewind will fail - it must go to center of gzip + # also we don't need it - this is last step in dump restore + end + Archive::Tar::Minitar.open(assets_tar) do |inp| + inp.each do |entry| + yield inp, Dump.rails_root, entry, nil + Progress.step if assets_count + end + end + found_assets = true + end + + unless found_assets + # new style - in same tar + dump.send :assets_root_link do |tmpdir, prefix| + dump.stream.each do |entry| + if entry.full_name.starts_with?("#{prefix}/") + yield dump.stream, tmpdir, entry, prefix + Progress.step if assets_count + end + end + end + end + end + end + + def read_asset?(path, prefix) + Dump::Env.filter(:restore_assets, Dump::Assets::SPLITTER).custom_pass? do |value| + File.fnmatch(File.join(prefix, value), path) || + File.fnmatch(File.join(prefix, value, '**'), path) + end + end + end + end +end diff --git a/spec/dump/reader_spec.rb b/spec/dump/reader_spec.rb index 4877b80..33b543a 100644 --- a/spec/dump/reader_spec.rb +++ b/spec/dump/reader_spec.rb @@ -22,7 +22,10 @@ expect(@dump).to receive(:migrate_down).ordered expect(@dump).to receive(:read_schema).ordered expect(@dump).to receive(:read_tables).ordered - expect(@dump).to receive(:read_assets).ordered + + @assets = double('assets') + expect(Dump::Reader::Assets).to receive(:new).with(@dump).and_return(@assets).ordered + expect(@assets).to receive(:read).ordered described_class.restore('/abc/123.tmp') end @@ -499,187 +502,203 @@ def create_entry(rows_count) end end - describe 'read_assets' do - before do - @task = double('task') - allow(Rake::Task).to receive(:[]).with('assets:delete').and_return(@task) - allow(@task).to receive(:invoke) - allow(@dump).to receive(:assets_root_link).and_yield('/tmp', 'assets') - end + describe 'assets' do + Assets = Reader::Assets + describe Assets do + before do + @dump = Dump::Reader.new('123.tgz') + @assets_reader = described_class.new(@dump) + + @e1 = double('e1', :full_name => 'config', :read => 'config_data') + @e2 = double('e2', :full_name => 'first.dump', :read => 'first.dump_data') + @e3 = double('e3', :full_name => 'second.dump', :read => 'second.dump_data') + @stream = [@e1, @e2, @e3] + allow(@dump).to receive(:stream).and_return(@stream) + end - it 'does not read assets if config[:assets] is nil' do - allow(@dump).to receive(:config).and_return({}) - expect(@dump).not_to receive(:find_entry) - @dump.read_assets - end + describe 'read_assets' do + before do + @task = double('task') + allow(Rake::Task).to receive(:[]).with('assets:delete').and_return(@task) + allow(@task).to receive(:invoke) + allow(@dump).to receive(:assets_root_link).and_yield('/tmp', 'assets') + end - it 'does not read assets if config[:assets] is blank' do - allow(@dump).to receive(:config).and_return({:assets => []}) - expect(@dump).not_to receive(:find_entry) - @dump.read_assets - end + it 'does not read assets if config[:assets] is nil' do + allow(@dump).to receive(:config).and_return({}) + expect(@dump).not_to receive(:find_entry) + @assets_reader.read + end - describe 'deleting existing assets' do - before do - allow(@stream).to receive(:each) - end + it 'does not read assets if config[:assets] is blank' do + allow(@dump).to receive(:config).and_return({:assets => []}) + expect(@dump).not_to receive(:find_entry) + @assets_reader.read + end - it 'calls assets:delete' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) - allow(@dump).to receive(:find_entry) + describe 'deleting existing assets' do + before do + allow(@stream).to receive(:each) + end - expect(@task).to receive(:invoke) + it 'calls assets:delete' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) + allow(@dump).to receive(:find_entry) - @dump.read_assets - end + expect(@task).to receive(:invoke) - it 'calls assets:delete with ASSETS set to config[:assets] joined with :' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) - allow(@dump).to receive(:find_entry) + @assets_reader.read + end - expect(@task).to receive(:invoke) do - expect(Dump::Env[:assets]).to eq('images:videos') - end + it 'calls assets:delete with ASSETS set to config[:assets] joined with :' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) + allow(@dump).to receive(:find_entry) - @dump.read_assets - end + expect(@task).to receive(:invoke) do + expect(Dump::Env[:assets]).to eq('images:videos') + end - describe 'when called with restore_assets' do - it 'deletes files and dirs only in requested paths' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) - - expect(Dump::Assets).to receive('glob_asset_children').with('images', '**/*').and_return(%w[images images/a.jpg images/b.jpg]) - expect(Dump::Assets).to receive('glob_asset_children').with('videos', '**/*').and_return(%w[videos videos/a.mov]) - - expect(@dump).to receive('read_asset?').with('images/b.jpg', Dump.rails_root).ordered.and_return(false) - expect(@dump).to receive('read_asset?').with('images/a.jpg', Dump.rails_root).ordered.and_return(true) - expect(@dump).to receive('read_asset?').with('images', Dump.rails_root).ordered.and_return(true) - expect(@dump).to receive('read_asset?').with('videos/a.mov', Dump.rails_root).ordered.and_return(false) - expect(@dump).to receive('read_asset?').with('videos', Dump.rails_root).ordered.and_return(false) - - expect(File).to receive('file?').with('images/a.jpg').and_return(true) - expect(File).to receive('unlink').with('images/a.jpg') - expect(File).not_to receive('file?').with('images/b.jpg') - expect(File).to receive('file?').with('images').and_return(false) - expect(File).to receive('directory?').with('images').and_return(true) - expect(Dir).to receive('unlink').with('images').and_raise(Errno::ENOTEMPTY) - - Dump::Env.with_env(:restore_assets => 'images/a.*:stylesheets') do - @dump.read_assets + @assets_reader.read end - end - it 'does not delete any files and dirs for empty list' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) + describe 'when called with restore_assets' do + it 'deletes files and dirs only in requested paths' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) + + expect(Dump::Assets).to receive('glob_asset_children').with('images', '**/*').and_return(%w[images images/a.jpg images/b.jpg]) + expect(Dump::Assets).to receive('glob_asset_children').with('videos', '**/*').and_return(%w[videos videos/a.mov]) + + expect(@assets_reader).to receive('read_asset?').with('images/b.jpg', Dump.rails_root).ordered.and_return(false) + expect(@assets_reader).to receive('read_asset?').with('images/a.jpg', Dump.rails_root).ordered.and_return(true) + expect(@assets_reader).to receive('read_asset?').with('images', Dump.rails_root).ordered.and_return(true) + expect(@assets_reader).to receive('read_asset?').with('videos/a.mov', Dump.rails_root).ordered.and_return(false) + expect(@assets_reader).to receive('read_asset?').with('videos', Dump.rails_root).ordered.and_return(false) + + expect(File).to receive('file?').with('images/a.jpg').and_return(true) + expect(File).to receive('unlink').with('images/a.jpg') + expect(File).not_to receive('file?').with('images/b.jpg') + expect(File).to receive('file?').with('images').and_return(false) + expect(File).to receive('directory?').with('images').and_return(true) + expect(Dir).to receive('unlink').with('images').and_raise(Errno::ENOTEMPTY) + + Dump::Env.with_env(:restore_assets => 'images/a.*:stylesheets') do + @assets_reader.read + end + end + + it 'does not delete any files and dirs for empty list' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) - expect(Dump::Assets).not_to receive('glob_asset_children') + expect(Dump::Assets).not_to receive('glob_asset_children') - expect(@dump).not_to receive('read_asset?') + expect(@dump).not_to receive('read_asset?') - expect(File).not_to receive('directory?') - expect(File).not_to receive('file?') - expect(File).not_to receive('unlink') + expect(File).not_to receive('directory?') + expect(File).not_to receive('file?') + expect(File).not_to receive('unlink') - Dump::Env.with_env(:restore_assets => '') do - @dump.read_assets + Dump::Env.with_env(:restore_assets => '') do + @assets_reader.read + end + end end end - end - end - describe 'old style' do - it 'finds assets.tar' do - @assets = %w[images videos] - allow(@dump).to receive(:config).and_return({:assets => @assets}) - allow(Dir).to receive(:glob).and_return([]) - allow(FileUtils).to receive(:remove_entry) - allow(@stream).to receive(:each) + describe 'old style' do + it 'finds assets.tar' do + @assets = %w[images videos] + allow(@dump).to receive(:config).and_return({:assets => @assets}) + allow(Dir).to receive(:glob).and_return([]) + allow(FileUtils).to receive(:remove_entry) + allow(@stream).to receive(:each) - expect(@dump).to receive(:find_entry).with('assets.tar') - @dump.read_assets - end - - [ - %w[images videos], - {'images' => 0, 'videos' => 0}, - {'images' => {:files => 0, :total => 0}, 'videos' => {:files => 0, :total => 0}}, - ].each do |assets| - it 'rewrites rewind method to empty method - to not raise exception, opens tar and extracts each entry' do - allow(@dump).to receive(:config).and_return({:assets => assets}) - allow(Dir).to receive(:glob).and_return([]) - allow(FileUtils).to receive(:remove_entry) - - @assets_tar = double('assets_tar') - allow(@assets_tar).to receive(:rewind).and_raise('hehe - we want to rewind to center of gzip') - allow(@dump).to receive(:find_entry).and_yield(@assets_tar) - - @inp = double('inp') - each_excpectation = expect(@inp).to receive(:each) - @entries = %w[a b c d].map do |s| - file = double("file_#{s}") - each_excpectation.and_yield(file) - expect(@inp).to receive(:extract_entry).with(Dump.rails_root, file) - file + expect(@dump).to receive(:find_entry).with('assets.tar') + @assets_reader.read end - expect(Archive::Tar::Minitar).to receive(:open).with(@assets_tar).and_yield(@inp) - @dump.read_assets + [ + %w[images videos], + {'images' => 0, 'videos' => 0}, + {'images' => {:files => 0, :total => 0}, 'videos' => {:files => 0, :total => 0}}, + ].each do |assets| + it 'rewrites rewind method to empty method - to not raise exception, opens tar and extracts each entry' do + allow(@dump).to receive(:config).and_return({:assets => assets}) + allow(Dir).to receive(:glob).and_return([]) + allow(FileUtils).to receive(:remove_entry) + + @assets_tar = double('assets_tar') + allow(@assets_tar).to receive(:rewind).and_raise('hehe - we want to rewind to center of gzip') + allow(@dump).to receive(:find_entry).and_yield(@assets_tar) + + @inp = double('inp') + each_excpectation = expect(@inp).to receive(:each) + @entries = %w[a b c d].map do |s| + file = double("file_#{s}") + each_excpectation.and_yield(file) + expect(@inp).to receive(:extract_entry).with(Dump.rails_root, file) + file + end + expect(Archive::Tar::Minitar).to receive(:open).with(@assets_tar).and_yield(@inp) + + @assets_reader.read + end + end end - end - end - describe 'new style' do - before do - expect(@dump).to receive(:find_entry).with('assets.tar') - end - - [ - %w[images videos], - {'images' => 0, 'videos' => 0}, - {'images' => {:files => 0, :total => 0}, 'videos' => {:files => 0, :total => 0}}, - ].each do |assets| - it 'extracts each entry' do - allow(@dump).to receive(:config).and_return({:assets => assets}) - allow(Dir).to receive(:glob).and_return([]) - allow(FileUtils).to receive(:remove_entry) - - expect(@dump).to receive(:assets_root_link).and_yield('/tmp/abc', 'assets') - each_excpectation = expect(@stream).to receive(:each) - @entries = %w[a b c d].map do |s| - file = double("file_#{s}", :full_name => "assets/#{s}") - each_excpectation.and_yield(file) - expect(@stream).to receive(:extract_entry).with('/tmp/abc', file) - file + describe 'new style' do + before do + expect(@dump).to receive(:find_entry).with('assets.tar') end - other_file = double('other_file', :full_name => 'other_file') - each_excpectation.and_yield(other_file) - expect(@stream).not_to receive(:extract_entry).with('/tmp/abc', other_file) - @dump.read_assets + [ + %w[images videos], + {'images' => 0, 'videos' => 0}, + {'images' => {:files => 0, :total => 0}, 'videos' => {:files => 0, :total => 0}}, + ].each do |assets| + it 'extracts each entry' do + allow(@dump).to receive(:config).and_return({:assets => assets}) + allow(Dir).to receive(:glob).and_return([]) + allow(FileUtils).to receive(:remove_entry) + + expect(@dump).to receive(:assets_root_link).and_yield('/tmp/abc', 'assets') + each_excpectation = expect(@stream).to receive(:each) + @entries = %w[a b c d].map do |s| + file = double("file_#{s}", :full_name => "assets/#{s}") + each_excpectation.and_yield(file) + expect(@stream).to receive(:extract_entry).with('/tmp/abc', file) + file + end + other_file = double('other_file', :full_name => 'other_file') + each_excpectation.and_yield(other_file) + expect(@stream).not_to receive(:extract_entry).with('/tmp/abc', other_file) + + @assets_reader.read + end + end end end - end - end - describe 'read_asset?' do - it 'creates filter and call custom_pass? on it' do - @filter = double('filter') - allow(@filter).to receive('custom_pass?') + describe 'read_asset?' do + it 'creates filter and call custom_pass? on it' do + @filter = double('filter') + allow(@filter).to receive('custom_pass?') - expect(Dump::Env).to receive('filter').with(:restore_assets, Dump::Assets::SPLITTER).and_return(@filter) + expect(Dump::Env).to receive('filter').with(:restore_assets, Dump::Assets::SPLITTER).and_return(@filter) - @dump.read_asset?('a', 'b') - end + @assets_reader.read_asset?('a', 'b') + end - it 'tests path usint fnmatch' do - Dump::Env.with_env(:restore_assets => '[a-b]') do - expect(@dump.read_asset?('x/a', 'x')).to be_truthy - expect(@dump.read_asset?('x/b/file', 'x')).to be_truthy - expect(@dump.read_asset?('x/c', 'x')).to be_falsey + it 'tests path usint fnmatch' do + Dump::Env.with_env(:restore_assets => '[a-b]') do + expect(@assets_reader.read_asset?('x/a', 'x')).to be_truthy + expect(@assets_reader.read_asset?('x/b/file', 'x')).to be_truthy + expect(@assets_reader.read_asset?('x/c', 'x')).to be_falsey + end + end end end end