From f0f2b7efbef162ca64e5c57821b8b6a12bf0df3d Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Wed, 10 Dec 2025 22:33:18 +0200 Subject: [PATCH 1/5] remove DEPRECATION --- features/support/hooks.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/support/hooks.rb b/features/support/hooks.rb index 4bf7a29514..4eff6149b2 100644 --- a/features/support/hooks.rb +++ b/features/support/hooks.rb @@ -7,7 +7,7 @@ end After do - ActiveRecord::Base.clear_active_connections! + ActiveRecord::Base.connection_handler.clear_active_connections! end Around '@security' do |scenario, block| From 49d6769d98f6cea73d7e61b48644b64b2503d8a0 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Wed, 10 Dec 2025 23:03:36 +0200 Subject: [PATCH 2/5] oracle performance --- Gemfile | 1 + config/initializers/oracle.rb | 29 +++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 911ad1fcaa..e8048d2d92 100644 --- a/Gemfile +++ b/Gemfile @@ -251,6 +251,7 @@ gem 'unicorn', require: false, group: %i[production] # NOTE: Use ENV['DB'] only to install oracle dependencies group :oracle do oracle = -> { (ENV['ORACLE'] == '1') || ENV.fetch('DATABASE_URL', ENV['DB'])&.start_with?('oracle') } + ENV['NLS_LANG'] ||= 'AMERICAN_AMERICA.UTF8' if oracle gem 'activerecord-oracle_enhanced-adapter', '~> 7.1.0', install_if: oracle gem 'ruby-oci8', require: false, install_if: oracle end diff --git a/config/initializers/oracle.rb b/config/initializers/oracle.rb index a40511a009..4e62c65a53 100644 --- a/config/initializers/oracle.rb +++ b/config/initializers/oracle.rb @@ -2,7 +2,7 @@ ActiveSupport.on_load(:active_record) do if System::Database.oracle? - require 'arel/visitors/oracle12_hack' + require 'arel/visitors/oracle12_hack' || next # once done, we can skip setup # in 6.0.6 automatic detection of max identifier length was introduced # see https://github.com/rsim/oracle-enhanced/pull/1703 @@ -30,9 +30,20 @@ def column(name, type, **options) end end) - ENV['NLS_LANG'] ||= 'AMERICAN_AMERICA.UTF8' + # clean-up prepared statements/cursors on connection return to pool + module OracleStatementCleanup + def self.included(base) + base.set_callback :checkin, :after, :close_and_clear_statements + end + + def close_and_clear_statements + @statements&.clear + end + end ActiveRecord::ConnectionAdapters::OracleEnhancedAdapter.class_eval do + include OracleStatementCleanup + # Fixing OCIError: ORA-01741: illegal zero-length identifier # because of https://github.com/rails/rails/commit/c18a95e38e9860953236aed94c1bfb877fa3be84 # the value of `columns` is [ "\"ACCOUNTS\".\"ID\"" ] which forms an incorrect query @@ -273,5 +284,19 @@ def column_definitions(table_name) end ActiveRecord::ConnectionAdapters::OracleEnhancedAdapter.prepend OracleEnhancedAdapterSchemaIssue2276 + + # see https://github.com/kubo/ruby-oci8/pull/271 + module OCI8DisableArrayFetch + private + def define_one_column(pos, param) + @fetch_array_size = nil # disable memory array fetching anytime + super # call original + end + end + + OCI8::Cursor.prepend(OCI8DisableArrayFetch) + + OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long + OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw end end From 8762d669eaa9c518eeefe84c8a9ebbe5bf1ce920 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Fri, 12 Dec 2025 21:50:55 +0200 Subject: [PATCH 3/5] proper fix for writing lobs --- config/initializers/oracle.rb | 67 +++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/config/initializers/oracle.rb b/config/initializers/oracle.rb index 4e62c65a53..6e4e59adae 100644 --- a/config/initializers/oracle.rb +++ b/config/initializers/oracle.rb @@ -41,8 +41,53 @@ def close_and_clear_statements end end + ActiveRecord::Base.skip_callback(:update, :after, :enhanced_write_lobs) + + # We need to patch Oracle Adapter quoting to actually serialize CLOB columns. + # https://github.com/rsim/oracle-enhanced/issues/1588#issue-272289146 + # The default behaviour is to serialize them to 'empty_clob()' basically wiping out the data. + # The team behind it believes `Table.update_all(column: 'text')` + # should wipe all your data in that column: https://github.com/rsim/oracle-enhanced/issues/1588#issuecomment-343353756 + # So we try to convert the text to using `to_clob` function. + module OracleEnhancedSmartQuoting + CLOB_INLINE_LIMIT = 32767 # 32KB - 1 + BLOB_INLINE_LIMIT = 16383 # 16KB - 1 (hex encoding doubles size) + + def quote(value) + case value + when ActiveModel::Type::Binary::Data, ActiveRecord::Type::OracleEnhanced::Text::Data + raise ArgumentError, "trying to prove that we never reach here" + when ActiveModel::Type::Binary::Data + raw = value.to_s + size = raw.bytesize + + if size == 0 + "empty_blob()" + elsif size <= BLOB_INLINE_LIMIT + "hextoraw('#{raw.unpack1('H*')}')" + else + raise ArgumentError, "BLOB too large for inline quoting (#{size} bytes, max #{BLOB_INLINE_LIMIT} bytes). Use bind parameters instead." + end + when ActiveRecord::Type::OracleEnhanced::Text::Data + text = value.to_s + size = text.bytesize + + if size == 0 + "empty_clob()" + elsif size <= CLOB_INLINE_LIMIT + "to_clob(#{super(text)})" + else + raise ArgumentError, "CLOB too large for inline quoting (#{size} bytes, max #{CLOB_INLINE_LIMIT} bytes). Use bind parameters instead." + end + else + super + end + end + end + ActiveRecord::ConnectionAdapters::OracleEnhancedAdapter.class_eval do include OracleStatementCleanup + prepend OracleEnhancedSmartQuoting # Fixing OCIError: ORA-01741: illegal zero-length identifier # because of https://github.com/rails/rails/commit/c18a95e38e9860953236aed94c1bfb877fa3be84 @@ -76,24 +121,6 @@ def add_column(table_name, column_name, type, **options) end end - # We need to patch Oracle Adapter quoting to actually serialize CLOB columns. - # https://github.com/rsim/oracle-enhanced/issues/1588#issue-272289146 - # The default behaviour is to serialize them to 'empty_clob()' basically wiping out the data. - # The team behind it believes `Table.update_all(column: 'text')` - # should wipe all your data in that column: https://github.com/rsim/oracle-enhanced/issues/1588#issuecomment-343353756 - # So we try to convert the text to using `to_clob` function. - def _quote(value) - case value - when ActiveModel::Type::Binary::Data - # I know this looks ugly, but that just modified copy paste of what the adapter does (minus the rescue). - # It is a bit improved in next version due to ActiveRecord Attributes API. - %{to_blob(#{quote(value.to_s)})} - when ActiveRecord::Type::OracleEnhanced::Text::Data - %{to_clob(#{quote(value.to_s)})} - else - super - end - end end) end @@ -296,6 +323,10 @@ def define_one_column(pos, param) OCI8::Cursor.prepend(OCI8DisableArrayFetch) + # see https://github.com/kubo/ruby-oci8/pull/271 + # Enable piecewise retrieval for both CLOBs and BLOBs + # With the OCIConnectionCursorLobFix above, we can safely use both mappings + # because LOBs are bound as OCI8::CLOB/BLOB objects, not LONG data OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw end From 4c3fd2340b77a5810e8cfc677ac1f9bbdc0d16e4 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Mon, 15 Dec 2025 20:52:32 +0200 Subject: [PATCH 4/5] add tests --- config/initializers/oracle.rb | 6 +- .../oracle_lob_large_update_test.rb | 110 ++++++++++++++++++ 2 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 test/integration/oracle_lob_large_update_test.rb diff --git a/config/initializers/oracle.rb b/config/initializers/oracle.rb index 6e4e59adae..2e7eeb9622 100644 --- a/config/initializers/oracle.rb +++ b/config/initializers/oracle.rb @@ -43,20 +43,18 @@ def close_and_clear_statements ActiveRecord::Base.skip_callback(:update, :after, :enhanced_write_lobs) - # We need to patch Oracle Adapter quoting to actually serialize CLOB columns. + # Supposedly in the past `update_all` and `where` used this with issues. # https://github.com/rsim/oracle-enhanced/issues/1588#issue-272289146 # The default behaviour is to serialize them to 'empty_clob()' basically wiping out the data. # The team behind it believes `Table.update_all(column: 'text')` # should wipe all your data in that column: https://github.com/rsim/oracle-enhanced/issues/1588#issuecomment-343353756 - # So we try to convert the text to using `to_clob` function. + # Might be dead code now. module OracleEnhancedSmartQuoting CLOB_INLINE_LIMIT = 32767 # 32KB - 1 BLOB_INLINE_LIMIT = 16383 # 16KB - 1 (hex encoding doubles size) def quote(value) case value - when ActiveModel::Type::Binary::Data, ActiveRecord::Type::OracleEnhanced::Text::Data - raise ArgumentError, "trying to prove that we never reach here" when ActiveModel::Type::Binary::Data raw = value.to_s size = raw.bytesize diff --git a/test/integration/oracle_lob_large_update_test.rb b/test/integration/oracle_lob_large_update_test.rb new file mode 100644 index 0000000000..7cdeb33fda --- /dev/null +++ b/test/integration/oracle_lob_large_update_test.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +require 'test_helper' + +class OracleLobLargeUpdateTest < ActiveSupport::TestCase + # Test large LOB (CLOB and BLOB) updates with OCI8::BindType::Long/LongRaw piecewise retrieval + # This verifies the OCIConnectionCursorLobFix in config/initializers/oracle.rb works correctly + # by using OCI8::CLOB.new() and OCI8::BLOB.new() for direct binding + + setup do + skip "Only run on Oracle database" unless System::Database.oracle? + + @service = FactoryBot.create(:simple_service) + @proxy = @service.proxy + end + + test "update and retrieve large CLOB (policies_config) with 512KB data" do + # Generate ~512KB of JSON data for policies_config + large_policy_data = generate_large_policies_config(2.kilobytes) + + # Initial save using ActiveRecord + @proxy.policies_config = large_policy_data + @proxy.save! + + @proxy.reload + retrieved_policies = @proxy.policies_config + assert retrieved_policies.to_json.bytesize >= 2.kilobytes + + expected_policy = JSON.parse(large_policy_data).first + assert_includes retrieved_policies.map(&:to_h), expected_policy + + # NOW UPDATE with different data + large_policy_data_v2 = generate_large_policies_config(512.kilobytes, version: "2.0") + @proxy.reload + @proxy.policies_config = large_policy_data_v2 + @proxy.save! + + @proxy.reload + retrieved_policies_v2 = @proxy.policies_config + + assert retrieved_policies_v2.to_json.bytesize >= 512.kilobytes + + expected_policy_v2 = JSON.parse(large_policy_data_v2).first + assert_includes retrieved_policies_v2.map(&:to_h), expected_policy_v2 + end + + test "update and retrieve large BLOB (MemberPermission service_ids) with 512KB data" do + # Simple test model double for MemberPermission to avoid the JSON serialization logic + test_class = Class.new(ActiveRecord::Base) do + self.table_name = 'member_permissions' + end + + small_binary_data = Random.bytes(2.kilobytes) + test_record = test_class.create!(service_ids: small_binary_data) + test_record.reload + assert_equal small_binary_data, test_record.service_ids + + # NOW UPDATE with different random binary data + large_binary_data = Random.bytes(512.kilobytes) + test_record.service_ids = large_binary_data + test_record.save! + + test_record.reload + retrieved_value = test_record.service_ids + + assert_equal large_binary_data.bytesize, retrieved_value.bytesize,"Updated binary data size should match" + assert_equal large_binary_data, retrieved_value,"Updated service_ids should match new binary data" + end + + # practical max is 2GB - 8 bytes; can be increased with a OCI8::BLOB and CLOB fixes to write big data in chunks + test "update CLOB with multiple sizes to verify across internal thresholds" do + sizes = [ + 3.kilobytes, # Smaller than Oracle VARCHAR2 limit + 31.kilobytes, # Larger than standard VARCHAR2, within EXTENDED limit + 256.kilobytes # Large size + ] + + sizes.each do |size| + large_data = generate_large_policies_config(size) + + # Update using ActiveRecord (this will trigger write_lobs) + @proxy.policies_config = large_data + @proxy.save! + + actual_policy = @proxy.reload.policies_config + assert_includes actual_policy, OpenStruct.new(JSON.parse(large_data).first) + end + end + + private + + # Generate large JSON policies config data + def generate_large_policies_config(target_size, version="1.0") + # Generate a test JSON and calculate actual overhead + test_policy = { + "name" => "test_policy", + "version" => version, + "configuration" => { "data" => "" }, + "enabled" => true + } + + test_json = [test_policy].to_json + padding_size = target_size - test_json.bytesize + + return test_json if padding_size <= 0 + + test_policy["configuration"]["data"] = "X" * padding_size + [test_policy].to_json + end +end From 0044b2f3eaad024ef47ac1aba212c2be258e165e Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Thu, 8 Jan 2026 12:22:53 +0200 Subject: [PATCH 5/5] debug oracle cursor cleanup --- config/initializers/oracle.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/config/initializers/oracle.rb b/config/initializers/oracle.rb index 2e7eeb9622..81a2c56143 100644 --- a/config/initializers/oracle.rb +++ b/config/initializers/oracle.rb @@ -37,7 +37,26 @@ def self.included(base) end def close_and_clear_statements + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + statement_count = @statements&.length || 0 @statements&.clear + + # Query V$OPEN_CURSOR to see current cursor count for this session + # Count application cursors (OPEN for regular SQL, PL/SQL for stored procs/funcs) + # Exclude Oracle's internal cursors (DICTIONARY LOOKUP, OPEN RECURSIVE, etc.) + begin + cursor_count = select_value(<<~SQL) + SELECT COUNT(*) + FROM V$OPEN_CURSOR + WHERE SID = SYS_CONTEXT('USERENV', 'SID') + AND CURSOR_TYPE IN ('OPEN', 'PL/SQL') + AND (SQL_TEXT IS NULL OR SQL_TEXT NOT LIKE '%V$OPEN_CURSOR%') + SQL + duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time + Rails.logger.info "#{statement_count} statements cleared from AR pool on checkin in #{duration.round(3)}s (Oracle reports #{cursor_count} cursors in session)" + rescue => e + Rails.logger.warn "Failed to query V$OPEN_CURSOR: #{e.message}" + end end end