diff --git a/Gemfile b/Gemfile index 73ae19d5..88217cff 100644 --- a/Gemfile +++ b/Gemfile @@ -75,6 +75,11 @@ gem 'newrelic_rpm', '~> 9.16' # For console gem 'pry', '~> 0.15' +# opentelemetry (otlp) +gem 'opentelemetry-sdk' +gem 'opentelemetry-exporter-otlp' +gem 'opentelemetry-instrumentation-all' + # Development tools group :development do gem 'grape-raketasks' diff --git a/Gemfile.lock b/Gemfile.lock index a697d6fa..aca2d486 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -145,6 +145,9 @@ GEM fiddle (1.1.6) globalid (1.2.1) activesupport (>= 6.1) + google-protobuf (3.25.8) + googleapis-common-protos-types (1.20.0) + google-protobuf (>= 3.18, < 5.a) grape (2.2.0) activesupport (>= 6) dry-types (>= 1.1) @@ -225,6 +228,191 @@ GEM nokogiri (1.18.8) mini_portile2 (~> 2.8.2) racc (~> 1.4) + opentelemetry-api (1.1.0) + opentelemetry-common (0.19.7) + opentelemetry-api (~> 1.0) + opentelemetry-exporter-otlp (0.25.0) + google-protobuf (~> 3.19) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.19.6) + opentelemetry-sdk (~> 1.2) + opentelemetry-semantic_conventions + opentelemetry-instrumentation-action_pack (0.5.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rack (~> 0.21) + opentelemetry-instrumentation-action_view (0.4.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-active_support (~> 0.1) + opentelemetry-instrumentation-base (~> 0.20) + opentelemetry-instrumentation-active_job (0.4.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-active_model_serializers (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-active_record (0.5.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + ruby2_keywords + opentelemetry-instrumentation-active_support (0.3.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-all (0.33.0) + opentelemetry-instrumentation-active_model_serializers (~> 0.19.0) + opentelemetry-instrumentation-aws_sdk (~> 0.3.0) + opentelemetry-instrumentation-bunny (~> 0.19.0) + opentelemetry-instrumentation-concurrent_ruby (~> 0.20.0) + opentelemetry-instrumentation-dalli (~> 0.22.0) + opentelemetry-instrumentation-delayed_job (~> 0.19.0) + opentelemetry-instrumentation-ethon (~> 0.20.0) + opentelemetry-instrumentation-excon (~> 0.20.0) + opentelemetry-instrumentation-faraday (~> 0.22.0) + opentelemetry-instrumentation-graphql (~> 0.24.0) + opentelemetry-instrumentation-http (~> 0.21.0) + opentelemetry-instrumentation-http_client (~> 0.21.0) + opentelemetry-instrumentation-koala (~> 0.19.0) + opentelemetry-instrumentation-lmdb (~> 0.21.0) + opentelemetry-instrumentation-mongo (~> 0.21.0) + opentelemetry-instrumentation-mysql2 (~> 0.22.0) + opentelemetry-instrumentation-net_http (~> 0.21.0) + opentelemetry-instrumentation-pg (~> 0.23.0) + opentelemetry-instrumentation-que (~> 0.5.0) + opentelemetry-instrumentation-racecar (~> 0.1.0) + opentelemetry-instrumentation-rack (~> 0.22.0) + opentelemetry-instrumentation-rails (~> 0.25.0) + opentelemetry-instrumentation-rake (~> 0.1.0) + opentelemetry-instrumentation-rdkafka (~> 0.2.0) + opentelemetry-instrumentation-redis (~> 0.24.0) + opentelemetry-instrumentation-resque (~> 0.3.0) + opentelemetry-instrumentation-restclient (~> 0.21.0) + opentelemetry-instrumentation-ruby_kafka (~> 0.19.0) + opentelemetry-instrumentation-sidekiq (~> 0.22.0) + opentelemetry-instrumentation-sinatra (~> 0.21.0) + opentelemetry-instrumentation-trilogy (~> 0.52.0) + opentelemetry-instrumentation-aws_sdk (0.3.2) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-base (0.21.1) + opentelemetry-api (~> 1.0) + opentelemetry-registry (~> 0.1) + opentelemetry-instrumentation-bunny (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-concurrent_ruby (0.20.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-dalli (0.22.2) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-delayed_job (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-ethon (0.20.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-excon (0.20.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-faraday (0.22.0) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-graphql (0.24.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-http (0.21.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-http_client (0.21.0) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-koala (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-lmdb (0.21.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-mongo (0.21.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-mysql2 (0.22.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-net_http (0.21.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-pg (0.23.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-que (0.5.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-racecar (0.1.2) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rack (0.22.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rails (0.25.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-action_pack (~> 0.5.0) + opentelemetry-instrumentation-action_view (~> 0.4.0) + opentelemetry-instrumentation-active_job (~> 0.4.0) + opentelemetry-instrumentation-active_record (~> 0.5.0) + opentelemetry-instrumentation-active_support (~> 0.3.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rake (0.1.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rdkafka (0.2.3) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-redis (0.24.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-resque (0.3.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-restclient (0.21.0) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-ruby_kafka (0.19.1) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-sidekiq (0.22.1) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-sinatra (0.21.5) + opentelemetry-api (~> 1.0) + opentelemetry-common (~> 0.19.3) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-instrumentation-rack (~> 0.21) + opentelemetry-instrumentation-trilogy (0.52.0) + opentelemetry-api (~> 1.0) + opentelemetry-instrumentation-base (~> 0.21.0) + opentelemetry-semantic_conventions (>= 1.8.0) + opentelemetry-registry (0.2.0) + opentelemetry-api (~> 1.1) + opentelemetry-sdk (1.2.1) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.19.3) + opentelemetry-registry (~> 0.2) + opentelemetry-semantic_conventions + opentelemetry-semantic_conventions (1.11.0) + opentelemetry-api (~> 1.0) ostruct (0.6.1) overcommit (0.64.1) childprocess (>= 0.6.3, < 6) @@ -414,6 +602,9 @@ DEPENDENCIES kramdown (~> 2.5) kramdown-parser-gfm (~> 1.1) newrelic_rpm (~> 9.16) + opentelemetry-exporter-otlp + opentelemetry-instrumentation-all + opentelemetry-sdk ostruct (~> 0.6) overcommit (~> 0.64) paper_trail (~> 16.0) diff --git a/config/application.rb b/config/application.rb index 2e93bb20..ec72aad4 100644 --- a/config/application.rb +++ b/config/application.rb @@ -8,6 +8,12 @@ require 'boot' Bundler.require :default, ENV.fetch('RACK_ENV', nil) +# Modern OpenTelemetry requires (v1.0+) +require 'opentelemetry/sdk' +require 'opentelemetry/exporter/otlp' +require 'opentelemetry-instrumentation-active_job' +require 'opentelemetry-instrumentation-redis' + require 'dotenv_load' require 'airbrake_load' require 'arel_nodes_cte' @@ -57,18 +63,54 @@ def env def logger @logger ||= begin - file_logger = Logger.new(MR.root_path.join('log', "#{MR.env}.log")) - stdout_logger = Logger.new($stdout) - - loggers = [file_logger] - loggers << stdout_logger if MR.env == 'production' - + # Initialize OpenTelemetry if not already done + configure_opentelemetry unless @opentelemetry_configured + + # Create loggers + loggers = [ + Logger.new(root_path.join('log', "#{env}.log")), # File logger + OpenTelemetry.logger # OTLP logger + ] + + # Add stdout in production + loggers << Logger.new($stdout) if env == 'production' + + # Set log level if specified if (log_level = ENV.fetch('LOG_LEVEL', nil)).present? - loggers.each { _1.level = Logger.const_get(log_level) } + loggers.each { |l| l.level = Logger.const_get(log_level.upcase) } + end + + # Combine loggers + ActiveSupport::BroadcastLogger.new(*loggers).tap do |bl| + bl.level = Logger::INFO # Default level end + end + end - ActiveSupport::BroadcastLogger.new(*loggers) + def configure_opentelemetry + return if @opentelemetry_configured + + OpenTelemetry::SDK.configure do |c| + c.service_name = ENV.fetch('OTEL_SERVICE_NAME', 'metadata-registry') + c.service_version = VERSION + + # Auto-instrumentation + c.use 'OpenTelemetry::Instrumentation::Rails' + c.use 'OpenTelemetry::Instrumentation::ActiveJob' + c.use 'OpenTelemetry::Instrumentation::Redis' + + # Configure both traces and logs + c.add_span_processor( + OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: ENV.fetch('OTLP_ENDPOINT', 'http://localhost:4318'), + headers: {} + ) + ) + ) end + + @opentelemetry_configured = true end attr_reader :redis_pool @@ -81,24 +123,26 @@ def root_path MR = MetadataRegistry # Alias for application module +# Configure ActiveJob ActiveJob::Base.queue_adapter = :sidekiq +# ActiveRecord settings ActiveRecord.schema_format = :sql - ActiveRecord::SchemaDumper.ignore_tables = %w[] -# rubocop:todo Layout/LineLength -ActiveSupport.to_time_preserves_timezone = :zone # Opt in to the future behavior in ActiveSupport 8.0 -# rubocop:enable Layout/LineLength - +# Timezone settings +ActiveSupport.to_time_preserves_timezone = :zone Time.zone_default = Time.find_zone!('UTC') Chronic.time_class = Time.zone +# Initialize application components MetadataRegistry.connect MetadataRegistry.statement_timeout(ENV.fetch('STATEMENT_TIMEOUT', '300000')) # 5 min MetadataRegistry.connect_redis +MetadataRegistry.configure_opentelemetry # Ensure OTEL is initialized early +# Load remaining dependencies require 'init_sidekiq' require 'paper_trail' require 'paper_trail/frameworks/active_record' -require 'base' +require 'base' \ No newline at end of file