From a1d64a71b9b8492c02778c36e8e102883ad76297 Mon Sep 17 00:00:00 2001 From: Andrei Lazarescu Date: Thu, 13 Feb 2025 10:08:03 +0200 Subject: [PATCH 1/5] Support specifying the ingest host --- lib/fluent/plugin/out_logtail.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin/out_logtail.rb b/lib/fluent/plugin/out_logtail.rb index 19d2cea..98c2204 100644 --- a/lib/fluent/plugin/out_logtail.rb +++ b/lib/fluent/plugin/out_logtail.rb @@ -7,7 +7,6 @@ class LogtailOutput < Fluent::BufferedOutput VERSION = "0.1.1".freeze CONTENT_TYPE = "application/msgpack".freeze - HOST = "in.logtail.com".freeze PORT = 443 PATH = "/".freeze MAX_ATTEMPTS = 3.freeze @@ -16,9 +15,11 @@ class LogtailOutput < Fluent::BufferedOutput config_param :source_token, :string, secret: true config_param :ip, :string, default: nil + config_param :ingesting_host, :string, default: "in.logs.betterstack.com" def configure(conf) @source_token = conf["source_token"] + @ingesting_host = conf["ingesting_host"] super end @@ -90,7 +91,7 @@ def force_utf8_string_values(data) end def build_http_client - http = Net::HTTP.new(HOST, PORT) + http = Net::HTTP.new(@ingesting_host, PORT) http.use_ssl = true # Verification on Windows fails despite having a valid certificate. http.verify_mode = OpenSSL::SSL::VERIFY_NONE From 871d0512e7a6fa7416fce3876ea9384a2f8161d2 Mon Sep 17 00:00:00 2001 From: Andrei Lazarescu Date: Thu, 13 Feb 2025 10:09:06 +0200 Subject: [PATCH 2/5] Bump version --- fluent-plugin-logtail.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-plugin-logtail.gemspec b/fluent-plugin-logtail.gemspec index 380f44d..9a67cc5 100644 --- a/fluent-plugin-logtail.gemspec +++ b/fluent-plugin-logtail.gemspec @@ -3,7 +3,7 @@ require 'date' Gem::Specification.new do |s| s.name = 'fluent-plugin-logtail' - s.version = '0.1.1' + s.version = '0.1.2' s.date = Date.today.to_s s.summary = 'Logtail.com plugin for Fluentd' s.description = 'Streams Fluentd logs to the Logtail.com logging service.' From 44d4a27b4a63cd9cdbfed488b22544b7246a77d3 Mon Sep 17 00:00:00 2001 From: Andrei Lazarescu Date: Thu, 13 Feb 2025 10:12:28 +0200 Subject: [PATCH 3/5] updated tests --- spec/fluent/plugin/out_logtail_spec.rb | 41 ++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/spec/fluent/plugin/out_logtail_spec.rb b/spec/fluent/plugin/out_logtail_spec.rb index be71afc..067af84 100644 --- a/spec/fluent/plugin/out_logtail_spec.rb +++ b/spec/fluent/plugin/out_logtail_spec.rb @@ -8,6 +8,13 @@ } end + let(:cloud_config) do + %{ + source_token abcd1234 + ingesting_host s1234.g1.betterstackdata.com + } + end + let(:driver) do tag = "test" Fluent::Test::BufferedOutputTestDriver.new(Fluent::LogtailOutput, tag) { @@ -19,6 +26,19 @@ def format(tag, time, record) end }.configure(config) end + + let(:cloud_driver) do + tag = "test" + Fluent::Test::BufferedOutputTestDriver.new(Fluent::LogtailOutput, tag) { + # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format + if !defined?(Fluent::Plugin::Output) + def format(tag, time, record) + [time, record].to_msgpack + end + end + }.configure(cloud_config) + end + let(:record) do {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'} end @@ -29,7 +49,22 @@ def format(tag, time, record) describe "#write" do it "should send a chunked request to the Logtail API" do - stub = stub_request(:post, "https://in.logtail.com/"). + stub = stub_request(:post, "https://s1234.g1.betterstackdata.com/"). + with( + :body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")), + :headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'Logtail Fluentd/0.1.1'} + ). + to_return(:status => 202, :body => "", :headers => {}) + + cloud_driver.emit(record) + cloud_driver.run + + expect(stub).to have_been_requested.times(1) + end + + describe "#write to cloud" do + it "should send a chunked request to the Logtail API" do + stub = stub_request(:post, "https://in.logs.betterstack.com/"). with( :body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")), :headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'Logtail Fluentd/0.1.1'} @@ -43,7 +78,7 @@ def format(tag, time, record) end it "handles 500s" do - stub = stub_request(:post, "https://in.logtail.com/").to_return(:status => 500, :body => "", :headers => {}) + stub = stub_request(:post, "https://in.logs.betterstack.com/").to_return(:status => 500, :body => "", :headers => {}) driver.emit(record) driver.run @@ -52,7 +87,7 @@ def format(tag, time, record) end it "handle auth failures" do - stub = stub_request(:post, "https://in.logtail.com/").to_return(:status => 403, :body => "", :headers => {}) + stub = stub_request(:post, "https://in.logs.betterstack.com/").to_return(:status => 403, :body => "", :headers => {}) driver.emit(record) driver.run From e4c8a9a68cc1dcc4f242df2988ce2e3921e6fc3d Mon Sep 17 00:00:00 2001 From: Andrei Lazarescu Date: Thu, 13 Feb 2025 10:30:03 +0200 Subject: [PATCH 4/5] updated tests :white_check_mark: --- spec/fluent/plugin/out_logtail_spec.rb | 29 +++++++++++++------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/spec/fluent/plugin/out_logtail_spec.rb b/spec/fluent/plugin/out_logtail_spec.rb index 067af84..bc479a2 100644 --- a/spec/fluent/plugin/out_logtail_spec.rb +++ b/spec/fluent/plugin/out_logtail_spec.rb @@ -62,32 +62,33 @@ def format(tag, time, record) expect(stub).to have_been_requested.times(1) end - describe "#write to cloud" do - it "should send a chunked request to the Logtail API" do - stub = stub_request(:post, "https://in.logs.betterstack.com/"). - with( - :body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")), - :headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'Logtail Fluentd/0.1.1'} - ). - to_return(:status => 202, :body => "", :headers => {}) + it "handles 500s" do + stub = stub_request(:post, "https://in.logs.betterstack.com/").to_return(:status => 500, :body => "", :headers => {}) driver.emit(record) driver.run - expect(stub).to have_been_requested.times(1) + expect(stub).to have_been_requested.times(3) end - it "handles 500s" do - stub = stub_request(:post, "https://in.logs.betterstack.com/").to_return(:status => 500, :body => "", :headers => {}) + it "handle auth failures" do + stub = stub_request(:post, "https://in.logs.betterstack.com/").to_return(:status => 403, :body => "", :headers => {}) driver.emit(record) driver.run - expect(stub).to have_been_requested.times(3) + expect(stub).to have_been_requested.times(1) end + end - it "handle auth failures" do - stub = stub_request(:post, "https://in.logs.betterstack.com/").to_return(:status => 403, :body => "", :headers => {}) + describe "#write to cloud" do + it "should send a chunked request to the Logtail API" do + stub = stub_request(:post, "https://in.logs.betterstack.com/"). + with( + :body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")), + :headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'Logtail Fluentd/0.1.1'} + ). + to_return(:status => 202, :body => "", :headers => {}) driver.emit(record) driver.run From 7420fb0dff9f3de124ab263aef399d9c61c4ff93 Mon Sep 17 00:00:00 2001 From: Andrei Lazarescu Date: Thu, 13 Feb 2025 23:22:31 +0200 Subject: [PATCH 5/5] :lipstick: --- fluent-plugin-logtail.gemspec | 2 +- spec/fluent/plugin/out_logtail_spec.rb | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fluent-plugin-logtail.gemspec b/fluent-plugin-logtail.gemspec index 9a67cc5..0706d15 100644 --- a/fluent-plugin-logtail.gemspec +++ b/fluent-plugin-logtail.gemspec @@ -3,7 +3,7 @@ require 'date' Gem::Specification.new do |s| s.name = 'fluent-plugin-logtail' - s.version = '0.1.2' + s.version = '0.2.1' s.date = Date.today.to_s s.summary = 'Logtail.com plugin for Fluentd' s.description = 'Streams Fluentd logs to the Logtail.com logging service.' diff --git a/spec/fluent/plugin/out_logtail_spec.rb b/spec/fluent/plugin/out_logtail_spec.rb index bc479a2..51c3404 100644 --- a/spec/fluent/plugin/out_logtail_spec.rb +++ b/spec/fluent/plugin/out_logtail_spec.rb @@ -11,7 +11,7 @@ let(:cloud_config) do %{ source_token abcd1234 - ingesting_host s1234.g1.betterstackdata.com + ingesting_host s1234.eu-nbg-2.betterstackdata.com } end @@ -48,16 +48,16 @@ def format(tag, time, record) end describe "#write" do - it "should send a chunked request to the Logtail API" do - stub = stub_request(:post, "https://s1234.g1.betterstackdata.com/"). + it "should send a chunked request to the Logtail API using default host" do + stub = stub_request(:post, "https://in.logs.betterstack.com/"). with( :body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")), :headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'Logtail Fluentd/0.1.1'} ). to_return(:status => 202, :body => "", :headers => {}) - cloud_driver.emit(record) - cloud_driver.run + driver.emit(record) + driver.run expect(stub).to have_been_requested.times(1) end @@ -83,15 +83,15 @@ def format(tag, time, record) describe "#write to cloud" do it "should send a chunked request to the Logtail API" do - stub = stub_request(:post, "https://in.logs.betterstack.com/"). + stub = stub_request(:post, "https://s1234.eu-nbg-2.betterstackdata.com/"). with( :body => start_with("\xDD\x00\x00\x00\x01\x85\xA3age\x1A\xAArequest_id\xA242\xA9parent_id\xA6parent\xAArouting_id\xA7routing\xA2dt\xB4".force_encoding("ASCII-8BIT")), :headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Authorization'=>'Bearer abcd1234', 'Content-Type'=>'application/msgpack', 'User-Agent'=>'Logtail Fluentd/0.1.1'} ). to_return(:status => 202, :body => "", :headers => {}) - driver.emit(record) - driver.run + cloud_driver.emit(record) + cloud_driver.run expect(stub).to have_been_requested.times(1) end