From 61ca592c0546b705ed5e44de1bbf5f9e0be3efaf Mon Sep 17 00:00:00 2001 From: Mark Murphy Date: Mon, 27 Jul 2026 11:09:23 -0400 Subject: [PATCH] feat(platform): Add protorabbit instrumentor protorabbit worker processes ran no embedded Prometheus exporter server, so any metric pushed from inside message processing (e.g. gRPC client interceptor metrics) was refused with Errno::ECONNREFUSED on /send-metrics and silently dropped. Adds Instrumentors::Protorabbit, following the existing Hutch instrumentor pattern minus the tracer/middleware wiring that protorabbit has no hook for, plus protorabbit_collectors/protorabbit_type_collectors configuration. --- CHANGELOG.md | 4 + README.md | 17 ++++ lib/bigcommerce/prometheus.rb | 1 + lib/bigcommerce/prometheus/configuration.rb | 2 + .../prometheus/instrumentors/protorabbit.rb | 74 ++++++++++++++ lib/bigcommerce/prometheus/version.rb | 2 +- .../instrumentors/protorabbit_spec.rb | 96 +++++++++++++++++++ 7 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 lib/bigcommerce/prometheus/instrumentors/protorabbit.rb create mode 100644 spec/bigcommerce/prometheus/instrumentors/protorabbit_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 50dd450..873ad5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ Changelog for the bc-prometheus-ruby gem. +## 0.9.0 + +- Add `Bigcommerce::Prometheus::Instrumentors::Protorabbit` so protorabbit (RabbitMQ protobuf consumer) processes run an embedded Prometheus exporter server, fixing dropped/refused metric pushes (`Errno::ECONNREFUSED` on `/send-metrics`) from those processes. + ## 0.8.3 - Add opt-in per-Resque-job histograms `resque_job_queue_latency_seconds` and `resque_job_perform_duration_seconds`, labelled by `job_class`. diff --git a/README.md b/README.md index a95b20d..698bc53 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,21 @@ Vanilla Resque jobs (`Resque.enqueue`) carry no enqueue timestamps, so `queue_la Requires resque >= 1.27 workers running in the default fork-per-job mode — the hooks instrument `Resque::Worker#perform_with_fork`. Non-forking workers (`FORK_PER_JOB=false`, or platforms without `fork`) are not instrumented; a warning is logged at worker boot when per-job metrics are enabled but cannot record. +## Protorabbit + +In your protorabbit worker boot code, do: + +```ruby +require 'bigcommerce/prometheus' +Bigcommerce::Prometheus::Instrumentors::Protorabbit.new.start +``` + +Note: protorabbit has no built-in tracer/middleware hook (unlike Hutch), so this instrumentor +only starts the embedded exporter server and registers the generic ActiveRecord collector plus +any custom collectors you configure via `protorabbit_collectors`/`protorabbit_type_collectors`. +This unblocks any metrics already being pushed from within message processing (e.g. gRPC client +interceptor metrics) that were previously refused because no exporter server was listening. + ## Configuration After requiring the main file, you can further configure with: @@ -177,6 +192,8 @@ if we want these collectors to run on our web, resque, and hutch processes, we'l c.resque_type_collectors = [AppTypeCollector.new] c.hutch_collectors = [AppCollector] c.hutch_type_collectors = [AppTypeCollector.new] + c.protorabbit_collectors = [AppCollector] + c.protorabbit_type_collectors = [AppTypeCollector.new] end ``` diff --git a/lib/bigcommerce/prometheus.rb b/lib/bigcommerce/prometheus.rb index e10d2d9..d88266e 100644 --- a/lib/bigcommerce/prometheus.rb +++ b/lib/bigcommerce/prometheus.rb @@ -42,6 +42,7 @@ require_relative 'prometheus/instrumentors/web' require_relative 'prometheus/instrumentors/hutch' require_relative 'prometheus/instrumentors/resque' +require_relative 'prometheus/instrumentors/protorabbit' require_relative 'prometheus/integrations/railtie' if defined?(Rails) require_relative 'prometheus/integrations/puma' require_relative 'prometheus/integrations/resque' diff --git a/lib/bigcommerce/prometheus/configuration.rb b/lib/bigcommerce/prometheus/configuration.rb index f3366ee..e60f188 100644 --- a/lib/bigcommerce/prometheus/configuration.rb +++ b/lib/bigcommerce/prometheus/configuration.rb @@ -49,6 +49,8 @@ module Configuration collector_collection_frequency: ENV.fetch('PROMETHEUS_DEFAULT_COLLECTOR_COLLECTION_FREQUENCY_SEC', 15).to_i, hutch_collectors: [], hutch_type_collectors: [], + protorabbit_collectors: [], + protorabbit_type_collectors: [], resque_collectors: [], resque_type_collectors: [], web_collectors: [], diff --git a/lib/bigcommerce/prometheus/instrumentors/protorabbit.rb b/lib/bigcommerce/prometheus/instrumentors/protorabbit.rb new file mode 100644 index 0000000..a9a158f --- /dev/null +++ b/lib/bigcommerce/prometheus/instrumentors/protorabbit.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# Copyright (c) 2019-present, BigCommerce Pty. Ltd. All rights reserved +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the +# Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +module Bigcommerce + module Prometheus + module Instrumentors + ## + # Instrumentors for protorabbit process + # + # protorabbit has no built-in tracer/middleware hook (unlike Hutch), so this instrumentor only + # starts the embedded exporter server and registers collectors - it takes no `app:` argument. + # + class Protorabbit + include Bigcommerce::Prometheus::Loggable + + def initialize + @enabled = Bigcommerce::Prometheus.enabled + @process_name = Bigcommerce::Prometheus.process_name + @server_port = Bigcommerce::Prometheus.server_port + @server_timeout = Bigcommerce::Prometheus.server_timeout + @server_prefix = Bigcommerce::Prometheus.server_prefix + @collectors = Bigcommerce::Prometheus.protorabbit_collectors || [] + @type_collectors = Bigcommerce::Prometheus.protorabbit_type_collectors || [] + end + + ## + # Start the protorabbit instrumentor + # + def start + unless @enabled + logger.debug "[bigcommerce-prometheus][#{@process_name}] Prometheus disabled, skipping protorabbit start..." + return + end + + server.add_type_collector(PrometheusExporter::Server::ActiveRecordCollector.new) + ::Bigcommerce::Prometheus::Integrations::ActiveRecordSql.register_type_collector(server, process_name: @process_name) + @type_collectors.each do |tc| + server.add_type_collector(tc) + end + server.start + ::Bigcommerce::Prometheus::Integrations::ActiveRecordSql.start_safe(client: Bigcommerce::Prometheus.client, process_name: @process_name) + logger.info "[bigcommerce-prometheus][#{@process_name}] Setting up protorabbit prometheus collectors" + @collectors.each(&:start) + rescue StandardError => e + logger.error "[bigcommerce-prometheus][#{@process_name}] Failed to start protorabbit instrumentation - #{e.message} - #{e.backtrace[0..4].join("\n")}" + end + + private + + def server + @server ||= ::Bigcommerce::Prometheus::Server.new( + port: @server_port, + timeout: @server_timeout, + prefix: @server_prefix + ) + end + end + end + end +end diff --git a/lib/bigcommerce/prometheus/version.rb b/lib/bigcommerce/prometheus/version.rb index 6ae1b43..9ac36ce 100644 --- a/lib/bigcommerce/prometheus/version.rb +++ b/lib/bigcommerce/prometheus/version.rb @@ -17,6 +17,6 @@ # module Bigcommerce module Prometheus - VERSION = '0.8.3' + VERSION = '0.9.0' end end diff --git a/spec/bigcommerce/prometheus/instrumentors/protorabbit_spec.rb b/spec/bigcommerce/prometheus/instrumentors/protorabbit_spec.rb new file mode 100644 index 0000000..10c8417 --- /dev/null +++ b/spec/bigcommerce/prometheus/instrumentors/protorabbit_spec.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +# Copyright (c) 2019-present, BigCommerce Pty. Ltd. All rights reserved +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the +# Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +require 'spec_helper' + +describe Bigcommerce::Prometheus::Instrumentors::Protorabbit do + subject(:instrumentor) { described_class.new } + + let(:registered_type_collectors) { [] } + let(:server) do + instance_double( + Bigcommerce::Prometheus::Server, + add_type_collector: nil, + start: nil + ) + end + + before do + allow(Bigcommerce::Prometheus::Server).to receive(:new).and_return(server) + allow(server).to receive(:add_type_collector) { |tc| registered_type_collectors << tc } + end + + describe '#start' do + context 'when prometheus is disabled' do + before { allow(Bigcommerce::Prometheus).to receive(:enabled).and_return(false) } + + it 'does not start the server' do + instrumentor.start + + expect(server).not_to have_received(:start) + end + end + + context 'when prometheus is enabled' do + let(:collector) { class_double(Bigcommerce::Prometheus::Collectors::Base, start: nil) } + let(:type_collector) { instance_double(Bigcommerce::Prometheus::TypeCollectors::Base) } + + before do + allow(Bigcommerce::Prometheus).to receive_messages( + enabled: true, + protorabbit_collectors: [collector], + protorabbit_type_collectors: [type_collector] + ) + + instrumentor.start + end + + it 'registers the ActiveRecordCollector' do + expect(server).to have_received(:add_type_collector).with(instance_of(PrometheusExporter::Server::ActiveRecordCollector)) + end + + it 'registers the ActiveRecord SQL type collector' do + expect(registered_type_collectors).to include(instance_of(Bigcommerce::Prometheus::TypeCollectors::ActiveRecordSql)) + end + + # Consumers rely on being able to override a built-in type collector by configuring one with + # the same type, which only works if theirs is registered last + it 'registers the configured type collector last' do + expect(registered_type_collectors.last).to eq type_collector + end + + it 'starts the server' do + expect(server).to have_received(:start) + end + + it 'starts the configured collector' do + expect(collector).to have_received(:start) + end + end + + context 'when the server raises' do + before do + allow(Bigcommerce::Prometheus).to receive(:enabled).and_return(true) + allow(server).to receive(:start).and_raise(StandardError, 'boom') + end + + it 'rescues and logs the error instead of raising' do + expect { instrumentor.start }.not_to raise_error + end + end + end +end