Propose opt-in Freno transport circuit breaking

.github/workflows/ruby.yml

@@ -8,29 +8,29 @@
     - main
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
         ruby-version:
         - "3.0"
         - "3.1"
         - "3.2"
         - "3.3"
         - head
         gemfile-path:
         - Gemfile
         - gemfiles/faraday_0.gemfile
         - gemfiles/faraday_1.gemfile
         - gemfiles/faraday_2.gemfile
     env:
       BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile-path }}
     steps:
     - uses: actions/checkout@v4
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true
     - name: Run tests
       run: bin/test

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Opt-in transport circuit breaker integration for Freno client requests
+
 ## [0.9.0] - 2024-04-18
 
 ### Added

README.md

@@ -122,6 +122,24 @@ freno.replication_delay(app: :my_app, store_name: :my_cluster)
 # => 0.125
 ```
 
+### Protecting Freno requests during endpoint outages
+
+Transport circuit breaking is opt-in. Inject a breaker that responds to `allow_request?`, `success`, and `failure` when constructing the shared client. Enable it after configuring all other Faraday middleware but before the connection's first request. Faraday locks its middleware stack after that request, and middleware added after the breaker could run inside its response observer:
+
+```ruby
+freno = Freno::Client.new(faraday) do |client|
+  client.default_app = :my_app
+  client.default_store_name = :my_cluster
+  client.enable_transport_circuit_breaker(circuit_breaker: MyApp.freno_transport_breaker)
+end
+```
+
+This breaker classifies endpoint availability for direct client calls and throttler checks. Transport errors, timeouts, HTTP 408, and HTTP 5xx responses are failures; other HTTP responses prove that Freno is reachable and keep their existing result semantics. Use one shared `Freno::Client` per endpoint. The adapter serializes access to its injected breaker, but callers that reuse a non-thread-safe breaker across multiple clients are responsible for synchronizing it.
+
+The transport breaker is separate from the [throttler circuit breaker](#adding-resiliency), which governs the complete logical throttle and wait operation. They are not composed automatically. Use separate breaker instances and policies, or enable only one seam. The same stateful breaker must not be supplied to both because duplicate `allow_request?` calls can consume a half-open probe before the HTTP request.
+
+When the transport breaker rejects a request, the base client raises `Freno::Client::TransportCircuitBreaker::CircuitOpen` by default. With `raise_on_timeout: false`, it instead returns a failed `:request_timeout` result without making a network request. A fail-open wrapper such as `Freno::FailOpenClient` can convert either outcome to its allowed fallback.
+
 #### Cross-cutting concerns with decorators
 
 Decorators can be used augment the client with custom features.

lib/freno/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "freno/client/version"
+require "freno/client/transport_circuit_breaker"
 require "freno/client/requests/check_read"
 require "freno/client/requests/check"
 require "freno/client/requests/replication_delay"
@@ -15,7 +16,7 @@ class DecorationError < ArgumentError; end
       replication_delay: Requests::ReplicationDelay
     }.freeze
 
-    attr_reader   :faraday, :decorators, :decorated_requests
+    attr_reader   :faraday, :decorators, :decorated_requests, :transport_circuit_breaker
     attr_accessor :default_app, :default_store_name, :default_store_type, :options
 
     # Creates a new instance of the client, that uses faraday to make http calls.
@@ -220,9 +221,23 @@ def decorate(request_or_all, with:)
       end
     end
 
+    # Enables endpoint availability protection for every request made by this
+    # client. Use one client per endpoint; sharing a non-thread-safe breaker
+    # across clients requires synchronization by the caller. Enable this after
+    # configuring all other Faraday middleware and before the first request.
+    def enable_transport_circuit_breaker(circuit_breaker:)
+      raise ArgumentError, "transport circuit breaker is already enabled" if transport_circuit_breaker
+
+      @transport_circuit_breaker = TransportCircuitBreaker.new(
+        circuit_breaker: circuit_breaker,
+        faraday: faraday
+      )
+    end
+
     private
 
     def perform(request, **kwargs)
+      kwargs[:transport_circuit_breaker] = transport_circuit_breaker if transport_circuit_breaker
       decorated(request).perform(faraday: faraday, **kwargs)
     end
 
@@ -233,7 +248,7 @@ def decorated(request)
         outermost = to_decorate[0]
         current = outermost
 
-        (to_decorate[1..]).each do |decorator|
+        to_decorate[1..].each do |decorator|
           current.request = decorator
           current = current.request
         end

lib/freno/client/errors.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Freno
-  Error = Class.new(StandardError)
+  class Error < StandardError; end
 end

lib/freno/client/preconditions.rb

@@ -5,7 +5,7 @@ class Client
     module Preconditions
       module_function
 
-      PreconditionNotMet = Class.new(ArgumentError)
+      class PreconditionNotMet < ArgumentError; end
 
       class Checker
         attr_reader :errors

lib/freno/client/request.rb

@@ -3,13 +3,14 @@
 require_relative "preconditions"
 require_relative "result"
 require_relative "errors"
+require_relative "transport_circuit_breaker"
 
 module Freno
   class Client
     class Request
       include Freno::Client::Preconditions
 
-      attr_reader :faraday, :args, :options, :raise_on_timeout
+      attr_reader :faraday, :args, :options, :raise_on_timeout, :transport_circuit_breaker
 
       def self.perform(**kwargs)
         new(**kwargs).perform
@@ -19,14 +20,25 @@ def initialize(**kwargs)
         @args    = kwargs
         @faraday = kwargs.delete(:faraday) || nil
         @options = kwargs.delete(:options) || {}
+        @transport_circuit_breaker = kwargs.delete(:transport_circuit_breaker)
 
         @raise_on_timeout = options.fetch(:raise_on_timeout, true)
         @verb = options.fetch(:verb, :head)
       end
 
       def perform
-        response = request(verb, path, params)
+        response =
+          if transport_circuit_breaker
+            transport_circuit_breaker.call { request(verb, path, params) }
+          else
+            request(verb, path, params)
+          end
         process_response(response)
+      rescue TransportCircuitBreaker::CircuitOpen
+        # Fail-open wrappers may intentionally rescue the raised Freno::Error.
+        raise if raise_on_timeout
+
+        Result.from_meaning(:request_timeout)
       rescue Faraday::TimeoutError => error
         raise Freno::Error, error if raise_on_timeout
 

lib/freno/client/transport_circuit_breaker.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require "faraday"
+require_relative "errors"
+
+module Freno
+  class Client
+    class TransportCircuitBreaker
+      class CircuitOpen < Freno::Error; end
+
+      METHODS = %i[allow_request? success failure].freeze
+      CONTEXT_KEY = :freno_transport_circuit_breaker_context
+
+      class RequestMarker < Faraday::Middleware
+        def initialize(app, transport_circuit_breaker)
+          super(app)
+          @transport_circuit_breaker = transport_circuit_breaker
+        end
+
+        def call(env)
+          @transport_circuit_breaker.bind_request(env)
+          @app.call(env)
+        end
+      end
+      private_constant :RequestMarker
+
+      class ResponseObserver < Faraday::Middleware
+        def initialize(app, transport_circuit_breaker)
+          super(app)
+          @transport_circuit_breaker = transport_circuit_breaker
+        end
+
+        def call(env)
+          context = @transport_circuit_breaker.begin_attempt(env)
+          @app.call(env).on_complete do |response_env|
+            on_complete(context, response_env)
+          end
+        end
+
+        def on_complete(context, env)
+          @transport_circuit_breaker.observe(context, env.status)
+        end
+      end
+      private_constant :ResponseObserver
+
+      def initialize(circuit_breaker:, faraday:)
+        missing_methods = METHODS.reject { |method| circuit_breaker.respond_to?(method) }
+        unless missing_methods.empty?
+          raise ArgumentError, "circuit breaker must respond to #{missing_methods.join(', ')}"
+        end
+
+        @circuit_breaker = circuit_breaker
+        @mutex = Mutex.new
+        install_middleware(faraday)
+      end
+
+      def call
+        raise CircuitOpen, "Freno transport circuit is open" unless allow_request?
+
+        contexts = (Thread.current[CONTEXT_KEY] ||= {})
+        previous_context = contexts[self]
+        context = {
+          outcome_claimed: false,
+          request_env: nil,
+          status: nil,
+          status_observed: false
+        }
+        contexts[self] = context
+
+        begin
+          response = yield
+          status = context[:status_observed] ? context[:status] : response.status
+          claim_outcome(context, status)
+          response
+        rescue StandardError => error
+          status = context[:status_observed] ? context[:status] : status_from_error(error)
+          claim_outcome(context, status)
+          raise
+        ensure
+          if previous_context
+            contexts[self] = previous_context
+          else
+            contexts.delete(self)
+            Thread.current[CONTEXT_KEY] = nil if contexts.empty?
+          end
+        end
+      end
+
+      def bind_request(env)
+        context = Thread.current[CONTEXT_KEY]&.fetch(self, nil)
+        context[:request_env] ||= env if context
+      end
+
+      def begin_attempt(env)
+        context = Thread.current[CONTEXT_KEY]&.fetch(self, nil)
+        return unless context
+        return unless context.fetch(:request_env).equal?(env)
+
+        context[:status] = nil
+        context[:status_observed] = false
+        context
+      end
+
+      def observe(context, status)
+        return unless context
+
+        context[:status] = status
+        context[:status_observed] = true
+      end
+
+      private
+
+      def claim_outcome(context, status)
+        return if context[:outcome_claimed]
+
+        context[:outcome_claimed] = true
+        record(failure_status?(status) ? :failure : :success)
+      end
+
+      def install_middleware(faraday)
+        builder = faraday.builder
+        if builder.locked?
+          raise ArgumentError, "transport circuit breaker must be enabled before the first Faraday request"
+        end
+
+        builder.insert(0, RequestMarker, self)
+        handlers = builder.handlers
+        index = handlers.length
+        last_handler = handlers.last
+        index -= 1 if last_handler && last_handler.klass <= Faraday::Adapter
+
+        # Response middleware unwinds in reverse order, so the observer goes
+        # immediately inside existing validators and before the adapter.
+        builder.insert(index, ResponseObserver, self)
+      end
+
+      def allow_request?
+        @mutex.synchronize { @circuit_breaker.allow_request? }
+      end
+
+      def record(outcome)
+        @mutex.synchronize { @circuit_breaker.public_send(outcome) }
+      end
+
+      def failure_status?(status)
+        !status.is_a?(Integer) ||
+          !(100..599).cover?(status) ||
+          status == 408 ||
+          (500..599).cover?(status)
+      end
+
+      def status_from_error(error)
+        cause = error
+
+        while cause
+          return cause.response_status if cause.respond_to?(:response_status) && cause.response_status
+
+          if cause.respond_to?(:response)
+            response = cause.response
+            return response[:status] if response.respond_to?(:[]) && response[:status]
+          end
+
+          cause = cause.cause
+        end
+
+        nil
+      end
+    end
+  end
+end