-
# frozen_string_literal: true
-
-
30
require "httpx/version"
-
-
# Top-Level Namespace
-
#
-
30
module HTTPX
-
30
EMPTY = [].freeze
-
30
EMPTY_HASH = {}.freeze
-
-
# All plugins should be stored under this module/namespace. Can register and load
-
# plugins.
-
#
-
30
module Plugins
-
30
@plugins = {}
-
30
@plugins_mutex = Thread::Mutex.new
-
-
# Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
-
# it from the load path under "httpx/plugins/" directory.
-
#
-
30
def self.load_plugin(name)
-
10933
h = @plugins
-
10933
m = @plugins_mutex
-
21866
unless (plugin = m.synchronize { h[name] })
-
329
require "httpx/plugins/#{name}"
-
658
raise "Plugin #{name} hasn't been registered" unless (plugin = m.synchronize { h[name] })
-
end
-
10933
plugin
-
end
-
-
# Registers a plugin (+mod+) in the central store indexed by +name+.
-
#
-
30
def self.register_plugin(name, mod)
-
909
h = @plugins
-
909
m = @plugins_mutex
-
1784
m.synchronize { h[name] = mod }
-
end
-
end
-
end
-
-
30
require "httpx/extensions"
-
-
30
require "httpx/errors"
-
30
require "httpx/utils"
-
30
require "httpx/punycode"
-
30
require "httpx/domain_name"
-
30
require "httpx/altsvc"
-
30
require "httpx/callbacks"
-
30
require "httpx/loggable"
-
30
require "httpx/transcoder"
-
30
require "httpx/timers"
-
30
require "httpx/pool"
-
30
require "httpx/headers"
-
30
require "httpx/request"
-
30
require "httpx/response"
-
30
require "httpx/options"
-
30
require "httpx/chainable"
-
-
30
require "httpx/session"
-
30
require "httpx/session_extensions"
-
-
# load integrations when possible
-
-
30
require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog::Tracing)
-
30
require "httpx/adapters/sentry" if defined?(Sentry)
-
30
require "httpx/adapters/webmock" if defined?(WebMock)
-
# frozen_string_literal: true
-
-
7
require "datadog/tracing/contrib/integration"
-
7
require "datadog/tracing/contrib/configuration/settings"
-
7
require "datadog/tracing/contrib/patcher"
-
-
7
module Datadog::Tracing
-
7
module Contrib
-
7
module HTTPX
-
7
DATADOG_VERSION = defined?(::DDTrace) ? ::DDTrace::VERSION : ::Datadog::VERSION
-
-
7
METADATA_MODULE = Datadog::Tracing::Metadata
-
-
7
TYPE_OUTBOUND = Datadog::Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
-
-
7
TAG_BASE_SERVICE = if Gem::Version.new(DATADOG_VERSION::STRING) < Gem::Version.new("1.15.0")
-
"_dd.base_service"
-
else
-
7
Datadog::Tracing::Contrib::Ext::Metadata::TAG_BASE_SERVICE
-
end
-
7
TAG_PEER_HOSTNAME = Datadog::Tracing::Metadata::Ext::TAG_PEER_HOSTNAME
-
7
TAG_PEER_SERVICE = Datadog::Tracing::Metadata::Ext::TAG_PEER_SERVICE
-
-
7
TAG_KIND = Datadog::Tracing::Metadata::Ext::TAG_KIND
-
7
TAG_CLIENT = Datadog::Tracing::Metadata::Ext::SpanKind::TAG_CLIENT
-
7
TAG_COMPONENT = Datadog::Tracing::Metadata::Ext::TAG_COMPONENT
-
7
TAG_OPERATION = Datadog::Tracing::Metadata::Ext::TAG_OPERATION
-
7
TAG_URL = Datadog::Tracing::Metadata::Ext::HTTP::TAG_URL
-
7
TAG_METHOD = Datadog::Tracing::Metadata::Ext::HTTP::TAG_METHOD
-
7
TAG_TARGET_HOST = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_HOST
-
7
TAG_TARGET_PORT = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_PORT
-
-
7
TAG_STATUS_CODE = Datadog::Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE
-
-
# HTTPX Datadog Plugin
-
#
-
# Enables tracing for httpx requests.
-
#
-
# A span will be created for each request transaction; the span is created lazily only when
-
# buffering a request, and it is fed the start time stored inside the tracer object.
-
#
-
7
module Plugin
-
7
module RequestTracer
-
7
extend Contrib::HttpAnnotationHelper
-
-
7
module_function
-
-
7
SPAN_REQUEST = "httpx.request"
-
-
7
def enabled?(request)
-
275
configuration(request).enabled
-
end
-
-
7
def start(request)
-
112
request.datadog_span = initialize_span(request, request.init_time)
-
end
-
-
7
def reset(request)
-
28
request.datadog_span = nil
-
end
-
-
7
def finish(request, response)
-
112
request.datadog_span ||= initialize_span(request, request.init_time) if request.init_time
-
-
112
finish_span(response, request.datadog_span)
-
end
-
-
7
def finish_span(response, span)
-
112
if response.is_a?(::HTTPX::ErrorResponse)
-
7
span.set_error(response.error)
-
else
-
105
span.set_tag(TAG_STATUS_CODE, response.status.to_s)
-
-
105
span.set_error(::HTTPX::HTTPError.new(response)) if response.status.between?(400, 599)
-
-
span.set_tags(
-
Datadog.configuration.tracing.header_tags.response_tags(response.headers.to_h)
-
105
) if Datadog.configuration.tracing.respond_to?(:header_tags)
-
end
-
-
112
span.finish
-
end
-
-
# return a span initialized with the +@request+ state.
-
7
def initialize_span(request, start_time)
-
119
verb = request.verb
-
119
uri = request.uri
-
-
119
config = configuration(request)
-
-
119
span = create_span(request, config, start_time)
-
-
119
span.resource = verb
-
-
# Tag original global service name if not used
-
119
span.set_tag(TAG_BASE_SERVICE, Datadog.configuration.service) if span.service != Datadog.configuration.service
-
-
119
span.set_tag(TAG_KIND, TAG_CLIENT)
-
-
119
span.set_tag(TAG_COMPONENT, "httpx")
-
119
span.set_tag(TAG_OPERATION, "request")
-
-
119
span.set_tag(TAG_URL, request.path)
-
119
span.set_tag(TAG_METHOD, verb)
-
-
119
span.set_tag(TAG_TARGET_HOST, uri.host)
-
119
span.set_tag(TAG_TARGET_PORT, uri.port)
-
-
119
span.set_tag(TAG_PEER_HOSTNAME, uri.host)
-
-
# Tag as an external peer service
-
119
if (peer_service = config[:peer_service])
-
span.set_tag(TAG_PEER_SERVICE, peer_service)
-
end
-
-
119
if config[:distributed_tracing]
-
112
propagate_trace_http(
-
Datadog::Tracing.active_trace,
-
request.headers
-
)
-
end
-
-
# Set analytics sample rate
-
119
if Contrib::Analytics.enabled?(config[:analytics_enabled])
-
14
Contrib::Analytics.set_sample_rate(span, config[:analytics_sample_rate])
-
end
-
-
span.set_tags(
-
Datadog.configuration.tracing.header_tags.request_tags(request.headers.to_h)
-
119
) if Datadog.configuration.tracing.respond_to?(:header_tags)
-
-
119
span
-
rescue StandardError => e
-
Datadog.logger.error("error preparing span for http request: #{e}")
-
Datadog.logger.error(e.backtrace)
-
end
-
-
7
def configuration(request)
-
394
Datadog.configuration.tracing[:httpx, request.uri.host]
-
end
-
-
7
if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
-
4
def propagate_trace_http(trace, headers)
-
64
Datadog::Tracing::Contrib::HTTP.inject(trace, headers)
-
end
-
-
4
def create_span(request, configuration, start_time)
-
68
Datadog::Tracing.trace(
-
SPAN_REQUEST,
-
service: service_name(request.uri.host, configuration),
-
type: TYPE_OUTBOUND,
-
start_time: start_time
-
)
-
end
-
else
-
3
def propagate_trace_http(trace, headers)
-
48
Datadog::Tracing::Propagation::HTTP.inject!(trace.to_digest, headers)
-
end
-
-
3
def create_span(request, configuration, start_time)
-
51
Datadog::Tracing.trace(
-
SPAN_REQUEST,
-
service: service_name(request.uri.host, configuration),
-
span_type: TYPE_OUTBOUND,
-
start_time: start_time
-
)
-
end
-
end
-
end
-
-
7
class << self
-
7
def load_dependencies(klass)
-
7
klass.plugin(:tracing)
-
end
-
-
7
def extra_options(options)
-
7
options.merge(tracer: RequestTracer)
-
end
-
end
-
-
7
module RequestMethods
-
7
attr_accessor :datadog_span
-
end
-
end
-
-
7
module Configuration
-
# Default settings for httpx
-
#
-
7
class Settings < Datadog::Tracing::Contrib::Configuration::Settings
-
7
DEFAULT_ERROR_HANDLER = lambda do |response|
-
Datadog::Ext::HTTP::ERROR_RANGE.cover?(response.status)
-
end
-
-
7
option :service_name, default: "httpx"
-
7
option :distributed_tracing, default: true
-
7
option :split_by_domain, default: false
-
-
7
if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
-
7
option :enabled do |o|
-
7
o.type :bool
-
7
o.env "DD_TRACE_HTTPX_ENABLED"
-
7
o.default true
-
end
-
-
7
option :analytics_enabled do |o|
-
7
o.type :bool
-
7
o.env "DD_TRACE_HTTPX_ANALYTICS_ENABLED"
-
7
o.default false
-
end
-
-
7
option :analytics_sample_rate do |o|
-
7
o.type :float
-
7
o.env "DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE"
-
7
o.default 1.0
-
end
-
-
7
option :peer_service do |o|
-
7
o.type :string, nilable: true
-
7
o.env "DD_TRACE_HTTPX_PEER_SERVICE"
-
end
-
else
-
option :enabled do |o|
-
o.default { env_to_bool("DD_TRACE_HTTPX_ENABLED", true) }
-
o.lazy
-
end
-
-
option :analytics_enabled do |o|
-
o.default { env_to_bool(%w[DD_TRACE_HTTPX_ANALYTICS_ENABLED DD_HTTPX_ANALYTICS_ENABLED], false) }
-
o.lazy
-
end
-
-
option :analytics_sample_rate do |o|
-
o.default { env_to_float(%w[DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE DD_HTTPX_ANALYTICS_SAMPLE_RATE], 1.0) }
-
o.lazy
-
end
-
-
option :peer_service do |o|
-
o.default { env_to_string("DD_TRACE_HTTPX_PEER_SERVICE", nil) }
-
o.lazy
-
end
-
end
-
-
7
if defined?(Datadog::Tracing::Contrib::SpanAttributeSchema)
-
7
option :service_name do |o|
-
7
o.default do
-
81
Datadog::Tracing::Contrib::SpanAttributeSchema.fetch_service_name(
-
"DD_TRACE_HTTPX_SERVICE_NAME",
-
"httpx"
-
)
-
end
-
7
o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
-
end
-
else
-
option :service_name do |o|
-
o.default do
-
ENV.fetch("DD_TRACE_HTTPX_SERVICE_NAME", "httpx")
-
end
-
o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
-
end
-
end
-
-
7
option :distributed_tracing, default: true
-
-
7
if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.15.0")
-
7
option :error_handler do |o|
-
7
o.type :proc
-
7
o.default_proc(&DEFAULT_ERROR_HANDLER)
-
end
-
elsif Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
-
option :error_handler do |o|
-
o.type :proc
-
o.experimental_default_proc(&DEFAULT_ERROR_HANDLER)
-
end
-
else
-
option :error_handler, default: DEFAULT_ERROR_HANDLER
-
end
-
end
-
end
-
-
# Patcher enables patching of 'httpx' with datadog components.
-
#
-
7
module Patcher
-
7
include Datadog::Tracing::Contrib::Patcher
-
-
7
module_function
-
-
7
def target_version
-
14
Integration.version
-
end
-
-
# loads a session instannce with the datadog plugin, and replaces the
-
# base HTTPX::Session with the patched session class.
-
7
def patch
-
7
datadog_session = ::HTTPX.plugin(Plugin)
-
-
7
::HTTPX.send(:remove_const, :Session)
-
7
::HTTPX.send(:const_set, :Session, datadog_session.class)
-
end
-
end
-
-
# Datadog Integration for HTTPX.
-
#
-
7
class Integration
-
7
include Contrib::Integration
-
-
7
MINIMUM_VERSION = Gem::Version.new("0.10.2")
-
-
7
register_as :httpx
-
-
7
def self.version
-
287
Gem.loaded_specs["httpx"] && Gem.loaded_specs["httpx"].version
-
end
-
-
7
def self.loaded?
-
91
defined?(::HTTPX::Request)
-
end
-
-
7
def self.compatible?
-
91
super && version >= MINIMUM_VERSION
-
end
-
-
7
def new_configuration
-
182
Configuration::Settings.new
-
end
-
-
7
def patcher
-
182
Patcher
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
17
require "delegate"
-
17
require "httpx"
-
17
require "faraday"
-
-
17
module Faraday
-
17
class Adapter
-
17
class HTTPX < Faraday::Adapter
-
17
def initialize(app = nil, opts = {}, &block)
-
334
@connection = @bind = nil
-
334
super
-
end
-
-
17
module RequestMixin
-
17
def build_connection(env)
-
325
return @connection if @connection
-
-
325
@connection = ::HTTPX.plugin(:persistent).plugin(ReasonPlugin)
-
325
@connection = @connection.with(@connection_options) unless @connection_options.empty?
-
325
connection_opts = options_from_env(env)
-
-
325
if (bind = env.request.bind)
-
9
@bind = TCPSocket.new(bind[:host], bind[:port])
-
8
connection_opts[:io] = @bind
-
end
-
325
@connection = @connection.with(connection_opts)
-
-
325
if (proxy = env.request.proxy)
-
9
proxy_options = { uri: proxy.uri }
-
9
proxy_options[:username] = proxy.user if proxy.user
-
9
proxy_options[:password] = proxy.password if proxy.password
-
-
9
@connection = @connection.plugin(:proxy).with(proxy: proxy_options)
-
end
-
325
@connection = @connection.plugin(OnDataPlugin) if env.request.stream_response?
-
-
325
@connection = @config_block.call(@connection) || @connection if @config_block
-
325
@connection
-
end
-
-
17
def close
-
333
@connection.close if @connection
-
333
@bind.close if @bind
-
end
-
-
17
private
-
-
17
def connect(env, &blk)
-
325
connection(env, &blk)
-
rescue ::HTTPX::TLSError => e
-
9
raise Faraday::SSLError, e
-
rescue Errno::ECONNABORTED,
-
Errno::ECONNREFUSED,
-
Errno::ECONNRESET,
-
Errno::EHOSTUNREACH,
-
Errno::EINVAL,
-
Errno::ENETUNREACH,
-
Errno::EPIPE,
-
::HTTPX::ConnectionError => e
-
9
raise Faraday::ConnectionFailed, e
-
rescue ::HTTPX::TimeoutError => e
-
27
raise Faraday::TimeoutError, e
-
end
-
-
17
def build_request(env)
-
334
meth = env[:method]
-
-
30
request_options = {
-
303
headers: env.request_headers,
-
body: env.body,
-
**options_from_env(env),
-
}
-
334
[meth.to_s.upcase, env.url, request_options]
-
end
-
-
17
def options_from_env(env)
-
659
timeout_options = {}
-
659
req_opts = env.request
-
659
if (sec = request_timeout(:read, req_opts))
-
32
timeout_options[:read_timeout] = sec
-
end
-
-
659
if (sec = request_timeout(:write, req_opts))
-
16
timeout_options[:write_timeout] = sec
-
end
-
-
659
if (sec = request_timeout(:open, req_opts))
-
16
timeout_options[:connect_timeout] = sec
-
end
-
-
59
{
-
599
ssl: ssl_options_from_env(env),
-
timeout: timeout_options,
-
}
-
end
-
-
17
if defined?(::OpenSSL)
-
17
def ssl_options_from_env(env)
-
659
ssl_options = {}
-
-
659
unless env.ssl.verify.nil?
-
32
ssl_options[:verify_mode] = env.ssl.verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
-
end
-
-
659
ssl_options[:ca_file] = env.ssl.ca_file if env.ssl.ca_file
-
659
ssl_options[:ca_path] = env.ssl.ca_path if env.ssl.ca_path
-
659
ssl_options[:cert_store] = env.ssl.cert_store if env.ssl.cert_store
-
659
ssl_options[:cert] = env.ssl.client_cert if env.ssl.client_cert
-
659
ssl_options[:key] = env.ssl.client_key if env.ssl.client_key
-
659
ssl_options[:ssl_version] = env.ssl.version if env.ssl.version
-
659
ssl_options[:verify_depth] = env.ssl.verify_depth if env.ssl.verify_depth
-
659
ssl_options[:min_version] = env.ssl.min_version if env.ssl.min_version
-
659
ssl_options[:max_version] = env.ssl.max_version if env.ssl.max_version
-
659
ssl_options
-
end
-
else
-
skipped
# :nocov:
-
skipped
def ssl_options_from_env(*)
-
skipped
{}
-
skipped
end
-
skipped
# :nocov:
-
end
-
end
-
-
17
include RequestMixin
-
-
17
module OnDataPlugin
-
17
module RequestMethods
-
17
attr_writer :response_on_data
-
-
17
def response=(response)
-
18
super
-
-
18
return unless @response
-
-
18
return if @response.is_a?(::HTTPX::ErrorResponse)
-
-
18
@response.body.on_data = @response_on_data
-
end
-
end
-
-
17
module ResponseBodyMethods
-
17
attr_writer :on_data
-
-
17
def write(chunk)
-
54
return super unless @on_data
-
-
54
@on_data.call(chunk, chunk.bytesize)
-
end
-
end
-
end
-
-
17
module ReasonPlugin
-
17
def self.load_dependencies(*)
-
325
require "net/http/status"
-
end
-
-
17
module ResponseMethods
-
17
def reason
-
273
Net::HTTP::STATUS_CODES.fetch(@status, "Non-Standard status code")
-
end
-
end
-
end
-
-
17
class ParallelManager
-
17
class ResponseHandler < SimpleDelegator
-
17
attr_reader :env
-
-
17
def initialize(env)
-
36
@env = env
-
36
super
-
end
-
-
17
def on_response(&blk)
-
72
if blk
-
36
@on_response = ->(response) do
-
36
blk.call(response)
-
end
-
36
self
-
else
-
36
@on_response
-
end
-
end
-
-
17
def on_complete(&blk)
-
36
if blk
-
@on_complete = blk
-
self
-
else
-
36
@on_complete
-
end
-
end
-
end
-
-
17
include RequestMixin
-
-
17
def initialize(options)
-
36
@handlers = []
-
36
@connection_options = options
-
end
-
-
17
def enqueue(request)
-
36
handler = ResponseHandler.new(request)
-
36
@handlers << handler
-
36
handler
-
end
-
-
17
def run
-
36
return unless @handlers.last
-
-
27
env = @handlers.last.env
-
-
27
connect(env) do |session|
-
63
requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
-
-
27
if env.request.stream_response?
-
9
requests.each do |request|
-
9
request.response_on_data = env.request.on_data
-
end
-
end
-
-
27
responses = session.request(*requests)
-
27
Array(responses).each_with_index do |response, index|
-
36
handler = @handlers[index]
-
36
handler.on_response.call(response)
-
36
handler.on_complete.call(handler.env) if handler.on_complete
-
end
-
end
-
end
-
-
17
private
-
-
# from Faraday::Adapter#connection
-
17
def connection(env)
-
27
conn = build_connection(env)
-
27
return conn unless block_given?
-
-
27
yield conn
-
end
-
-
# from Faraday::Adapter#request_timeout
-
17
def request_timeout(type, options)
-
189
key = Faraday::Adapter::TIMEOUT_KEYS[type]
-
189
options[key] || options[:timeout]
-
end
-
end
-
-
17
self.supports_parallel = true
-
-
17
class << self
-
17
def setup_parallel_manager(options = {})
-
36
ParallelManager.new(options)
-
end
-
end
-
-
17
def call(env)
-
334
super
-
334
if parallel?(env)
-
36
handler = env[:parallel_manager].enqueue(env)
-
36
handler.on_response do |response|
-
36
if response.is_a?(::HTTPX::Response)
-
27
save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
-
27
response_headers.merge!(response.headers)
-
end
-
else
-
8
env[:error] = response.error
-
9
save_response(env, 0, "", {}, nil)
-
end
-
end
-
32
return handler
-
end
-
-
298
response = connect_and_request(env)
-
246
save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
-
246
response_headers.merge!(response.headers)
-
end
-
246
@app.call(env)
-
end
-
-
17
private
-
-
17
def connect_and_request(env)
-
298
connect(env) do |session|
-
298
request = session.build_request(*build_request(env))
-
-
298
request.response_on_data = env.request.on_data if env.request.stream_response?
-
-
298
response = session.request(request)
-
# do not call #raise_for_status for HTTP 4xx or 5xx, as faraday has a middleware for that.
-
298
response.raise_for_status unless response.is_a?(::HTTPX::Response)
-
246
response
-
end
-
end
-
-
17
def parallel?(env)
-
334
env[:parallel_manager]
-
end
-
end
-
-
17
register_middleware httpx: HTTPX
-
end
-
end
-
# frozen_string_literal: true
-
-
7
require "sentry-ruby"
-
-
7
module HTTPX::Plugins
-
7
module Sentry
-
7
module Tracer
-
7
module_function
-
-
7
def call(request)
-
140
sentry_span = start_sentry_span
-
-
140
return unless sentry_span
-
-
140
set_sentry_trace_header(request, sentry_span)
-
-
140
request.on(:response, &method(:finish_sentry_span).curry(3)[sentry_span, request])
-
end
-
-
7
def start_sentry_span
-
140
return unless ::Sentry.initialized? && (span = ::Sentry.get_current_scope.get_span)
-
140
return if span.sampled == false
-
-
140
span.start_child(op: "httpx.client", start_timestamp: ::Sentry.utc_now.to_f)
-
end
-
-
7
def set_sentry_trace_header(request, sentry_span)
-
140
return unless sentry_span
-
-
140
config = ::Sentry.configuration
-
140
url = request.uri.to_s
-
-
280
return unless config.propagate_traces && config.trace_propagation_targets.any? { |target| url.match?(target) }
-
-
140
trace = sentry_span.to_sentry_trace
-
140
request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
-
end
-
-
7
def finish_sentry_span(span, request, response)
-
146
return unless ::Sentry.initialized?
-
-
146
record_sentry_breadcrumb(request, response)
-
146
record_sentry_span(request, response, span)
-
end
-
-
7
def record_sentry_breadcrumb(req, res)
-
146
return unless ::Sentry.configuration.breadcrumbs_logger.include?(:http_logger)
-
-
146
request_info = extract_request_info(req)
-
-
146
data = if res.is_a?(HTTPX::ErrorResponse)
-
14
{ error: res.error.message, **request_info }
-
else
-
132
{ status: res.status, **request_info }
-
end
-
-
146
crumb = ::Sentry::Breadcrumb.new(
-
level: :info,
-
category: "httpx",
-
type: :info,
-
data: data
-
)
-
146
::Sentry.add_breadcrumb(crumb)
-
end
-
-
7
def record_sentry_span(req, res, sentry_span)
-
146
return unless sentry_span
-
-
146
request_info = extract_request_info(req)
-
146
sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
-
146
if res.is_a?(HTTPX::ErrorResponse)
-
14
sentry_span.set_data(:error, res.error.message)
-
else
-
132
sentry_span.set_data(:status, res.status)
-
end
-
146
sentry_span.set_timestamp(::Sentry.utc_now.to_f)
-
end
-
-
7
def extract_request_info(req)
-
292
uri = req.uri
-
-
result = {
-
292
method: req.verb,
-
}
-
-
292
if ::Sentry.configuration.send_default_pii
-
28
uri += "?#{req.query}" unless req.query.empty?
-
28
result[:body] = req.body.to_s unless req.body.empty? || req.body.unbounded_body?
-
end
-
-
292
result[:url] = uri.to_s
-
-
292
result
-
end
-
end
-
-
7
module RequestMethods
-
7
def __sentry_enable_trace!
-
146
return if @__sentry_enable_trace
-
-
140
Tracer.call(self)
-
140
@__sentry_enable_trace = true
-
end
-
end
-
-
7
module ConnectionMethods
-
7
def send(request)
-
146
request.__sentry_enable_trace!
-
-
146
super
-
end
-
end
-
end
-
end
-
-
7
Sentry.register_patch(:httpx) do
-
35
sentry_session = HTTPX.plugin(HTTPX::Plugins::Sentry)
-
-
35
HTTPX.send(:remove_const, :Session)
-
35
HTTPX.send(:const_set, :Session, sentry_session.class)
-
end
-
# frozen_string_literal: true
-
-
9
module WebMock
-
9
module HttpLibAdapters
-
9
require "net/http/status"
-
9
HTTP_REASONS = Net::HTTP::STATUS_CODES
-
-
#
-
# HTTPX plugin for webmock.
-
#
-
# Requests are "hijacked" at the session, before they're distributed to a connection.
-
#
-
9
module Plugin
-
9
class << self
-
9
def build_webmock_request_signature(request)
-
291
uri = WebMock::Util::URI.heuristic_parse(request.uri)
-
291
uri.query = request.query
-
291
uri.path = uri.normalized_path.gsub("[^:]//", "/")
-
-
291
WebMock::RequestSignature.new(
-
request.verb.downcase.to_sym,
-
uri.to_s,
-
body: request.body.to_s,
-
headers: request.headers.to_h
-
)
-
end
-
-
9
def build_webmock_response(_request, response)
-
7
webmock_response = WebMock::Response.new
-
7
webmock_response.status = [response.status, HTTP_REASONS[response.status]]
-
7
webmock_response.body = response.body.to_s
-
7
webmock_response.headers = response.headers.to_h
-
7
webmock_response
-
end
-
-
9
def build_from_webmock_response(request, webmock_response)
-
256
return build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
-
-
235
return build_error_response(request, webmock_response.exception) if webmock_response.exception
-
-
227
request
-
.options
-
.response_class
-
.new(
-
request,
-
webmock_response.status[0],
-
"2.0",
-
webmock_response.headers
-
).tap(&:mock!)
-
end
-
-
9
def build_error_response(request, exception)
-
29
HTTPX::ErrorResponse.new(request, exception)
-
end
-
end
-
-
9
module InstanceMethods
-
9
private
-
-
9
def do_init_connection(connection, selector)
-
256
super
-
-
256
connection.once(:unmock_connection) do
-
28
next unless connection.current_session == self
-
-
28
unless connection.addresses?
-
# reset Happy Eyeballs, fail early
-
28
connection.sibling = nil
-
-
28
deselect_connection(connection, selector)
-
end
-
28
resolve_connection(connection, selector)
-
end
-
end
-
end
-
-
9
module ResponseMethods
-
9
def initialize(*)
-
255
super
-
255
@mocked = false
-
end
-
-
9
def mock!
-
227
@mocked = true
-
227
@body.mock!
-
end
-
-
9
def mocked?
-
@mocked
-
end
-
end
-
-
9
module ResponseBodyMethods
-
9
def mock!
-
227
@inflaters = nil
-
end
-
end
-
-
9
module ConnectionMethods
-
9
def initialize(*)
-
256
super
-
256
@mocked = true
-
end
-
-
9
def open?
-
296
return true if @mocked
-
-
40
super
-
end
-
-
9
def interests
-
301
return if @mocked
-
-
287
super
-
end
-
-
9
def terminate
-
227
force_reset
-
end
-
-
9
def send(request)
-
291
request_signature = Plugin.build_webmock_request_signature(request)
-
291
WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
-
-
291
if (mock_response = WebMock::StubRegistry.instance.response_for_request(request_signature))
-
256
response = Plugin.build_from_webmock_response(request, mock_response)
-
256
WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, request_signature, mock_response)
-
256
log { "mocking #{request.uri} with #{mock_response.inspect}" }
-
256
request.transition(:headers)
-
256
request.transition(:body)
-
256
request.transition(:trailers)
-
256
request.transition(:done)
-
256
response.finish!
-
256
request.response = response
-
256
request.emit_response(response)
-
256
request_signature.headers = request.headers.to_h
-
-
256
response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)
-
35
elsif WebMock.net_connect_allowed?(request_signature.uri)
-
28
if WebMock::CallbackRegistry.any_callbacks?
-
7
request.on(:response) do |resp|
-
7
unless resp.is_a?(HTTPX::ErrorResponse)
-
7
webmock_response = Plugin.build_webmock_response(request, resp)
-
7
WebMock::CallbackRegistry.invoke_callbacks(
-
{ lib: :httpx, real_request: true }, request_signature,
-
webmock_response
-
)
-
end
-
end
-
end
-
28
@mocked = false
-
28
emit(:unmock_connection, self)
-
28
super
-
else
-
7
raise WebMock::NetConnectNotAllowedError, request_signature
-
end
-
end
-
end
-
end
-
-
9
class HttpxAdapter < HttpLibAdapter
-
9
adapter_for :httpx
-
-
9
class << self
-
9
def enable!
-
503
@original_session ||= HTTPX::Session
-
-
503
webmock_session = HTTPX.plugin(Plugin)
-
-
503
HTTPX.send(:remove_const, :Session)
-
503
HTTPX.send(:const_set, :Session, webmock_session.class)
-
end
-
-
9
def disable!
-
503
return unless @original_session
-
-
494
HTTPX.send(:remove_const, :Session)
-
494
HTTPX.send(:const_set, :Session, @original_session)
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "strscan"
-
-
30
module HTTPX
-
30
module AltSvc
-
# makes connections able to accept requests destined to primary service.
-
30
module ConnectionMixin
-
30
using URIExtensions
-
-
30
H2_ALTSVC_SCHEMES = %w[https h2].freeze
-
-
30
ALTSVC_IGNORE_IVARS = %i[@ssl].freeze
-
-
30
def send(request)
-
9
request.headers["alt-used"] = @origin.authority if @parser && !@write_buffer.full? && match_altsvcs?(request.uri)
-
-
9
super
-
end
-
-
30
def match?(uri, options)
-
9
return false if !used? && (@state == :closing || @state == :closed)
-
-
9
match_altsvcs?(uri) && match_altsvc_options?(uri, options)
-
end
-
-
30
private
-
-
# checks if this is connection is an alternative service of
-
# +uri+
-
30
def match_altsvcs?(uri)
-
51
@origins.any? { |origin| altsvc_match?(uri, origin) } ||
-
AltSvc.cached_altsvc(@origin).any? do |altsvc|
-
origin = altsvc["origin"]
-
altsvc_match?(origin, uri.origin)
-
end
-
end
-
-
30
def match_altsvc_options?(uri, options)
-
9
return @options.connection_options_match?(options) unless @options.ssl.all? do |k, v|
-
9
v == (k == :hostname ? uri.host : options.ssl[k])
-
end
-
-
9
@options.connection_options_match?(options, ALTSVC_IGNORE_IVARS)
-
end
-
-
30
def altsvc_match?(uri, other_uri)
-
34
other_uri = URI(other_uri) #: http_uri
-
-
34
uri.origin == other_uri.origin || begin
-
15
case uri.scheme
-
when "h2"
-
H2_ALTSVC_SCHEMES.include?(other_uri.scheme) &&
-
uri.host == other_uri.host &&
-
uri.port == other_uri.port
-
else
-
17
false
-
end
-
end
-
end
-
end
-
-
30
@altsvc_mutex = Thread::Mutex.new
-
54
@altsvcs = Hash.new { |h, k| h[k] = [] }
-
-
30
module_function
-
-
30
def cached_altsvc(origin)
-
45
now = Utils.now
-
45
@altsvc_mutex.synchronize do
-
45
lookup(origin, now)
-
end
-
end
-
-
30
def cached_altsvc_set(origin, entry)
-
27
now = Utils.now
-
27
@altsvc_mutex.synchronize do
-
27
return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
-
-
27
entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
-
27
@altsvcs[origin] << entry
-
27
entry
-
end
-
end
-
-
30
def lookup(origin, ttl)
-
45
return [] unless @altsvcs.key?(origin)
-
-
32
@altsvcs[origin] = @altsvcs[origin].select do |entry|
-
27
!entry.key?("TTL") || entry["TTL"] > ttl
-
end
-
54
@altsvcs[origin].reject { |entry| entry["noop"] }
-
end
-
-
30
def emit(request, response)
-
9849
return unless response.respond_to?(:headers)
-
# Alt-Svc
-
9805
return unless response.headers.key?("alt-svc")
-
-
120
origin = request.origin
-
120
host = request.uri.host
-
-
120
altsvc = response.headers["alt-svc"]
-
-
# https://datatracker.ietf.org/doc/html/rfc7838#section-3
-
# A field value containing the special value "clear" indicates that the
-
# origin requests all alternatives for that origin to be invalidated
-
# (including those specified in the same response, in case of an
-
# invalid reply containing both "clear" and alternative services).
-
120
if altsvc == "clear"
-
9
@altsvc_mutex.synchronize do
-
9
@altsvcs[origin].clear
-
end
-
-
8
return
-
end
-
-
111
parse(altsvc) do |alt_origin, alt_params|
-
18
alt_origin.host ||= host
-
18
yield(alt_origin, origin, alt_params)
-
end
-
end
-
-
30
def parse(altsvc)
-
237
return enum_for(__method__, altsvc) unless block_given?
-
-
174
scanner = StringScanner.new(altsvc)
-
181
until scanner.eos?
-
174
alt_service = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
-
-
174
alt_params = []
-
174
loop do
-
201
alt_param = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
-
201
alt_params << alt_param.strip if alt_param
-
201
scanner.skip(/;/)
-
201
break if scanner.eos? || scanner.scan(/ *, */)
-
end
-
339
alt_params = Hash[alt_params.map { |field| field.split("=", 2) }]
-
-
174
alt_proto, alt_authority = alt_service.split("=", 2)
-
174
alt_origin = parse_altsvc_origin(alt_proto, alt_authority)
-
174
return unless alt_origin
-
-
63
yield(alt_origin, alt_params.merge("proto" => alt_proto))
-
end
-
end
-
-
30
def parse_altsvc_scheme(alt_proto)
-
181
case alt_proto
-
when "h2c"
-
9
"http"
-
when "h2"
-
72
"https"
-
end
-
end
-
-
30
def parse_altsvc_origin(alt_proto, alt_origin)
-
174
alt_scheme = parse_altsvc_scheme(alt_proto)
-
-
174
return unless alt_scheme
-
-
63
alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
-
-
63
URI.parse("#{alt_scheme}://#{alt_origin}")
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "forwardable"
-
-
30
module HTTPX
-
# Internal class to abstract a string buffer, by wrapping a string and providing the
-
# minimum possible API and functionality required.
-
#
-
# buffer = Buffer.new(640)
-
# buffer.full? #=> false
-
# buffer << "aa"
-
# buffer.capacity #=> 638
-
#
-
30
class Buffer
-
30
extend Forwardable
-
-
30
def_delegator :@buffer, :to_s
-
-
30
def_delegator :@buffer, :to_str
-
-
30
def_delegator :@buffer, :empty?
-
-
30
def_delegator :@buffer, :bytesize
-
-
30
def_delegator :@buffer, :clear
-
-
30
def_delegator :@buffer, :replace
-
-
30
attr_reader :limit
-
-
30
if RUBY_VERSION >= "3.4.0"
-
18
def initialize(limit)
-
6409
@buffer = String.new("", encoding: Encoding::BINARY, capacity: limit)
-
6409
@limit = limit
-
end
-
-
18
def <<(chunk)
-
22559
@buffer.append_as_bytes(chunk)
-
end
-
else
-
12
def initialize(limit)
-
20938
@buffer = "".b
-
20938
@limit = limit
-
end
-
-
12
def_delegator :@buffer, :<<
-
end
-
-
30
def full?
-
68548
@buffer.bytesize >= @limit
-
end
-
-
30
def capacity
-
14
@limit - @buffer.bytesize
-
end
-
-
30
def shift!(fin)
-
24409
@buffer = @buffer.byteslice(fin..-1) || "".b
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Callbacks
-
30
def on(type, &action)
-
302456
callbacks(type) << action
-
302456
action
-
end
-
-
30
def once(type, &block)
-
157197
on(type) do |*args, &callback|
-
84121
block.call(*args, &callback)
-
84049
:delete
-
end
-
end
-
-
30
def emit(type, *args)
-
153932
log { "emit #{type.inspect} callbacks" } if respond_to?(:log)
-
269306
callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
-
end
-
-
30
def callbacks_for?(type)
-
3898
@callbacks && @callbacks.key?(type) && @callbacks[type].any?
-
end
-
-
30
protected
-
-
30
def callbacks(type = nil)
-
484327
return @callbacks unless type
-
-
705428
@callbacks ||= Hash.new { |h, k| h[k] = [] }
-
484220
@callbacks[type]
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
# Session mixin, implements most of the APIs that the users call.
-
# delegates to a default session when extended.
-
30
module Chainable
-
30
%w[head get post put delete trace options connect patch].each do |meth|
-
261
class_eval(<<-MOD, __FILE__, __LINE__ + 1)
-
9
def #{meth}(*uri, **options) # def get(*uri, **options)
-
18
request("#{meth.upcase}", uri, **options) # request("GET", uri, **options)
-
end # end
-
MOD
-
end
-
-
# delegates to the default session (see HTTPX::Session#request).
-
30
def request(*args, **options)
-
3162
branch(default_options).request(*args, **options)
-
end
-
-
30
def accept(type)
-
18
with(headers: { "accept" => String(type) })
-
end
-
-
# delegates to the default session (see HTTPX::Session#wrap).
-
30
def wrap(&blk)
-
100
branch(default_options).wrap(&blk)
-
end
-
-
# returns a new instance loaded with the +pl+ plugin and +options+.
-
30
def plugin(pl, options = nil, &blk)
-
7194
klass = is_a?(S) ? self.class : Session
-
7194
klass = Class.new(klass)
-
7194
klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
-
7194
klass.plugin(pl, options, &blk).new
-
end
-
-
# returns a new instance loaded with +options+.
-
30
def with(options, &blk)
-
3697
branch(default_options.merge(options), &blk)
-
end
-
-
30
private
-
-
# returns default instance of HTTPX::Options.
-
30
def default_options
-
14216
@options || Session.default_options
-
end
-
-
# returns a default instance of HTTPX::Session.
-
30
def branch(options, &blk)
-
6941
return self.class.new(options, &blk) if is_a?(S)
-
-
3790
Session.new(options, &blk)
-
end
-
-
30
def method_missing(meth, *args, **options, &blk)
-
938
case meth
-
when /\Awith_(.+)/
-
-
1038
option = Regexp.last_match(1)
-
-
1038
return super unless option
-
-
1038
with(option.to_sym => args.first || options)
-
when /\Aon_(.+)/
-
10
callback = Regexp.last_match(1)
-
-
6
return super unless %w[
-
connection_opened connection_closed
-
request_error
-
request_started request_body_chunk request_completed
-
response_started response_body_chunk response_completed
-
3
].include?(callback)
-
-
10
warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
-
1
"Use `HTTPX.plugin(:callbacks).#{meth}` instead."
-
-
10
plugin(:callbacks).__send__(meth, *args, **options, &blk)
-
else
-
super
-
end
-
end
-
-
30
def respond_to_missing?(meth, *)
-
56
case meth
-
when /\Awith_(.+)/
-
45
option = Regexp.last_match(1)
-
-
45
default_options.respond_to?(option) || super
-
when /\Aon_(.+)/
-
18
callback = Regexp.last_match(1)
-
-
12
%w[
-
connection_opened connection_closed
-
request_error
-
request_started request_body_chunk request_completed
-
response_started response_body_chunk response_completed
-
5
].include?(callback) || super
-
else
-
super
-
end
-
end
-
end
-
-
30
extend Chainable
-
end
-
# frozen_string_literal: true
-
-
30
require "resolv"
-
30
require "forwardable"
-
30
require "httpx/io"
-
30
require "httpx/buffer"
-
-
30
module HTTPX
-
# The Connection can be watched for IO events.
-
#
-
# It contains the +io+ object to read/write from, and knows what to do when it can.
-
#
-
# It defers connecting until absolutely necessary. Connection should be triggered from
-
# the IO selector (until then, any request will be queued).
-
#
-
# A connection boots up its parser after connection is established. All pending requests
-
# will be redirected there after connection.
-
#
-
# A connection can be prevented from closing by the parser, that is, if there are pending
-
# requests. This will signal that the connection was prematurely closed, due to a possible
-
# number of conditions:
-
#
-
# * Remote peer closed the connection ("Connection: close");
-
# * Remote peer doesn't support pipelining;
-
#
-
# A connection may also route requests for a different host for which the +io+ was connected
-
# to, provided that the IP is the same and the port and scheme as well. This will allow to
-
# share the same socket to send HTTP/2 requests to different hosts.
-
#
-
30
class Connection
-
30
extend Forwardable
-
30
include Loggable
-
30
include Callbacks
-
-
30
using URIExtensions
-
-
30
def_delegator :@write_buffer, :empty?
-
-
30
attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
-
-
30
attr_writer :current_selector
-
-
30
attr_accessor :current_session, :family
-
-
30
protected :ssl_session, :sibling
-
-
30
def initialize(uri, options)
-
1776
@current_session = @current_selector = @max_concurrent_requests =
-
@parser = @sibling = @coalesced_connection = @altsvc_connection =
-
@family = @io = @ssl_session = @timeout =
-
7623
@connected_at = @response_received_at = nil
-
-
9399
@exhausted = @cloned = @main_sibling = false
-
-
9399
@options = Options.new(options)
-
9399
@type = initialize_type(uri, @options)
-
9399
@origins = [uri.origin]
-
9399
@origin = Utils.to_uri(uri.origin)
-
9399
@window_size = @options.window_size
-
9399
@read_buffer = Buffer.new(@options.buffer_size)
-
9399
@write_buffer = Buffer.new(@options.buffer_size)
-
9399
@pending = []
-
9399
@inflight = 0
-
9399
@keep_alive_timeout = @options.timeout[:keep_alive_timeout]
-
9399
@no_more_requests_counter = 0
-
-
9399
if @options.io
-
# if there's an already open IO, get its
-
# peer address, and force-initiate the parser
-
75
transition(:already_open)
-
75
@io = build_socket
-
75
parser
-
else
-
9324
transition(:idle)
-
end
-
9399
self.addresses = @options.addresses if @options.addresses
-
end
-
-
30
def peer
-
19775
@origin
-
end
-
-
# this is a semi-private method, to be used by the resolver
-
# to initiate the io object.
-
30
def addresses=(addrs)
-
9057
if @io
-
439
@io.add_addresses(addrs)
-
else
-
8618
@io = build_socket(addrs)
-
end
-
end
-
-
30
def addresses
-
18016
@io && @io.addresses
-
end
-
-
30
def addresses?
-
10388
@io && @io.addresses?
-
end
-
-
30
def match?(uri, options)
-
3168
return false if !used? && (@state == :closing || @state == :closed)
-
-
3005
@origins.include?(uri.origin) &&
-
# if there is more than one origin to match, it means that this connection
-
# was the result of coalescing. To prevent blind trust in the case where the
-
# origin came from an ORIGIN frame, we're going to verify the hostname with the
-
# SSL certificate
-
2766
(@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host))) &&
-
@options.connection_options_match?(options)
-
end
-
-
30
def mergeable?(connection)
-
501
return false if @state == :closing || @state == :closed || !@io
-
-
132
return false unless connection.addresses
-
-
4
(
-
132
(open? && @origin == connection.origin) ||
-
132
!(@io.addresses & (connection.addresses || [])).empty?
-
) && @options.connection_options_match?(connection.options)
-
end
-
-
# coalesces +self+ into +connection+.
-
30
def coalesce!(connection)
-
29
@coalesced_connection = connection
-
-
29
close_sibling
-
29
connection.merge(self)
-
end
-
-
30
def coalesced?
-
10340
@coalesced_connection
-
end
-
-
# coalescable connections need to be mergeable!
-
# but internally, #mergeable? is called before #coalescable?
-
30
def coalescable?(connection)
-
58
if @io.protocol == "h2" &&
-
@origin.scheme == "https" &&
-
connection.origin.scheme == "https" &&
-
@io.can_verify_peer?
-
29
@io.verify_hostname(connection.origin.host)
-
else
-
29
@origin == connection.origin
-
end
-
end
-
-
30
def merge(connection)
-
70
@origins |= connection.instance_variable_get(:@origins)
-
74
if @ssl_session.nil? && connection.ssl_session
-
24
@ssl_session = connection.ssl_session
-
3
@io.session_new_cb do |sess|
-
48
@ssl_session = sess
-
24
end if @io
-
end
-
74
connection.purge_pending do |req|
-
35
req.transition(:idle)
-
35
send(req)
-
end
-
end
-
-
30
def purge_pending(&block)
-
74
pendings = []
-
74
if @parser
-
36
pending = @parser.pending
-
32
@inflight -= pending.size
-
36
pendings << pending
-
end
-
74
pendings << @pending
-
74
pendings.each do |pending|
-
110
pending.reject!(&block)
-
end
-
end
-
-
30
def io_connected?
-
14
return @coalesced_connection.io_connected? if @coalesced_connection
-
-
14
@io && @io.state == :connected
-
end
-
-
30
def connecting?
-
231811
@state == :idle
-
end
-
-
30
def inflight?
-
3701
@parser && (
-
# parser may be dealing with other requests (possibly started from a different fiber)
-
3124
!@parser.empty? ||
-
# connection may be doing connection termination handshake
-
!@write_buffer.empty?
-
)
-
end
-
-
30
def interests
-
# connecting
-
193531
if connecting?
-
14814
connect
-
-
14813
return @io.interests if connecting?
-
end
-
-
179540
return @parser.interests if @parser
-
-
35
nil
-
rescue StandardError => e
-
on_error(e)
-
nil
-
end
-
-
30
def to_io
-
58621
@io.to_io
-
end
-
-
30
def call
-
35467
case @state
-
when :idle
-
13725
connect
-
-
# when opening the tcp or ssl socket fails
-
13708
return if @state == :closed
-
-
13682
consume
-
when :closed
-
return
-
when :closing
-
consume
-
transition(:closed)
-
when :open
-
23755
consume
-
end
-
20102
nil
-
rescue IOError => e
-
@write_buffer.clear
-
on_io_error(e)
-
rescue StandardError => e
-
61
@write_buffer.clear
-
61
on_error(e)
-
rescue Exception => e # rubocop:disable Lint/RescueException
-
116
force_close(true)
-
107
raise e
-
end
-
-
30
def close
-
3434
transition(:active) if @state == :inactive
-
-
3434
@parser.close if @parser
-
end
-
-
30
def terminate
-
3036
case @state
-
when :idle
-
purge_after_closed
-
disconnect
-
when :closed
-
31
@connected_at = nil
-
end
-
-
3389
close
-
end
-
-
# bypasses state machine rules while setting the connection in the
-
# :closed state.
-
30
def force_close(delete_pending = false)
-
454
force_purge
-
454
return unless @state == :closed
-
-
454
if delete_pending
-
284
@pending.clear
-
170
elsif (parser = @parser)
-
enqueue_pending_requests_from_parser(parser)
-
end
-
-
454
return unless @pending.empty?
-
-
454
disconnect
-
445
emit(:force_closed, delete_pending)
-
end
-
-
# bypasses the state machine to force closing of connections still connecting.
-
# **only** used for Happy Eyeballs v2.
-
30
def force_reset(cloned = false)
-
241
@state = :closing
-
241
@cloned = cloned
-
241
transition(:closed)
-
end
-
-
30
def reset
-
10024
return if @state == :closing || @state == :closed
-
-
# do not reset a connection which may have restarted back to :idle, such when the parser resets
-
# (example: HTTP/1 parser disabling pipelining)
-
9981
return if @state == :idle && @pending.any?
-
-
9960
parser = @parser
-
-
9960
if parser && parser.respond_to?(:max_concurrent_requests)
-
# if connection being reset has at some downgraded the number of concurrent
-
# requests, such as in the case where an attempt to use HTTP/1 pipelining failed,
-
# keep that information around.
-
5473
@max_concurrent_requests = parser.max_concurrent_requests
-
end
-
-
9960
transition(:closing)
-
-
9960
transition(:closed)
-
end
-
-
30
def send(request)
-
11728
return @coalesced_connection.send(request) if @coalesced_connection
-
-
11707
if @parser && !@write_buffer.full?
-
580
if @response_received_at && @keep_alive_timeout &&
-
Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
-
# when pushing a request into an existing connection, we have to check whether there
-
# is the possibility that the connection might have extended the keep alive timeout.
-
# for such cases, we want to ping for availability before deciding to shovel requests.
-
31
log(level: 3) { "keep alive timeout expired, pinging connection..." }
-
31
@pending << request
-
31
transition(:active) if @state == :inactive
-
31
request.ping!
-
31
ping
-
28
return
-
end
-
-
549
send_request_to_parser(request)
-
else
-
11127
@pending << request
-
end
-
end
-
-
30
def timeout
-
45492
return if @state == :closed || @state == :inactive
-
-
45491
return @timeout if @timeout
-
-
18761
return @options.timeout[:connect_timeout] if @state == :idle
-
-
18761
@options.timeout[:operation_timeout]
-
end
-
-
30
def idling
-
1392
purge_after_closed
-
1392
@write_buffer.clear
-
1392
transition(:idle)
-
1392
return unless @parser
-
-
1392
enqueue_pending_requests_from_parser(parser)
-
1392
@parser = nil
-
end
-
-
30
def used?
-
6998
@connected_at
-
end
-
-
30
def deactivate
-
535
transition(:inactive)
-
end
-
-
30
def open?
-
9451
@state == :open || @state == :inactive
-
end
-
-
30
def handle_socket_timeout(interval)
-
49
error = OperationTimeoutError.new(interval, "timed out while waiting on select")
-
49
error.set_backtrace(caller)
-
49
on_error(error)
-
end
-
-
30
def sibling=(connection)
-
112
@sibling = connection
-
-
112
return unless connection
-
-
84
@main_sibling = connection.sibling.nil?
-
-
84
return unless @main_sibling
-
-
42
connection.sibling = self
-
end
-
-
30
def handle_connect_error(error)
-
397
return on_error(error) unless @sibling && @sibling.connecting?
-
-
7
@sibling.merge(self)
-
-
7
force_reset(true)
-
end
-
-
# disconnects from the current session it's attached to
-
30
def disconnect
-
10766
return if @exhausted # it'll reset
-
-
10766
return unless (current_session = @current_session) && (current_selector = @current_selector)
-
-
10622
@current_session = @current_selector = nil
-
-
10622
current_session.deselect_connection(self, current_selector, @cloned)
-
end
-
-
30
def on_connect_error(e)
-
# connect errors, exit gracefully
-
93
error = ConnectionError.new(e.message)
-
93
error.set_backtrace(e.backtrace)
-
93
handle_connect_error(error) if connecting?
-
93
force_close
-
end
-
-
30
def on_io_error(e)
-
on_error(e)
-
force_close(true)
-
end
-
-
30
def on_error(error, request = nil)
-
1431
if error.is_a?(OperationTimeoutError)
-
-
# inactive connections do not contribute to the select loop, therefore
-
# they should not fail due to such errors.
-
49
return if @state == :inactive
-
-
49
if @timeout
-
37
@timeout -= error.timeout
-
42
return unless @timeout <= 0
-
end
-
-
49
error = error.to_connection_error if connecting?
-
end
-
1431
handle_error(error, request)
-
1413
reset
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id} " \
-
skipped
"@origin=#{@origin} " \
-
skipped
"@state=#{@state} " \
-
skipped
"@pending=#{@pending.size} " \
-
skipped
"@io=#{@io}>"
-
skipped
end
-
skipped
# :nocov:
-
-
30
private
-
-
30
def connect
-
27274
transition(:open)
-
end
-
-
30
def consume
-
41504
return unless @io
-
-
41504
catch(:called) do
-
41504
epiped = false
-
41504
loop do
-
# connection may have
-
59611
return if @state == :idle
-
-
55226
parser.consume
-
-
# we exit if there's no more requests to process
-
#
-
# this condition takes into account:
-
#
-
# * the number of pending requests
-
# * the number of inflight requests
-
# * whether the write buffer has bytes (i.e. for close handshake)
-
55208
if @pending.empty? && @inflight.zero? && @write_buffer.empty?
-
9552
no_more_requests_loop_check if @parser && @parser.pending.any?
-
-
# terminate if an altsvc connection has been established
-
9552
terminate if @altsvc_connection
-
-
9552
return
-
end
-
-
45656
@timeout = @current_timeout
-
-
45656
read_drained = false
-
45656
write_drained = nil
-
-
#
-
# tight read loop.
-
#
-
# read as much of the socket as possible.
-
#
-
# this tight loop reads all the data it can from the socket and pipes it to
-
# its parser.
-
#
-
11956
loop do
-
68788
siz = @io.read(@window_size, @read_buffer)
-
68940
log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
-
68744
unless siz
-
13
@write_buffer.clear
-
-
13
ex = EOFError.new("descriptor closed")
-
13
ex.set_backtrace(caller)
-
13
on_error(ex)
-
13
return
-
end
-
-
# socket has been drained. mark and exit the read loop.
-
68731
if siz.zero?
-
17362
read_drained = @read_buffer.empty?
-
17362
epiped = false
-
17362
break
-
end
-
-
51369
parser << @read_buffer.to_s
-
-
# continue reading if possible.
-
45899
break if interests == :w && !epiped
-
-
# exit the read loop if connection is preparing to be closed
-
39994
break if @state == :closing || @state == :closed
-
-
# exit #consume altogether if all outstanding requests have been dealt with
-
39856
if @pending.empty? && @inflight.zero? && @write_buffer.empty? # rubocop:disable Style/Next
-
3856
no_more_requests_loop_check if @parser && @parser.pending.any?
-
-
# terminate if an altsvc connection has been established
-
3856
terminate if @altsvc_connection
-
-
3856
return
-
end
-
45656
end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
-
-
#
-
# tight write loop.
-
#
-
# flush as many bytes as the sockets allow.
-
#
-
8577
loop do
-
# buffer has been drained, mark and exit the write loop.
-
26957
if @write_buffer.empty?
-
# we only mark as drained on the first loop
-
3369
write_drained = write_drained.nil? && @inflight.positive?
-
-
3369
break
-
end
-
-
2303
begin
-
23588
siz = @io.write(@write_buffer)
-
rescue Errno::EPIPE
-
# this can happen if we still have bytes in the buffer to send to the server, but
-
# the server wants to respond immediately with some message, or an error. An example is
-
# when one's uploading a big file to an unintended endpoint, and the server stops the
-
# consumption, and responds immediately with an authorization of even method not allowed error.
-
# at this point, we have to let the connection switch to read-mode.
-
6
log(level: 2) { "pipe broken, could not flush buffer..." }
-
6
epiped = true
-
6
read_drained = false
-
6
break
-
end
-
23689
log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
-
23581
unless siz
-
@write_buffer.clear
-
-
ex = EOFError.new("descriptor closed")
-
ex.set_backtrace(caller)
-
on_error(ex)
-
return
-
end
-
-
# socket closed for writing. mark and exit the write loop.
-
23581
if siz.zero?
-
18
write_drained = !@write_buffer.empty?
-
18
break
-
end
-
-
# exit write loop if marked to consume from peer, or is closing.
-
23563
break if interests == :r || @state == :closing || @state == :closed
-
-
3697
write_drained = false
-
36273
end unless (ints = interests) == :r
-
-
36272
send_pending if @state == :open
-
-
# return if socket is drained
-
36272
next unless (ints != :r || read_drained) && (ints != :w || write_drained)
-
-
# gotta go back to the event loop. It happens when:
-
#
-
# * the socket is drained of bytes or it's not the interest of the conn to read;
-
# * theres nothing more to write, or it's not in the interest of the conn to write;
-
18243
log(level: 3) { "(#{ints}): WAITING FOR EVENTS..." }
-
18165
return
-
end
-
end
-
end
-
-
30
def send_pending
-
95651
while !@write_buffer.full? && (request = @pending.shift)
-
21233
send_request_to_parser(request)
-
end
-
end
-
-
30
def parser
-
139565
@parser ||= build_parser
-
end
-
-
30
def send_request_to_parser(request)
-
20706
@inflight += 1
-
21782
request.peer_address = @io.ip && @io.ip.address
-
21782
set_request_timeouts(request)
-
-
21782
parser.send(request)
-
-
21782
return unless @state == :inactive
-
-
33
transition(:active)
-
# mark request as ping, as this inactive connection may have been
-
# closed by the server, and we don't want that to influence retry
-
# bookkeeping.
-
33
request.ping!
-
end
-
-
30
def enqueue_pending_requests_from_parser(parser)
-
6452
parser.reset_requests # move sequential requests back to pending queue.
-
6452
parser_pending_requests = parser.pending
-
-
6452
return if parser_pending_requests.empty?
-
-
# the connection will be reused, so parser requests must come
-
# back to the pending list before the parser is reset.
-
309
@inflight -= parser_pending_requests.size
-
332
@pending.unshift(*parser_pending_requests)
-
-
332
parser.pending.clear
-
end
-
-
30
def build_parser(protocol = @io.protocol)
-
9583
parser = parser_type(protocol).new(@write_buffer, @options)
-
9583
set_parser_callbacks(parser)
-
9583
parser.max_concurrent_requests = @max_concurrent_requests if @max_concurrent_requests && parser.respond_to?(:max_concurrent_requests=)
-
9583
parser
-
end
-
-
30
def set_parser_callbacks(parser)
-
9716
parser.on(:response) do |request, response|
-
9840
AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
-
18
build_altsvc_connection(alt_origin, origin, alt_params)
-
end
-
9840
@response_received_at = Utils.now
-
9840
@no_more_requests_counter = 0
-
8907
@inflight -= 1
-
9840
response.finish!
-
9840
request.emit_response(response)
-
end
-
9716
parser.on(:altsvc) do |alt_origin, origin, alt_params|
-
build_altsvc_connection(alt_origin, origin, alt_params)
-
end
-
-
9716
parser.on(:pong, &method(:pong))
-
-
9716
parser.on(:promise) do |request, stream|
-
27
request.emit(:promise, parser, stream)
-
end
-
9716
parser.on(:exhausted) do
-
9
enqueue_pending_requests_from_parser(parser)
-
-
9
@exhausted = true
-
9
parser.close
-
-
9
idling
-
9
@exhausted = false
-
end
-
9716
parser.on(:origin) do |origin|
-
@origins |= [origin]
-
end
-
9716
parser.on(:close) do
-
3532
reset
-
end
-
9716
parser.on(:close_handshake) do
-
21
consume unless @state == :closed
-
end
-
9716
parser.on(:reset) do
-
5016
enqueue_pending_requests_from_parser(parser)
-
-
5016
reset
-
# :reset event only fired in http/1.1, so this guarantees
-
# that the connection will be closed here.
-
5007
idling unless @pending.empty?
-
end
-
9716
parser.on(:current_timeout) do
-
4130
@current_timeout = @timeout = parser.timeout
-
end
-
9716
parser.on(:timeout) do |tout|
-
3420
@timeout = tout
-
end
-
9716
parser.on(:error) do |request, error|
-
226
case error
-
when :http_1_1_required
-
27
current_session = @current_session
-
27
current_selector = @current_selector
-
27
parser.close
-
-
27
other_connection = current_session.find_connection(@origin, current_selector,
-
@options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
-
27
other_connection.merge(self)
-
27
request.transition(:idle)
-
27
other_connection.send(request)
-
27
next
-
when OperationTimeoutError
-
# request level timeouts should take precedence
-
1
next unless request.active_timeouts.empty?
-
end
-
-
202
@inflight -= 1
-
216
response = ErrorResponse.new(request, error)
-
216
request.response = response
-
216
request.emit_response(response)
-
end
-
end
-
-
30
def transition(nextstate)
-
60783
handle_transition(nextstate)
-
rescue Errno::ECONNABORTED,
-
Errno::ECONNREFUSED,
-
Errno::ECONNRESET,
-
Errno::EADDRNOTAVAIL,
-
Errno::EHOSTUNREACH,
-
Errno::EINVAL,
-
Errno::ENETUNREACH,
-
Errno::EPIPE,
-
Errno::ENOENT,
-
SocketError,
-
IOError => e
-
93
on_connect_error(e)
-
rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
-
# connect errors, exit gracefully
-
27
handle_error(e)
-
27
handle_connect_error(e) if connecting?
-
27
force_close
-
end
-
-
30
def handle_transition(nextstate)
-
54688
case nextstate
-
when :idle
-
10724
@timeout = @current_timeout = @options.timeout[:connect_timeout]
-
-
10724
@connected_at = @response_received_at = nil
-
when :open
-
27685
return if @state == :closed
-
-
27685
@io.connect
-
27565
close_sibling if @io.state == :connected
-
-
27565
return unless @io.connected?
-
-
9623
@connected_at = Utils.now
-
-
9623
send_pending
-
-
9623
@timeout = @current_timeout = parser.timeout
-
9623
emit(:open)
-
when :inactive
-
535
return unless @state == :open
-
-
# @type ivar @parser: HTTP1 | HTTP2
-
-
# do not deactivate connection in use
-
510
return if @inflight.positive? || @parser.waiting_for_ping?
-
when :closing
-
9960
return unless connecting? || @state == :open
-
when :closed
-
10201
return unless @state == :closing
-
10201
return unless @write_buffer.empty?
-
-
10201
purge_after_closed
-
when :already_open
-
75
nextstate = :open
-
# the first check for given io readiness must still use a timeout.
-
# connect is the reasonable choice in such a case.
-
75
@timeout = @options.timeout[:connect_timeout]
-
75
send_pending
-
when :active
-
294
return unless @state == :inactive
-
-
294
nextstate = :open
-
-
# activate
-
294
@current_session.select_connection(self, @current_selector)
-
end
-
42369
log(level: 3) { "#{@state} -> #{nextstate}" }
-
42071
@state = nextstate
-
# post state change
-
37942
case nextstate
-
when :inactive
-
504
disconnect
-
when :closing
-
9960
return if @write_buffer.empty?
-
-
# try flushing termination handshakes
-
3421
consume
-
3421
@write_buffer.clear
-
when :closed
-
# TODO: should this raise an error instead?
-
10201
return unless @pending.empty?
-
-
9824
disconnect
-
end
-
end
-
-
30
def force_purge
-
454
return if @state == :closed
-
-
297
@state = :closed
-
297
@write_buffer.clear
-
14
begin
-
297
purge_after_closed
-
rescue IOError
-
# may be raised when closing the socket.
-
# due to connection reuse / fiber scheduling, it may
-
# have been reopened, to bail out in that case.
-
end
-
end
-
-
30
def close_sibling
-
12961
sibling = @sibling
-
-
12961
return unless sibling
-
-
14
if sibling.io_connected?
-
reset
-
# TODO: transition connection to closed
-
end
-
-
14
unless sibling.state == :closed
-
7
merge(sibling) unless @main_sibling
-
7
sibling.force_reset(true)
-
end
-
-
14
@sibling = nil
-
end
-
-
30
def purge_after_closed
-
12013
@io.close if @io
-
12013
@read_buffer.clear
-
12013
@timeout = nil
-
end
-
-
30
def initialize_type(uri, options)
-
8961
options.transport || begin
-
8086
case uri.scheme
-
when "http"
-
5054
"tcp"
-
when "https"
-
3875
"ssl"
-
else
-
raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
-
end
-
end
-
end
-
-
# returns an HTTPX::Connection for the negotiated Alternative Service (or none).
-
30
def build_altsvc_connection(alt_origin, origin, alt_params)
-
18
return if @altsvc_connection
-
-
# do not allow security downgrades on altsvc negotiation
-
9
return if @origin.scheme == "https" && alt_origin.scheme != "https"
-
-
9
altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
-
-
# altsvc already exists, somehow it wasn't advertised, probably noop
-
9
return unless altsvc
-
-
9
alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
-
-
9
connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
-
-
# advertised altsvc is the same origin being used, ignore
-
9
return if connection == self
-
-
9
connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
-
-
9
@altsvc_connection = connection
-
-
9
log(level: 1) { "#{origin}: alt-svc connection##{connection.object_id} established to #{alt_origin}" }
-
-
9
connection.merge(self)
-
rescue UnsupportedSchemeError
-
altsvc["noop"] = true
-
nil
-
end
-
-
30
def build_socket(addrs = nil)
-
7820
case @type
-
when "tcp"
-
5014
TCP.new(peer, addrs, @options)
-
when "ssl"
-
3647
SSL.new(peer, addrs, @options) do |sock|
-
3622
sock.ssl_session = @ssl_session
-
3622
sock.session_new_cb do |sess|
-
6449
@ssl_session = sess
-
-
6449
sock.ssl_session = sess
-
end
-
end
-
when "unix"
-
32
path = Array(addrs).first
-
-
32
path = String(path) if path
-
-
32
UNIX.new(peer, path, @options)
-
else
-
raise Error, "unsupported transport (#{@type})"
-
end
-
end
-
-
30
def ping
-
31
return if parser.waiting_for_ping?
-
-
31
parser.ping
-
31
call
-
end
-
-
30
def pong
-
18
@response_received_at = Utils.now
-
18
@no_more_requests_counter = 0
-
18
send_pending
-
end
-
-
30
def no_more_requests_loop_check
-
log(level: 3) { "NO MORE REQUESTS..." }
-
@no_more_requests_counter += 1
-
-
return if @no_more_requests_counter < 50
-
-
raise Error, "connection corrupted, aborted after looping for a while, " \
-
"please report this https://gitlab.com/os85/httpx/-/work_items " \
-
"along with debug logs"
-
end
-
-
# recover internal state and emit all relevant error responses when +error+ was raised.
-
# this takes an optiona +request+ which may have already been handled and can be opted out
-
# in the state recovery process.
-
30
def handle_error(error, request = nil)
-
1458
if request
-
682
@inflight -= 1
-
767
response = ErrorResponse.new(request, error)
-
767
request.response = response
-
767
request.emit_response(response)
-
end
-
-
1458
pending = @pending
-
1458
if (parser = @parser) && parser.respond_to?(:handle_error)
-
# parser.handle_error may disconnect the connection
-
852
pending = @pending.dup
-
852
@pending = []
-
-
852
parser.handle_error(error, request)
-
end
-
-
3195
while (req = pending.shift)
-
597
next if request && req == request
-
-
579
resp = ErrorResponse.new(req, error)
-
579
req.response = resp
-
561
req.emit_response(resp)
-
end
-
end
-
-
30
def set_request_timeouts(request)
-
21782
request.connection = self
-
21782
set_request_write_timeout(request)
-
21782
set_request_read_timeout(request)
-
21782
set_request_request_timeout(request)
-
end
-
-
30
def set_request_read_timeout(request)
-
21782
read_timeout = request.read_timeout
-
-
21782
return if read_timeout.nil? || read_timeout.infinite?
-
-
21335
set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
-
18
read_timeout_callback(request, read_timeout)
-
end
-
end
-
-
30
def set_request_write_timeout(request)
-
21782
write_timeout = request.write_timeout
-
-
21782
return if write_timeout.nil? || write_timeout.infinite?
-
-
21782
set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
-
18
write_timeout_callback(request, write_timeout)
-
end
-
end
-
-
30
def set_request_request_timeout(request)
-
21463
request_timeout = request.request_timeout
-
-
21463
return if request_timeout.nil? || request_timeout.infinite?
-
-
1417
set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
-
804
read_timeout_callback(request, request_timeout, RequestTimeoutError)
-
end
-
end
-
-
30
def write_timeout_callback(request, timeout)
-
18
return if request.state == :done
-
-
18
@write_buffer.clear
-
18
error = WriteTimeoutError.new(request, nil, timeout)
-
-
18
request.handle_error(error)
-
end
-
-
30
def read_timeout_callback(request, timeout, error_type = ReadTimeoutError)
-
822
response = request.response
-
-
822
return if response && response.finished?
-
-
767
@write_buffer.clear
-
767
error = error_type.new(request, response, timeout)
-
-
767
request.handle_error(error)
-
end
-
-
30
def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
-
44608
request.set_timeout_callback(start_event) do
-
22806
unless (selector = @current_selector)
-
raise Error, "request has been resend to an out-of-session connection, and this " \
-
"should never happen!!! Please report this error! " \
-
"(state:#{@state}, " \
-
"parser?:#{!!@parser}, " \
-
"bytes in write buffer?:#{!@write_buffer.empty?}, " \
-
"cloned?:#{@cloned}, " \
-
"sibling?:#{!!@sibling}, " \
-
"coalesced?:#{coalesced?})"
-
end
-
-
22806
timer = selector.after(timeout, callback)
-
22806
request.active_timeouts << label
-
-
22806
Array(finish_events).each do |event|
-
# clean up request timeouts if the connection errors out
-
33822
request.set_timeout_callback(event) do
-
32600
timer.cancel
-
32600
request.active_timeouts.delete(label)
-
end
-
end
-
end
-
end
-
-
30
def parser_type(protocol)
-
8795
case protocol
-
4161
when "h2" then @options.http2_class
-
5618
when "http/1.1" then @options.http1_class
-
else
-
raise Error, "unsupported protocol (##{protocol})"
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "httpx/parser/http1"
-
-
30
module HTTPX
-
30
class Connection::HTTP1
-
30
include Callbacks
-
30
include Loggable
-
-
30
MAX_REQUESTS = 200
-
30
CRLF = "\r\n"
-
-
30
UPCASED = {
-
"www-authenticate" => "WWW-Authenticate",
-
"http2-settings" => "HTTP2-Settings",
-
"content-md5" => "Content-MD5",
-
}.freeze
-
30
attr_reader :pending, :requests
-
-
30
attr_accessor :max_concurrent_requests
-
-
30
def initialize(buffer, options)
-
5627
@options = options
-
5627
@max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
-
5627
@max_requests = @options.max_requests
-
5627
@parser = Parser::HTTP1.new(self)
-
5627
@buffer = buffer
-
5627
@version = [1, 1]
-
5627
@pending = []
-
5627
@requests = []
-
5627
@request = nil
-
5627
@handshake_completed = @pipelining = false
-
end
-
-
30
def timeout
-
5479
@options.timeout[:operation_timeout]
-
end
-
-
30
def interests
-
58674
request = @request || @requests.first
-
-
58674
return unless request
-
-
58648
return :w if request.interests == :w || !@buffer.empty?
-
-
40623
:r
-
end
-
-
30
def reset
-
5160
@max_requests = @options.max_requests || MAX_REQUESTS
-
5160
@parser.reset!
-
5160
@handshake_completed = false
-
5160
reset_requests
-
end
-
-
30
def reset_requests
-
11122
@requests.reverse_each do |request|
-
10851
next if request.response
-
-
10833
request.transition(:idle)
-
10833
@pending.unshift(request)
-
end
-
11122
@requests.clear
-
end
-
-
30
def close
-
108
reset
-
108
emit(:close)
-
end
-
-
30
def exhausted?
-
553
!@max_requests.positive?
-
end
-
-
30
def empty?
-
# this means that for every request there's an available
-
# partial response, so there are no in-flight requests waiting.
-
55
@requests.empty? || (
-
# checking all responses can be time-consuming. Alas, as in HTTP/1, responses
-
# do not come out of order, we can get away with checking first and last.
-
!@requests.first.response.nil? &&
-
(@requests.size == 1 || !@requests.last.response.nil?)
-
)
-
end
-
-
30
def <<(data)
-
8588
@parser << data
-
end
-
-
30
def send(request)
-
17011
unless @max_requests.positive?
-
@pending << request
-
return
-
end
-
-
17011
return if @requests.include?(request)
-
-
17011
@requests << request
-
17011
@pipelining = @max_concurrent_requests > 1 && @requests.size > 1
-
end
-
-
30
def consume
-
20119
requests_limit = [@max_requests, @requests.size].min
-
20119
concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
-
20119
@requests.each_with_index do |request, idx|
-
23332
break if idx >= concurrent_requests_limit
-
20396
next unless request.can_buffer?
-
-
7718
handle(request)
-
end
-
end
-
-
# HTTP Parser callbacks
-
#
-
# must be public methods, or else they won't be reachable
-
-
30
def on_start
-
5855
log(level: 2) { "parsing begins" }
-
end
-
-
30
def on_headers(h)
-
5801
@request = @requests.first
-
-
5801
return if @request.response
-
-
5855
log(level: 2) { "headers received" }
-
5801
headers = @request.options.headers_class.new(h)
-
5801
response = @request.options.response_class.new(@request,
-
@parser.status_code,
-
@parser.http_version.join("."),
-
headers)
-
5855
log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
-
6287
log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{log_redact_headers(v)}" }.join("\n") }
-
-
5801
@request.response = response
-
5792
on_complete if response.finished?
-
end
-
-
30
def on_trailers(h)
-
9
return unless @request
-
-
9
response = @request.response
-
9
log(level: 2) { "trailer headers received" }
-
-
9
log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{log_redact_headers(v.join(", "))}" }.join("\n") }
-
9
response.merge_headers(h)
-
end
-
-
30
def on_data(chunk)
-
6692
request = @request
-
-
6692
return unless request
-
-
635
begin
-
6746
log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
-
6746
log(level: 2, color: :green) { "-> #{log_redact_body(chunk.inspect)}" }
-
6692
response = request.response
-
-
6692
response << chunk
-
12
rescue StandardError => e
-
16
error_response = ErrorResponse.new(request, e)
-
16
request.response = error_response
-
16
dispatch(request)
-
end
-
end
-
-
30
def on_complete
-
5758
request = @request
-
-
5758
return unless request
-
-
5812
log(level: 2) { "parsing complete" }
-
5758
dispatch(request)
-
end
-
-
30
def dispatch(request)
-
5774
if request.expects?
-
82
@parser.reset!
-
73
return handle(request)
-
end
-
-
5692
@request = nil
-
5692
@requests.shift
-
5692
response = request.response
-
5692
emit(:response, request, response)
-
-
5612
if @parser.upgrade?
-
37
response << @parser.upgrade_data
-
37
@parser.reset!
-
37
throw(:called)
-
end
-
-
5575
@parser.reset!
-
5056
@max_requests -= 1
-
5575
if response.is_a?(ErrorResponse)
-
16
disable
-
else
-
5559
manage_connection(request, response)
-
end
-
-
553
if exhausted?
-
@pending.unshift(*@requests)
-
@requests.clear
-
-
emit(:exhausted)
-
else
-
553
send(@pending.shift) unless @pending.empty?
-
end
-
end
-
-
30
def handle_error(ex, request = nil)
-
414
if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request &&
-
9
(response = @request.response) && response.is_a?(Response) &&
-
!response.headers.key?("content-length") &&
-
!response.headers.key?("transfer-encoding")
-
# if the response does not contain a content-length header, the server closing the
-
# connnection is the indicator of response consumed.
-
# https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
-
18
catch(:called) { on_complete }
-
8
return
-
end
-
-
405
if @pipelining
-
42
catch(:called) { disable }
-
else
-
1068
while (req = @requests.shift)
-
414
next if request && request == req
-
-
72
emit(:error, req, ex)
-
end
-
692
while (req = @pending.shift)
-
next if request && request == req
-
-
emit(:error, req, ex)
-
end
-
end
-
end
-
-
30
def ping
-
reset
-
emit(:reset)
-
emit(:exhausted)
-
end
-
-
30
def waiting_for_ping?
-
30
false
-
end
-
-
30
private
-
-
30
def manage_connection(request, response)
-
5559
connection = response.headers["connection"]
-
5041
case connection
-
when /keep-alive/i
-
553
if @handshake_completed
-
if @max_requests.zero?
-
@pending.unshift(*@requests)
-
@requests.clear
-
emit(:exhausted)
-
end
-
return
-
end
-
-
553
keep_alive = response.headers["keep-alive"]
-
553
return unless keep_alive
-
-
118
parameters = Hash[keep_alive.split(/ *, */).map do |pair|
-
118
pair.split(/ *= */, 2)
-
end]
-
118
@max_requests = parameters["max"].to_i - 1 if parameters.key?("max")
-
-
118
if parameters.key?("timeout")
-
9
keep_alive_timeout = parameters["timeout"].to_i
-
9
emit(:timeout, keep_alive_timeout)
-
end
-
118
@handshake_completed = true
-
when /close/i
-
5006
disable
-
when nil
-
# In HTTP/1.1, it's keep alive by default
-
return if response.version == "1.1" && request.headers["connection"] != "close"
-
-
disable
-
end
-
end
-
-
30
def disable
-
5043
disable_pipelining
-
5043
reset
-
5043
emit(:reset)
-
5034
throw(:called)
-
end
-
-
30
def disable_pipelining
-
# do not disable pipelining if already set to 1 request at a time
-
5043
return if @max_concurrent_requests == 1
-
-
3948
@requests.each do |r|
-
152
r.transition(:idle) if r.response.nil?
-
-
# when we disable pipelining, we still want to try keep-alive.
-
# only when keep-alive with one request fails, do we fallback to
-
# connection: close.
-
152
r.headers["connection"] = "close" if @max_concurrent_requests == 1
-
end
-
# server doesn't handle pipelining, and probably
-
# doesn't support keep-alive. Fallback to send only
-
# 1 keep alive request.
-
3948
@max_concurrent_requests = 1
-
3948
@pipelining = false
-
end
-
-
30
def set_protocol_headers(request)
-
6255
if !request.headers.key?("content-length") &&
-
request.body.bytesize == Float::INFINITY
-
36
request.body.chunk!
-
end
-
-
6255
extra_headers = {}
-
-
6255
unless request.headers.key?("connection")
-
6228
connection_value = if request.persistent?
-
# when in a persistent connection, the request can't be at
-
# the edge of a renegotiation
-
227
if @requests.index(request) + 1 < @max_requests
-
227
"keep-alive"
-
else
-
"close"
-
end
-
else
-
# when it's not a persistent connection, it sets "Connection: close" always
-
# on the last request of the possible batch (either allowed max requests,
-
# or if smaller, the size of the batch itself)
-
6001
requests_limit = [@max_requests, @requests.size].min
-
6001
if request == @requests[requests_limit - 1]
-
5292
"close"
-
else
-
709
"keep-alive"
-
end
-
end
-
-
5644
extra_headers["connection"] = connection_value
-
end
-
6255
extra_headers["host"] = request.authority unless request.headers.key?("host")
-
6255
extra_headers
-
end
-
-
30
def handle(request)
-
7800
catch(:buffer_full) do
-
7800
request.transition(:headers)
-
7791
join_headers(request) if request.state == :headers
-
7791
request.transition(:body)
-
7791
join_body(request) if request.state == :body
-
6490
request.transition(:trailers)
-
# HTTP/1.1 trailers should only work for chunked encoding
-
6490
join_trailers(request) if request.body.chunked? && request.state == :trailers
-
6490
request.transition(:done)
-
end
-
end
-
-
30
def join_headline(request)
-
5580
"#{request.verb} #{request.path} HTTP/#{@version.join(".")}"
-
end
-
-
30
def join_headers(request)
-
6255
headline = join_headline(request)
-
6255
@buffer << headline << CRLF
-
6309
log(color: :yellow) { "<- HEADLINE: #{headline.chomp.inspect}" }
-
6255
extra_headers = set_protocol_headers(request)
-
6255
join_headers2(request.headers.each(extra_headers))
-
6309
log { "<- " }
-
6255
@buffer << CRLF
-
end
-
-
30
def join_body(request)
-
7538
return if request.body.empty?
-
-
7856
while (chunk = request.drain_body)
-
3847
log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
-
3847
log(level: 2, color: :green) { "<- #{log_redact_body(chunk.inspect)}" }
-
3847
@buffer << chunk
-
3847
throw(:buffer_full, request) if @buffer.full?
-
end
-
-
1835
return unless (error = request.drain_error)
-
-
raise error
-
end
-
-
30
def join_trailers(request)
-
108
return unless request.trailers? && request.callbacks_for?(:trailers)
-
-
36
join_headers2(request.trailers)
-
36
log { "<- " }
-
36
@buffer << CRLF
-
end
-
-
30
def join_headers2(headers)
-
6291
headers.each do |field, value|
-
38266
field = capitalized(field)
-
38536
log(color: :yellow) { "<- HEADER: #{[field, log_redact_headers(value)].join(": ")}" }
-
38266
@buffer << "#{field}: #{value}#{CRLF}"
-
end
-
end
-
-
30
def capitalized(field)
-
38266
UPCASED[field] || field.split("-").map(&:capitalize).join("-")
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "securerandom"
-
30
require "http/2"
-
-
30
module HTTPX
-
30
class Connection::HTTP2
-
30
include Callbacks
-
30
include Loggable
-
-
30
MAX_CONCURRENT_REQUESTS = ::HTTP2::DEFAULT_MAX_CONCURRENT_STREAMS
-
-
30
class Error < Error
-
30
def initialize(id, error)
-
94
super("stream #{id} closed with error: #{error}")
-
end
-
end
-
-
30
class PingError < Error
-
30
def initialize
-
super(0, :ping_error)
-
end
-
end
-
-
30
class GoawayError < Error
-
30
def initialize(code = :no_error)
-
66
super(0, code)
-
end
-
end
-
-
30
attr_reader :streams, :pending
-
-
30
def initialize(buffer, options)
-
4196
@options = options
-
4196
@settings = @options.http2_settings
-
4196
@pending = []
-
4196
@streams = {}
-
4196
@drains = {}
-
4196
@pings = []
-
4196
@buffer = buffer
-
4196
@handshake_completed = false
-
4196
@wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
-
4196
@max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
-
4196
@max_requests = @options.max_requests
-
4196
init_connection
-
end
-
-
30
def timeout
-
8274
return @options.timeout[:operation_timeout] if @handshake_completed
-
-
4144
@options.timeout[:settings_timeout]
-
end
-
-
30
def interests
-
120181
if @connection.state == :closed
-
10587
return unless @handshake_completed
-
-
10493
return if @buffer.empty?
-
-
# HTTP/2 GOAWAY frame buffered.
-
6128
return :w
-
end
-
-
109594
unless @connection.state == :connected && @handshake_completed
-
# HTTP/2 in intermediate state or still completing initialization-
-
29217
return @buffer.empty? ? :r : :rw
-
end
-
-
77792
unless @connection.send_buffer.empty?
-
# HTTP/2 connection is buffering data chunks and failing to emit DATA frames,
-
# most likely because the flow control window is exhausted.
-
15
return :rw unless @buffer.empty?
-
-
# waiting for WINDOW_UPDATE frames
-
3
return :r
-
end
-
-
# there are pending bufferable requests
-
77777
return :w if !@pending.empty? && can_buffer_more_requests?
-
-
# there are pending frames from the last run
-
77777
return :w unless @drains.empty?
-
-
73209
if @buffer.empty?
-
# skip if no more requests or pings to process
-
61317
return if @streams.empty? && @pings.empty?
-
-
57680
:r
-
else
-
# buffered frames
-
11892
:w
-
end
-
end
-
-
30
def close
-
3429
unless @connection.state == :closed
-
3411
@connection.goaway
-
3411
emit(:timeout, @options.timeout[:close_handshake_timeout])
-
end
-
3429
emit(:close)
-
end
-
-
30
def empty?
-
3433
@connection.state == :closed || @streams.empty?
-
end
-
-
30
def exhausted?
-
4028
!@max_requests.positive?
-
end
-
-
30
def <<(data)
-
42439
@connection << data
-
end
-
-
30
def send(request, head = false)
-
9284
unless can_buffer_more_requests?
-
4499
head ? @pending.unshift(request) : @pending << request
-
4499
return false
-
end
-
4785
unless (stream = @streams[request])
-
4785
stream = @connection.new_stream
-
4785
handle_stream(stream, request)
-
4314
@streams[request] = stream
-
4314
@max_requests -= 1
-
end
-
4785
handle(request, stream)
-
4767
true
-
rescue ::HTTP2::Error::StreamLimitExceeded
-
@pending.unshift(request)
-
false
-
end
-
-
30
def consume
-
34065
@streams.each do |request, stream|
-
12869
next unless request.can_buffer?
-
-
1297
handle(request, stream)
-
end
-
end
-
-
30
def handle_error(ex, request = nil)
-
504
if ex.is_a?(OperationTimeoutError) && !@handshake_completed && @connection.state != :closed
-
21
@connection.goaway(:settings_timeout, "closing due to settings timeout")
-
21
emit(:close_handshake)
-
21
settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
-
21
settings_ex.set_backtrace(ex.backtrace)
-
21
ex = settings_ex
-
end
-
1355
while (req, _ = @streams.shift)
-
503
next if request && request == req
-
-
114
emit(:error, req, ex)
-
end
-
934
while (req = @pending.shift)
-
30
next if request && request == req
-
-
30
emit(:error, req, ex)
-
end
-
end
-
-
30
def ping
-
31
ping = SecureRandom.gen_random(8)
-
31
@connection.ping(ping.dup)
-
ensure
-
31
@pings << ping
-
end
-
-
30
def waiting_for_ping?
-
505
@pings.any?
-
end
-
-
30
def reset_requests; end
-
-
30
private
-
-
30
def can_buffer_more_requests?
-
10225
(@handshake_completed || !@wait_for_handshake) &&
-
@streams.size < @max_concurrent_requests &&
-
@streams.size < @max_requests
-
end
-
-
30
def send_pending
-
11257
while (request = @pending.shift)
-
4321
break unless send(request, true)
-
end
-
end
-
-
30
def handle(request, stream)
-
6387
catch(:buffer_full) do
-
6387
request.transition(:headers)
-
6378
join_headers(stream, request) if request.state == :headers
-
6378
request.transition(:body)
-
6378
join_body(stream, request) if request.state == :body
-
4844
request.transition(:trailers)
-
4844
join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
-
4844
request.transition(:done)
-
end
-
end
-
-
30
def init_connection
-
4196
@connection = ::HTTP2::Client.new(@settings)
-
4196
@connection.on(:frame, &method(:on_frame))
-
4196
@connection.on(:frame_sent, &method(:on_frame_sent))
-
4196
@connection.on(:frame_received, &method(:on_frame_received))
-
4196
@connection.on(:origin, &method(:on_origin))
-
4196
@connection.on(:promise, &method(:on_promise))
-
4196
@connection.on(:altsvc) { |frame| on_altsvc(frame[:origin], frame) }
-
4196
@connection.on(:settings_ack, &method(:on_settings))
-
4196
@connection.on(:ack, &method(:on_pong))
-
4196
@connection.on(:goaway, &method(:on_close))
-
#
-
# Some servers initiate HTTP/2 negotiation right away, some don't.
-
# As such, we have to check the socket buffer. If there is something
-
# to read, the server initiated the negotiation. If not, we have to
-
# initiate it.
-
#
-
4196
@connection.send_connection_preface
-
end
-
-
30
alias_method :reset, :init_connection
-
30
public :reset
-
-
30
def handle_stream(stream, request)
-
4803
request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
-
4803
stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
-
9531
stream.on(:half_close) { on_stream_half_close(stream, request) }
-
4803
stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
-
4803
stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
-
4803
stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
-
end
-
-
30
def set_protocol_headers(request)
-
470
{
-
4305
":scheme" => request.scheme,
-
":method" => request.verb,
-
":path" => request.path,
-
":authority" => request.authority,
-
}
-
end
-
-
30
def join_headers(stream, request)
-
4776
extra_headers = set_protocol_headers(request)
-
-
4776
if request.headers.key?("host")
-
9
log { "forbidden \"host\" header found (#{log_redact_headers(request.headers["host"])}), will use it as authority..." }
-
8
extra_headers[":authority"] = request.headers["host"]
-
end
-
-
4776
log(level: 1, color: :yellow) do
-
146
"\n#{request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")}"
-
end
-
4776
stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
-
end
-
-
30
def join_trailers(stream, request)
-
1770
unless request.trailers?
-
1761
stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
-
1597
return
-
end
-
-
9
log(level: 1, color: :yellow) do
-
17
request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")
-
end
-
9
stream.headers(request.trailers.each, end_stream: true)
-
end
-
-
30
def join_body(stream, request)
-
6224
return if request.body.empty?
-
-
3309
chunk = @drains.delete(request) || request.drain_body
-
3507
while chunk
-
3589
next_chunk = request.drain_body
-
3589
send_chunk(request, stream, chunk, next_chunk)
-
-
3249
if next_chunk && (@buffer.full? || request.body.unbounded_body?)
-
1066
@drains[request] = next_chunk
-
1194
throw(:buffer_full)
-
end
-
-
2055
chunk = next_chunk
-
end
-
-
1775
return unless (error = request.drain_error)
-
-
28
on_stream_refuse(stream, request, error)
-
end
-
-
30
def send_chunk(request, stream, chunk, next_chunk)
-
3613
log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
-
3613
log(level: 2, color: :green) { "#{stream.id}: -> #{log_redact_body(chunk.inspect)}" }
-
3589
stream.data(chunk, end_stream: end_stream?(request, next_chunk))
-
end
-
-
30
def end_stream?(request, next_chunk)
-
3215
!(next_chunk || request.trailers? || request.callbacks_for?(:trailers))
-
end
-
-
######
-
# HTTP/2 Callbacks
-
######
-
-
30
def on_stream_headers(stream, request, h)
-
4513
response = request.response
-
-
4513
if response.is_a?(Response) && response.version == "2.0"
-
133
on_stream_trailers(stream, response, h)
-
133
return
-
end
-
-
4380
log(color: :yellow) do
-
146
h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{k == ":status" ? v : log_redact_headers(v)}" }.join("\n")
-
end
-
4380
_, status = h.shift
-
4380
headers = request.options.headers_class.new(h)
-
4380
response = request.options.response_class.new(request, status, "2.0", headers)
-
4380
request.response = response
-
3942
@streams[request] = stream
-
-
4371
handle(request, stream) if request.expects?
-
end
-
-
30
def on_stream_trailers(stream, response, h)
-
133
log(color: :yellow) do
-
h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")
-
end
-
133
response.merge_headers(h)
-
end
-
-
30
def on_stream_data(stream, request, data)
-
8417
log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
-
8417
log(level: 2, color: :green) { "#{stream.id}: <- #{log_redact_body(data.inspect)}" }
-
8391
request.response << data
-
end
-
-
30
def on_stream_refuse(stream, request, error)
-
28
on_stream_close(stream, request, error)
-
28
stream.close
-
end
-
-
30
def on_stream_half_close(stream, _request)
-
4728
unless stream.send_buffer.empty?
-
1
stream.send_buffer.clear
-
1
stream.data("", end_stream: true)
-
end
-
-
# TODO: omit log line if response already here
-
4744
log(level: 2) { "#{stream.id}: waiting for response..." }
-
end
-
-
30
def on_stream_close(stream, request, error)
-
4283
return if error == :stream_closed && !@streams.key?(request)
-
-
4271
log(level: 2) { "#{stream.id}: closing stream" }
-
4255
teardown(request)
-
-
4255
if error
-
28
case error
-
when :http_1_1_required
-
emit(:error, request, error)
-
else
-
28
ex = Error.new(stream.id, error)
-
28
ex.set_backtrace(caller)
-
28
response = ErrorResponse.new(request, ex)
-
28
request.response = response
-
28
emit(:response, request, response)
-
end
-
else
-
4227
response = request.response
-
4227
if response && response.is_a?(Response) && response.status == 421
-
9
emit(:error, request, :http_1_1_required)
-
else
-
4218
emit(:response, request, response)
-
end
-
end
-
4246
send(@pending.shift) unless @pending.empty?
-
-
4246
return unless @streams.empty? && exhausted?
-
-
9
if @pending.empty?
-
close
-
else
-
9
emit(:exhausted)
-
end
-
end
-
-
30
def on_frame(bytes)
-
25846
@buffer << bytes
-
end
-
-
30
def on_settings(*)
-
4130
@handshake_completed = true
-
4130
emit(:current_timeout)
-
4130
@max_concurrent_requests = [@max_concurrent_requests, @connection.remote_settings[:settings_max_concurrent_streams]].min
-
4130
send_pending
-
end
-
-
30
def on_close(_last_frame, error, _payload)
-
84
is_connection_closed = @connection.state == :closed
-
84
if error
-
84
@buffer.clear if is_connection_closed
-
75
case error
-
when :http_1_1_required
-
48
while (request = @pending.shift)
-
18
emit(:error, request, error)
-
end
-
else
-
66
ex = GoawayError.new(error)
-
66
ex.set_backtrace(caller)
-
-
66
handle_error(ex)
-
66
teardown
-
-
end
-
end
-
84
return unless is_connection_closed && @streams.empty?
-
-
84
emit(:close) if is_connection_closed
-
end
-
-
30
def on_frame_sent(frame)
-
21741
log(level: 2) { "#{frame[:stream]}: frame was sent!" }
-
21741
log(level: 2, color: :blue) { "#{frame[:stream]}: #{frame_with_extra_info(frame)}" }
-
end
-
-
30
def on_frame_received(frame)
-
22941
log(level: 2) { "#{frame[:stream]}: frame was received!" }
-
22941
log(level: 2, color: :magenta) { "#{frame[:stream]}: #{frame_with_extra_info(frame)}" }
-
end
-
-
30
def frame_with_extra_info(frame)
-
191
case frame[:type]
-
when :data
-
56
frame.merge(payload: frame[:payload].bytesize)
-
when :headers, :ping
-
45
frame.merge(payload: log_redact_headers(frame[:payload]))
-
when :window_update
-
connection_or_stream = if (id = frame[:stream]).zero?
-
@connection
-
else
-
@streams.each_value.find { |s| s.id == id }
-
end
-
if connection_or_stream
-
frame.merge(
-
local_window: connection_or_stream.local_window,
-
remote_window: connection_or_stream.remote_window,
-
buffered_amount: connection_or_stream.buffered_amount,
-
stream_state: connection_or_stream.state,
-
)
-
else
-
frame
-
end
-
else
-
90
frame
-
end.merge(connection_state: @connection.state)
-
end
-
-
30
def on_altsvc(origin, frame)
-
log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
-
log(level: 2) { "#{frame[:stream]}: #{log_redact_headers(frame.inspect)}" }
-
alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
-
params = { "ma" => frame[:max_age] }
-
emit(:altsvc, origin, alt_origin, origin, params)
-
end
-
-
30
def on_promise(stream)
-
27
emit(:promise, @streams.key(stream.parent), stream)
-
end
-
-
30
def on_origin(origin)
-
emit(:origin, origin)
-
end
-
-
30
def on_pong(ping)
-
18
raise PingError unless @pings.delete(ping.to_s)
-
-
18
emit(:pong)
-
end
-
-
30
def teardown(request = nil)
-
4321
if request
-
4255
@drains.delete(request)
-
4255
@streams.delete(request)
-
else
-
66
@drains.clear
-
66
@streams.clear
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
#
-
# domain_name.rb - Domain Name manipulation library for Ruby
-
#
-
# Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
-
#
-
# Redistribution and use in source and binary forms, with or without
-
# modification, are permitted provided that the following conditions
-
# are met:
-
# 1. Redistributions of source code must retain the above copyright
-
# notice, this list of conditions and the following disclaimer.
-
# 2. Redistributions in binary form must reproduce the above copyright
-
# notice, this list of conditions and the following disclaimer in the
-
# documentation and/or other materials provided with the distribution.
-
#
-
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-
# SUCH DAMAGE.
-
-
30
require "ipaddr"
-
-
30
module HTTPX
-
# Represents a domain name ready for extracting its registered domain
-
# and TLD.
-
30
class DomainName
-
30
include Comparable
-
-
# The full host name normalized, ASCII-ized and downcased using the
-
# Unicode NFC rules and the Punycode algorithm. If initialized with
-
# an IP address, the string representation of the IP address
-
# suitable for opening a connection to.
-
30
attr_reader :hostname
-
-
# The Unicode representation of the #hostname property.
-
#
-
# :attr_reader: hostname_idn
-
-
# The least "universally original" domain part of this domain name.
-
# For example, "example.co.uk" for "www.sub.example.co.uk". This
-
# may be nil if the hostname does not have one, like when it is an
-
# IP address, an effective TLD or higher itself, or of a
-
# non-canonical domain.
-
30
attr_reader :domain
-
-
30
class << self
-
30
def new(domain)
-
963
return domain if domain.is_a?(self)
-
-
891
super
-
end
-
-
# Normalizes a _domain_ using the Punycode algorithm as necessary.
-
# The result will be a downcased, ASCII-only string.
-
30
def normalize(domain)
-
855
unless domain.ascii_only?
-
domain = domain.chomp(".").unicode_normalize(:nfc)
-
domain = Punycode.encode_hostname(domain)
-
end
-
-
855
domain.downcase
-
end
-
end
-
-
# Parses _hostname_ into a DomainName object. An IP address is also
-
# accepted. An IPv6 address may be enclosed in square brackets.
-
30
def initialize(hostname)
-
891
hostname = String(hostname)
-
-
891
raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
-
-
98
begin
-
891
@ipaddr = IPAddr.new(hostname)
-
36
@hostname = @ipaddr.to_s
-
36
return
-
rescue IPAddr::Error
-
855
nil
-
end
-
-
855
@hostname = DomainName.normalize(hostname)
-
855
tld = if (last_dot = @hostname.rindex("."))
-
207
@hostname[(last_dot + 1)..-1]
-
else
-
648
@hostname
-
end
-
-
# unknown/local TLD
-
855
@domain = if last_dot
-
# fallback - accept cookies down to second level
-
# cf. http://www.dkim-reputation.org/regdom-libs/
-
207
if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
-
54
@hostname[(penultimate_dot + 1)..-1]
-
else
-
153
@hostname
-
end
-
else
-
# no domain part - must be a local hostname
-
648
tld
-
end
-
end
-
-
# Checks if the server represented by this domain is qualified to
-
# send and receive cookies with a domain attribute value of
-
# _domain_. A true value given as the second argument represents
-
# cookies without a domain attribute value, in which case only
-
# hostname equality is checked.
-
30
def cookie_domain?(domain, host_only = false)
-
# RFC 6265 #5.3
-
# When the user agent "receives a cookie":
-
36
return self == @domain if host_only
-
-
36
domain = DomainName.new(domain)
-
-
# RFC 6265 #5.1.3
-
# Do not perform subdomain matching against IP addresses.
-
36
@hostname == domain.hostname if @ipaddr
-
-
# RFC 6265 #4.1.1
-
# Domain-value must be a subdomain.
-
36
@domain && self <= domain && domain <= @domain
-
end
-
-
30
def <=>(other)
-
54
other = DomainName.new(other)
-
54
othername = other.hostname
-
54
if othername == @hostname
-
18
0
-
35
elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
-
# The other is higher
-
18
-1
-
else
-
# The other is lower
-
18
1
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
# the default exception class for exceptions raised by HTTPX.
-
30
class Error < StandardError; end
-
-
30
class UnsupportedSchemeError < Error; end
-
-
30
class ConnectionError < Error; end
-
-
# Error raised when there was a timeout. Its subclasses allow for finer-grained
-
# control of which timeout happened.
-
30
class TimeoutError < Error
-
# The timeout value which caused this error to be raised.
-
30
attr_reader :timeout
-
-
# initializes the timeout exception with the +timeout+ causing the error, and the
-
# error +message+ for it.
-
30
def initialize(timeout, message)
-
950
@timeout = timeout
-
950
super(message)
-
end
-
-
# clones this error into a HTTPX::ConnectionTimeoutError.
-
30
def to_connection_error
-
27
ex = ConnectTimeoutError.new(@timeout, message)
-
27
ex.set_backtrace(backtrace)
-
27
ex
-
end
-
end
-
-
# Raise when it can't acquire a connection from the pool.
-
30
class PoolTimeoutError < TimeoutError; end
-
-
# Error raised when there was a timeout establishing the connection to a server.
-
# This may be raised due to timeouts during TCP and TLS (when applicable) connection
-
# establishment.
-
30
class ConnectTimeoutError < TimeoutError; end
-
-
# Error raised when there was a timeout while sending a request, or receiving a response
-
# from the server.
-
30
class RequestTimeoutError < TimeoutError
-
# The HTTPX::Request request object this exception refers to.
-
30
attr_reader :request
-
-
# initializes the exception with the +request+ and +response+ it refers to, and the
-
# +timeout+ causing the error, and the
-
30
def initialize(request, response, timeout)
-
785
@request = request
-
785
@response = response
-
785
super(timeout, "Timed out after #{timeout} seconds")
-
end
-
-
30
def marshal_dump
-
[message]
-
end
-
end
-
-
# Error raised when there was a timeout while receiving a response from the server.
-
30
class ReadTimeoutError < RequestTimeoutError; end
-
-
# Error raised when there was a timeout while sending a request from the server.
-
30
class WriteTimeoutError < RequestTimeoutError; end
-
-
# Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
-
30
class SettingsTimeoutError < TimeoutError; end
-
-
# Error raised when there was a timeout while resolving a domain to an IP.
-
30
class ResolveTimeoutError < TimeoutError; end
-
-
# Error raise when there was a timeout waiting for readiness of the socket the request is related to.
-
30
class OperationTimeoutError < TimeoutError; end
-
-
# Error raised when there was an error while resolving a domain to an IP.
-
30
class ResolveError < Error; end
-
-
# Error raised when there was an error while resolving a domain to an IP
-
# using a HTTPX::Resolver::Native resolver.
-
30
class NativeResolveError < ResolveError
-
30
attr_reader :host
-
-
30
attr_accessor :connection
-
-
# initializes the exception with the +connection+ it refers to, the +host+ domain
-
# which failed to resolve, and the error +message+.
-
30
def initialize(connection, host, message = "Can't resolve #{host}")
-
155
@connection = connection
-
155
@host = host
-
155
super(message)
-
end
-
end
-
-
# The exception class for HTTP responses with 4xx or 5xx status.
-
30
class HTTPError < Error
-
# The HTTPX::Response response object this exception refers to.
-
30
attr_reader :response
-
-
# Creates the instance and assigns the HTTPX::Response +response+.
-
30
def initialize(response)
-
105
@response = response
-
105
super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
-
end
-
-
# The HTTP response status.
-
#
-
# error.status #=> 404
-
30
def status
-
18
@response.status
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "uri"
-
-
30
module HTTPX
-
30
module ArrayExtensions
-
30
module Intersect
-
refine Array do
-
# Ruby 3.1 backport
-
4
def intersect?(arr)
-
19
if size < arr.size
-
smaller = self
-
else
-
19
smaller, arr = arr, self
-
end
-
19
(arr & smaller).size > 0
-
end
-
29
end unless Array.method_defined?(:intersect?)
-
end
-
end
-
-
30
module URIExtensions
-
# uri 0.11 backport, ships with ruby 3.1
-
30
refine URI::Generic do
-
-
30
def non_ascii_hostname
-
840
@non_ascii_hostname
-
end
-
-
30
def non_ascii_hostname=(hostname)
-
36
@non_ascii_hostname = hostname
-
end
-
-
def authority
-
6475
return host if port == default_port
-
-
751
"#{host}:#{port}"
-
29
end unless URI::HTTP.method_defined?(:authority)
-
-
def origin
-
5260
"#{scheme}://#{authority}"
-
29
end unless URI::HTTP.method_defined?(:origin)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
class Headers
-
30
class << self
-
30
def new(headers = nil)
-
33305
return headers if headers.is_a?(self)
-
-
14416
super
-
end
-
end
-
-
30
def initialize(headers = nil)
-
14416
if headers.nil? || headers.empty?
-
2193
@headers = headers.to_h
-
1976
return
-
end
-
-
12223
@headers = {}
-
-
12223
headers.each do |field, value|
-
75820
field = downcased(field)
-
-
75820
value = array_value(value)
-
-
75820
current = @headers[field]
-
-
75820
if current.nil?
-
68329
@headers[field] = value
-
else
-
64
current.concat(value)
-
end
-
end
-
end
-
-
# cloned initialization
-
30
def initialize_clone(orig, **kwargs)
-
9
super
-
9
@headers = orig.instance_variable_get(:@headers).clone(**kwargs)
-
end
-
-
# dupped initialization
-
30
def initialize_dup(orig)
-
28187
super
-
28187
@headers = orig.instance_variable_get(:@headers).transform_values(&:dup)
-
end
-
-
# freezes the headers hash
-
30
def freeze
-
28124
@headers.each_value(&:freeze).freeze
-
28124
super
-
end
-
-
# merges headers with another header-quack.
-
# the merge rule is, if the header already exists,
-
# ignore what the +other+ headers has. Otherwise, set
-
#
-
30
def merge(other)
-
6140
headers = dup
-
6140
other.each do |field, value|
-
15545
headers[downcased(field)] = value
-
end
-
6140
headers
-
end
-
-
# returns the comma-separated values of the header field
-
# identified by +field+, or nil otherwise.
-
#
-
30
def [](field)
-
78053
a = @headers[downcased(field)] || return
-
32527
a.join(", ")
-
end
-
-
# sets +value+ (if not nil) as single value for the +field+ header.
-
#
-
30
def []=(field, value)
-
30320
return unless value
-
-
27607
@headers[downcased(field)] = array_value(value)
-
end
-
-
# deletes all values associated with +field+ header.
-
#
-
30
def delete(field)
-
388
canonical = downcased(field)
-
388
@headers.delete(canonical) if @headers.key?(canonical)
-
end
-
-
# adds additional +value+ to the existing, for header +field+.
-
#
-
30
def add(field, value)
-
1896
(@headers[downcased(field)] ||= []) << String(value)
-
end
-
-
# helper to be used when adding an header field as a value to another field
-
#
-
# h2_headers.add_header("vary", "accept-encoding")
-
# h2_headers["vary"] #=> "accept-encoding"
-
# h1_headers.add_header("vary", "accept-encoding")
-
# h1_headers["vary"] #=> "Accept-Encoding"
-
#
-
30
alias_method :add_header, :add
-
-
# returns the enumerable headers store in pairs of header field + the values in
-
# the comma-separated string format
-
#
-
30
def each(extra_headers = nil)
-
46376
return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
-
-
26026
@headers.each do |field, value|
-
95216
yield(field, value.join(", ")) unless value.empty?
-
end
-
-
6913
extra_headers.each do |field, value|
-
31703
yield(field, value) unless value.empty?
-
26025
end if extra_headers
-
end
-
-
30
def ==(other)
-
6772
other == to_hash
-
end
-
-
30
def empty?
-
414
@headers.empty?
-
end
-
-
# the headers store in Hash format
-
30
def to_hash
-
9032
Hash[to_a]
-
end
-
30
alias_method :to_h, :to_hash
-
-
# the headers store in array of pairs format
-
30
def to_a
-
9058
Array(each)
-
end
-
-
# headers as string
-
30
def to_s
-
2426
@headers.to_s
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id} " \
-
skipped
"#{to_hash.inspect}>"
-
skipped
end
-
skipped
# :nocov:
-
-
# this is internal API and doesn't abide to other public API
-
# guarantees, like downcasing strings.
-
# Please do not use this outside of core!
-
#
-
30
def key?(downcased_key)
-
95940
@headers.key?(downcased_key)
-
end
-
-
# returns the values for the +field+ header in array format.
-
# This method is more internal, and for this reason doesn't try
-
# to "correct" the user input, i.e. it doesn't downcase the key.
-
#
-
30
def get(field)
-
449
@headers[field] || EMPTY
-
end
-
-
30
private
-
-
30
def array_value(value)
-
106140
Array(value)
-
end
-
-
30
def downcased(field)
-
203444
String(field).downcase
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "socket"
-
30
require "httpx/io/udp"
-
30
require "httpx/io/tcp"
-
30
require "httpx/io/unix"
-
-
begin
-
30
require "httpx/io/ssl"
-
rescue LoadError
-
end
-
# frozen_string_literal: true
-
-
30
require "openssl"
-
-
30
module HTTPX
-
30
TLSError = OpenSSL::SSL::SSLError
-
-
30
class SSL < TCP
-
# rubocop:disable Style/MutableConstant
-
30
TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
-
# https://github.com/jruby/jruby-openssl/issues/284
-
# TODO: remove when dropping support for jruby-openssl < 0.15.4
-
30
TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby" && JOpenSSL::VERSION < "0.15.4"
-
# rubocop:enable Style/MutableConstant
-
30
TLS_OPTIONS.freeze
-
-
30
attr_writer :ssl_session
-
-
30
def initialize(_, _, options)
-
3763
super
-
-
3763
@ssl_session = nil
-
3763
ctx_options = TLS_OPTIONS.merge(options.ssl)
-
3763
@sni_hostname = ctx_options.delete(:hostname) || @hostname
-
-
3763
if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
-
# externally initiated ssl socket
-
25
@ctx = @io.context
-
25
@state = :negotiated
-
else
-
3738
@ctx = OpenSSL::SSL::SSLContext.new
-
3738
@ctx.set_params(ctx_options) unless ctx_options.empty?
-
3738
unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
-
3379
@ctx.session_cache_mode =
-
OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
-
end
-
-
3738
yield(self) if block_given?
-
end
-
-
3763
@verify_hostname = @ctx.verify_hostname
-
end
-
-
30
if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
-
29
def session_new_cb(&pr)
-
9794
@ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
-
end
-
else
-
# session_new_cb not implemented under JRuby
-
1
def session_new_cb; end
-
end
-
-
30
def protocol
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
3779
@io.alpn_protocol || super
-
rescue StandardError
-
10
super
-
end
-
-
30
if RUBY_ENGINE == "jruby"
-
# in jruby, alpn_protocol may return ""
-
# https://github.com/jruby/jruby-openssl/issues/287
-
1
def protocol
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
405
proto = @io.alpn_protocol
-
-
404
return super if proto.nil? || proto.empty?
-
-
398
proto
-
rescue StandardError
-
1
super
-
end
-
end
-
-
30
def can_verify_peer?
-
29
@ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
-
end
-
-
30
def verify_hostname(host)
-
31
return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
31
return false if !@io.respond_to?(:peer_cert) || (peer_cert = @io.peer_cert).nil?
-
-
31
OpenSSL::SSL.verify_certificate_identity(peer_cert, host)
-
end
-
-
30
def connected?
-
16528
@state == :negotiated
-
end
-
-
30
def ssl_session_expired?
-
4141
ssl_session = @ssl_session
-
-
4141
ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (ssl_session.time.to_f + ssl_session.timeout)
-
end
-
-
30
def connect
-
16592
return if @state == :negotiated
-
-
16592
unless @state == :connected
-
9107
super
-
9070
return unless @state == :connected
-
end
-
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
-
11510
unless @io.is_a?(OpenSSL::SSL::SSLSocket)
-
4141
if (hostname_is_ip = (@ip == @sni_hostname)) && @ctx.verify_hostname
-
# IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
-
45
@sni_hostname = @ip.to_string
-
# IP addresses in SNI is not valid per RFC 6066, section 3.
-
45
@ctx.verify_hostname = false
-
end
-
-
4141
ssl = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
-
-
4141
ssl.hostname = @sni_hostname unless hostname_is_ip
-
4141
ssl.session = @ssl_session unless ssl_session_expired?
-
4141
ssl.sync_close = true
-
-
4141
@io = ssl
-
end
-
11510
try_ssl_connect
-
end
-
-
30
def try_ssl_connect
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
11510
ret = @io.connect_nonblock(exception: false)
-
11531
log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
-
10689
case ret
-
when :wait_readable
-
7396
@interests = :r
-
7396
return
-
when :wait_writable
-
@interests = :w
-
return
-
end
-
4088
@io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
-
4087
transition(:negotiated)
-
4087
@interests = :w
-
end
-
-
30
private
-
-
30
def transition(nextstate)
-
14587
case nextstate
-
when :negotiated
-
4087
return unless @state == :connected
-
-
when :closed
-
4011
return unless @state == :negotiated ||
-
@state == :connected
-
end
-
16213
do_transition(nextstate)
-
end
-
-
30
def log_transition_state(nextstate)
-
79
return super unless nextstate == :negotiated
-
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
-
18
server_cert = @io.peer_cert #: OpenSSL::X509::Certificate
-
-
16
"#{super}\n\n" \
-
2
"SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
-
2
"ALPN, server accepted to use #{protocol}\n" \
-
"Server certificate:\n " \
-
2
"subject: #{log_redact(server_cert.subject)}\n " \
-
2
"start date: #{log_redact(server_cert.not_before)}\n " \
-
2
"expire date: #{log_redact(server_cert.not_after)}\n " \
-
2
"issuer: #{log_redact(server_cert.issuer)}\n " \
-
"SSL certificate verify ok."
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "resolv"
-
-
30
module HTTPX
-
30
class TCP
-
30
include Loggable
-
-
30
using URIExtensions
-
-
30
attr_reader :ip, :port, :addresses, :state, :interests
-
-
30
alias_method :host, :ip
-
-
30
def initialize(origin, addresses, options)
-
8843
@state = :idle
-
8843
@keep_open = false
-
8843
@addresses = []
-
8843
@ip_index = -1
-
8843
@ip = nil
-
8843
@hostname = origin.host
-
8843
@options = options
-
8843
@fallback_protocol = @options.fallback_protocol
-
8843
@port = origin.port
-
8843
@interests = :w
-
8843
if (io = @options.io)
-
10
io =
-
49
case io
-
when Hash
-
18
io[origin.authority]
-
else
-
41
io
-
end
-
59
raise Error, "Given IO objects do not match the request authority" unless io
-
-
# @type var io: TCPSocket | OpenSSL::SSL::SSLSocket
-
-
59
_, _, _, ip = io.addr
-
59
@io = io
-
59
@addresses << (@ip = Resolver::Entry.new(ip))
-
59
@keep_open = true
-
59
@state = :connected
-
else
-
8784
add_addresses(addresses)
-
end
-
8843
@ip_index = @addresses.size - 1
-
end
-
-
30
def socket
-
219
@io
-
end
-
-
30
def add_addresses(addrs)
-
9268
return if addrs.empty?
-
-
9259
ip_index = @ip_index || (@addresses.size - 1)
-
9259
if addrs.first.ipv6?
-
# should be the next in line
-
422
@addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
-
else
-
8837
@addresses.unshift(*addrs)
-
end
-
8355
@ip_index += addrs.size
-
end
-
-
# eliminates expired entries and returns whether there are still any left.
-
30
def addresses?
-
1102
prev_addr_size = @addresses.size
-
-
1102
@addresses.delete_if(&:expired?).sort! do |addr1, addr2|
-
3250
if addr1.ipv6?
-
4
addr2.ipv6? ? 0 : 1
-
else
-
3246
addr2.ipv6? ? -1 : 0
-
end
-
end
-
-
1102
@ip_index = @addresses.size - 1 if prev_addr_size != @addresses.size
-
-
1102
@addresses.any?
-
end
-
-
30
def to_io
-
58797
@io.to_io
-
end
-
-
30
def protocol
-
5752
@fallback_protocol
-
end
-
-
30
def connect
-
22131
return unless closed?
-
-
21979
if @addresses.empty?
-
# an idle connection trying to connect with no available addresses is a connection
-
# out of the initial context which is back to the DNS resolution loop. This may
-
# happen in a fiber-aware context where a connection reconnects with expired addresses,
-
# and context is passed back to a fiber on the same connection while waiting for the
-
# DNS answer.
-
log { "tried connecting while resolving, skipping..." }
-
-
return
-
end
-
-
21979
if !@io || @io.closed?
-
9966
transition(:idle)
-
9966
@io = build_socket
-
end
-
21979
try_connect
-
rescue Errno::EHOSTUNREACH,
-
Errno::ENETUNREACH => e
-
14
@ip_index -= 1
-
-
14
raise e if @ip_index.negative?
-
-
7
log { "failed connecting to #{@ip} (#{e.message}), evict from cache and trying next..." }
-
7
@options.resolver_cache.evict(@hostname, @ip)
-
-
7
@io = build_socket
-
7
retry
-
rescue Errno::ECONNREFUSED,
-
Errno::EADDRNOTAVAIL,
-
SocketError,
-
IOError => e
-
1083
@ip_index -= 1
-
-
1106
raise e if @ip_index.negative?
-
-
1042
log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
-
1028
@io = build_socket
-
1028
retry
-
rescue Errno::ETIMEDOUT => e
-
@ip_index -= 1
-
-
raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index.negative?
-
-
log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
-
-
@io = build_socket
-
retry
-
end
-
-
30
def try_connect
-
21979
ret = @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
-
21012
log(level: 3, color: :cyan) { "TCP CONNECT: #{ret}..." }
-
18833
case ret
-
when :wait_readable
-
@interests = :r
-
return
-
when :wait_writable
-
10987
@interests = :w
-
10987
return
-
end
-
9853
transition(:connected)
-
9853
@interests = :w
-
rescue Errno::EALREADY
-
19
@interests = :w
-
end
-
30
private :try_connect
-
-
30
def read(size, buffer)
-
68841
ret = @io.read_nonblock(size, buffer, exception: false)
-
68797
if ret == :wait_readable
-
17380
buffer.clear
-
16170
return 0
-
end
-
51417
return if ret.nil?
-
-
51522
log { "READ: #{buffer.bytesize} bytes..." }
-
51404
buffer.bytesize
-
end
-
-
30
def write(buffer)
-
23608
siz = @io.write_nonblock(buffer, exception: false)
-
23602
return 0 if siz == :wait_writable
-
23584
return if siz.nil?
-
-
23692
log { "WRITE: #{siz} bytes..." }
-
-
23584
buffer.shift!(siz)
-
23584
siz
-
end
-
-
30
def close
-
11393
return if @keep_open || closed?
-
-
981
begin
-
9746
@io.close
-
rescue StandardError => e
-
log { "error closing socket" }
-
log { e.full_message(highlight: false) }
-
ensure
-
9746
transition(:closed)
-
end
-
end
-
-
30
def connected?
-
11957
@state == :connected
-
end
-
-
30
def closed?
-
33602
@state == :idle || @state == :closed
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id} " \
-
skipped
"#{@ip}:#{@port} " \
-
skipped
"@state=#{@state} " \
-
skipped
"@hostname=#{@hostname} " \
-
skipped
"@addresses=#{@addresses} " \
-
skipped
"@state=#{@state}>"
-
skipped
end
-
skipped
# :nocov:
-
-
30
private
-
-
30
def build_socket
-
11001
@ip = @addresses[@ip_index]
-
11001
Socket.new(@ip.family, :STREAM, 0)
-
end
-
-
30
def transition(nextstate)
-
15692
case nextstate
-
# when :idle
-
when :connected
-
5844
return unless @state == :idle
-
when :closed
-
5735
return unless @state == :connected
-
end
-
17463
do_transition(nextstate)
-
end
-
-
30
def do_transition(nextstate)
-
33924
log(level: 1) { log_transition_state(nextstate) }
-
33676
@state = nextstate
-
end
-
-
30
def log_transition_state(nextstate)
-
248
label = host
-
248
label = "#{label}(##{@io.fileno})" if nextstate == :connected
-
222
"#{label} #{@state} -> #{nextstate}"
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "ipaddr"
-
-
30
module HTTPX
-
30
class UDP
-
30
include Loggable
-
-
30
def initialize(ip, port, options)
-
534
@host = ip
-
534
@port = port
-
534
@io = UDPSocket.new(IPAddr.new(ip).family)
-
534
@options = options
-
end
-
-
30
def to_io
-
1384
@io.to_io
-
end
-
-
30
def connect; end
-
-
30
def connected?
-
534
true
-
end
-
-
30
def close
-
537
@io.close
-
end
-
-
30
if RUBY_ENGINE == "jruby"
-
# In JRuby, sendmsg_nonblock is not implemented
-
1
def write(buffer)
-
54
siz = @io.send(buffer.to_s, 0, @host, @port)
-
54
log { "WRITE: #{siz} bytes..." }
-
54
buffer.shift!(siz)
-
54
siz
-
end
-
else
-
29
def write(buffer)
-
771
siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
-
771
return 0 if siz == :wait_writable
-
771
return if siz.nil?
-
-
771
log { "WRITE: #{siz} bytes..." }
-
-
771
buffer.shift!(siz)
-
771
siz
-
end
-
end
-
-
30
def read(size, buffer)
-
1514
ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
-
1514
return 0 if ret == :wait_readable
-
762
return if ret.nil?
-
-
762
log { "READ: #{buffer.bytesize} bytes..." }
-
-
762
buffer.bytesize
-
rescue IOError
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
class UNIX < TCP
-
30
using URIExtensions
-
-
30
attr_reader :path
-
-
30
alias_method :host, :path
-
-
30
def initialize(origin, path, options)
-
32
@addresses = []
-
32
@hostname = origin.host
-
32
@state = :idle
-
32
@options = options
-
32
@fallback_protocol = @options.fallback_protocol
-
32
if (io = @options.io)
-
2
io =
-
14
case io
-
when Hash
-
8
io[origin.authority]
-
else
-
8
io
-
end
-
16
raise Error, "Given IO objects do not match the request authority" unless io
-
-
# @type var io: UNIXSocket
-
-
16
_, @path = io.addr
-
16
@io = io
-
16
@keep_open = true
-
16
@state = :connected
-
16
elsif path
-
16
@path = path
-
else
-
raise Error, "No path given where to store the socket"
-
end
-
32
@io ||= build_socket
-
end
-
-
30
def connect
-
24
return unless closed?
-
-
begin
-
24
if @io.closed?
-
8
transition(:idle)
-
8
@io = build_socket
-
end
-
24
@io.connect_nonblock(Socket.sockaddr_un(@path))
-
rescue Errno::EISCONN
-
end
-
16
transition(:connected)
-
rescue Errno::EINPROGRESS,
-
Errno::EALREADY,
-
IO::WaitReadable
-
end
-
-
# the path is always explicitly passed, so no point in resolving.
-
30
def addresses?
-
16
true
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id} @path=#{@path}) @state=#{@state})>"
-
skipped
end
-
skipped
# :nocov:
-
-
30
private
-
-
30
def build_socket
-
24
Socket.new(Socket::PF_UNIX, :STREAM, 0)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "fiber" if RUBY_VERSION < "3.0.0"
-
-
30
module HTTPX
-
30
module Loggable
-
30
COLORS = {
-
black: 30,
-
red: 31,
-
green: 32,
-
yellow: 33,
-
blue: 34,
-
magenta: 35,
-
cyan: 36,
-
white: 37,
-
}.freeze
-
-
30
USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
-
-
30
def log(
-
level: @options.debug_level,
-
color: nil,
-
debug_level: @options.debug_level,
-
debug: @options.debug,
-
&msg
-
)
-
935005
return unless debug_level >= level
-
-
362221
debug_stream = debug || ($stderr if USE_DEBUG_LOG)
-
-
362221
return unless debug_stream
-
-
4979
klass = self.class
-
-
10933
until (class_name = klass.name)
-
2370
klass = klass.superclass
-
end
-
-
4979
message = +"(time:#{Time.now.utc}, pid:#{Process.pid}, " \
-
491
"tid:#{Thread.current.object_id}, " \
-
491
"fid:#{Fiber.current.object_id}, " \
-
491
"self:#{class_name}##{object_id}) "
-
4979
message << msg.call << "\n"
-
4979
message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
-
4979
debug_stream << message
-
end
-
-
30
def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
-
1922
log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
-
end
-
-
30
def log_redact_headers(text)
-
1026
log_redact(text, @options.debug_redact == :headers)
-
end
-
-
30
def log_redact_body(text)
-
110
log_redact(text, @options.debug_redact == :body)
-
end
-
-
30
def log_redact(text, should_redact = nil)
-
1208
should_redact ||= @options.debug_redact == true
-
-
1208
return text.to_s unless should_redact
-
-
252
"[REDACTED]"
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
# Contains a set of options which are passed and shared across from session to its requests or
-
# responses.
-
30
class Options
-
30
BUFFER_SIZE = 1 << 14
-
30
WINDOW_SIZE = 1 << 14 # 16K
-
30
MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
-
30
KEEP_ALIVE_TIMEOUT = 20
-
30
SETTINGS_TIMEOUT = 10
-
30
CLOSE_HANDSHAKE_TIMEOUT = 10
-
30
CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
-
30
REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
-
30
RESOLVER_TYPES = %i[memory file].freeze
-
-
# default value used for "user-agent" header, when not overridden.
-
30
USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
-
-
30
@options_names = []
-
-
30
class << self
-
30
attr_reader :options_names
-
-
30
def inherited(klass)
-
49
super
-
49
klass.instance_variable_set(:@options_names, @options_names.dup)
-
end
-
-
30
def new(options = {})
-
# let enhanced options go through
-
16233
return options if self == Options && options.class < self
-
11646
return options if options.is_a?(self)
-
-
6807
super
-
end
-
-
30
def freeze
-
17946
@options_names.freeze
-
17946
super
-
end
-
-
30
def method_added(meth)
-
36923
super
-
-
36923
return unless meth =~ /^option_(.+)$/
-
-
15850
optname = Regexp.last_match(1) #: String
-
-
15850
if optname =~ /^(.+[^_])_+with/
-
# ignore alias method chain generated methods.
-
# this is the case with RBS runtime tests.
-
# it relies on the "_with/_without" separator, which is the most used convention,
-
# however it shouldn't be used in practice in httpx given the plugin architecture
-
# as the main extension API.
-
orig_name = Regexp.last_match(1) #: String
-
-
return if @options_names.include?(orig_name.to_sym)
-
end
-
-
15850
optname = optname.to_sym
-
-
15850
attr_reader(optname) unless method_defined?(optname)
-
-
15850
@options_names << optname unless @options_names.include?(optname)
-
end
-
end
-
-
# creates a new options instance from a given hash, which optionally define the following:
-
#
-
# :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
-
# :debug_level :: the log level of messages (can be 1, 2, or 3).
-
# :debug_redact :: whether header/body payload should be redacted (defaults to <tt>false</tt>).
-
# :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::SSL)
-
# :http2_settings :: a hash of options to be passed to a HTTP2::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
-
# :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
-
# like ALPN (defaults to <tt>"http/1.1"</tt>)
-
# :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
-
# :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
-
# :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
-
# :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
-
# <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>, <tt>:read_timeout</tt>, <tt>:write_timeout</tt>
-
# and <tt>:request_timeout</tt>
-
# :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
-
# :window_size :: number of bytes to read from a socket
-
# :buffer_size :: internal read and write buffer size in bytes
-
# :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
-
# :request_class :: class used to instantiate a request
-
# :response_class :: class used to instantiate a response
-
# :headers_class :: class used to instantiate headers
-
# :request_body_class :: class used to instantiate a request body
-
# :response_body_class :: class used to instantiate a response body
-
# :connection_class :: class used to instantiate connections
-
# :http1_class :: class used to manage HTTP1 sessions
-
# :http2_class :: class used to imanage HTTP2 sessions
-
# :resolver_native_class :: class used to resolve names using pure ruby DNS implementation
-
# :resolver_system_class :: class used to resolve names using system-based (getaddrinfo) name resolution
-
# :resolver_https_class :: class used to resolve names using DoH
-
# :pool_class :: class used to instantiate the session connection pool
-
# :options_class :: class used to instantiate options
-
# :transport :: type of transport to use (set to "unix" for UNIX sockets)
-
# :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
-
# paths should be used for UNIX sockets instead)
-
# :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
-
# :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
-
# :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
-
# using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class inheriting
-
# from HTTPX::Resolver::Resolver)
-
# :resolver_cache :: strategy to cache DNS results, ignored by the <tt>:system</tt> resolver, can be set to <tt>:memory<tt>
-
# or an instance of a custom class inheriting from HTTPX::Resolver::Cache::Base
-
# :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
-
# :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
-
# :ip_families :: which socket families are supported (system-dependent)
-
# :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
-
# :base_path :: path to prefix given relative paths with (ex: "/v2")
-
# :max_concurrent_requests :: max number of requests which can be set concurrently
-
# :max_requests :: max number of requests which can be made on socket before it reconnects.
-
# :close_on_fork :: whether the session automatically closes when the process is fork (defaults to <tt>false</tt>).
-
# it only works if the session is persistent (and ruby 3.1 or higher is used).
-
#
-
# This list of options are enhanced with each loaded plugin, see the plugin docs for details.
-
30
def initialize(options = EMPTY_HASH)
-
6807
options_names = self.class.options_names
-
-
1356
defaults =
-
5451
case options
-
when Options
-
4980
unknown_options = options.class.options_names - options_names
-
-
4980
raise Error, "unknown option: #{unknown_options.first}" unless unknown_options.empty?
-
-
4980
DEFAULT_OPTIONS.merge(options)
-
else
-
1827
options.each_key do |k|
-
12126
raise Error, "unknown option: #{k}" unless options_names.include?(k)
-
end
-
-
1818
options.empty? ? DEFAULT_OPTIONS : DEFAULT_OPTIONS.merge(options)
-
end
-
-
6798
options_names.each do |k|
-
299614
v = defaults[k]
-
-
299614
if v.nil?
-
73334
instance_variable_set(:"@#{k}", v)
-
-
73334
next
-
end
-
-
226280
value = __send__(:"option_#{k}", v)
-
226236
instance_variable_set(:"@#{k}", value)
-
end
-
-
6754
do_initialize
-
6754
freeze
-
end
-
-
# returns the class with which to instantiate the DNS resolver.
-
30
def resolver_class
-
19008
case @resolver_class
-
when Symbol
-
16341
public_send(:"resolver_#{@resolver_class}_class")
-
else
-
4516
@resolver_class
-
end
-
end
-
-
30
def resolver_cache
-
19619
cache_type = @resolver_cache
-
-
17766
case cache_type
-
when :memory
-
15316
Resolver::Cache::Memory.cache(cache_type)
-
when :file
-
Resolver::Cache::File.cache(cache_type)
-
else
-
4303
unless cache_type.respond_to?(:resolve) &&
-
cache_type.respond_to?(:get) &&
-
cache_type.respond_to?(:set) &&
-
cache_type.respond_to?(:evict)
-
raise TypeError, ":resolver_cache must be a compatible resolver cache and implement #get, #set and #evict"
-
end
-
-
4303
cache_type #: Object & Resolver::_Cache
-
end
-
end
-
-
30
def freeze
-
17885
self.class.options_names.each do |ivar|
-
# avoid freezing debug option, as when it's set, it's usually an
-
# object which cannot be frozen, like stderr or stdout. It's a
-
# documented exception then, and still does not defeat the purpose
-
# here, which is to make option objects shareable across ractors,
-
# and in most cases debug should be nil, or one of the objects
-
# which will eventually be shareable, like STDOUT or STDERR.
-
789808
next if ivar == :debug
-
-
771923
instance_variable_get(:"@#{ivar}").freeze
-
end
-
17885
super
-
end
-
-
30
REQUEST_BODY_IVARS = %i[@headers].freeze
-
-
# checks whether +other+ matches the same connection-level options
-
30
def connection_options_match?(other, ignore_ivars = nil)
-
2860
return true if self == other
-
-
# headers and other request options do not play a role, as they are
-
# relevant only for the request.
-
581
ivars = instance_variables
-
26014
ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
-
941
ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
-
-
581
other_ivars = other.instance_variables
-
26014
other_ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
-
941
other_ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
-
-
581
return false if ivars.size != other_ivars.size
-
-
581
return false if ivars.sort != other_ivars.sort
-
-
581
ivars.all? do |ivar|
-
23543
instance_variable_get(ivar) == other.instance_variable_get(ivar)
-
end
-
end
-
-
30
RESOLVER_IVARS = %i[
-
@resolver_class @resolver_cache @resolver_options
-
@resolver_native_class @resolver_system_class @resolver_https_class
-
].freeze
-
-
# checks whether +other+ matches the same resolver-level options
-
30
def resolver_options_match?(other)
-
327
self == other ||
-
RESOLVER_IVARS.all? do |ivar|
-
891
instance_variable_get(ivar) == other.instance_variable_get(ivar)
-
end
-
end
-
-
# returns a HTTPX::Options instance resulting of the merging of +other+ with self.
-
# it may return self if +other+ is self or equal to self.
-
30
def merge(other)
-
47406
if (is_options = other.is_a?(Options))
-
-
14373
return self if eql?(other)
-
-
3967
opts_names = other.class.options_names
-
-
132545
return self if opts_names.all? { |opt| public_send(opt) == other.public_send(opt) }
-
-
3696
other_opts = opts_names
-
else
-
33033
other_opts = other # : Hash[Symbol, untyped]
-
33033
other_opts = Hash[other] unless other.is_a?(Hash)
-
-
33025
return self if other_opts.empty?
-
-
31493
return self if other_opts.all? { |opt, v| !respond_to?(opt) || public_send(opt) == v }
-
end
-
-
17929
opts = dup
-
-
17929
other_opts.each do |opt, v|
-
192240
next unless respond_to?(opt)
-
-
192240
v = other.public_send(opt) if is_options
-
192240
ivar = :"@#{opt}"
-
-
192240
unless v
-
50792
opts.instance_variable_set(ivar, v)
-
50792
next
-
end
-
-
141448
v = opts.__send__(:"option_#{opt}", v)
-
-
141430
orig_v = public_send(opt)
-
-
141430
v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
-
-
141430
opts.instance_variable_set(ivar, v)
-
end
-
-
17911
opts
-
end
-
-
30
def to_hash
-
5505
instance_variables.each_with_object({}) do |ivar, hs|
-
232484
val = instance_variable_get(ivar)
-
-
232484
next if val.nil?
-
-
165877
hs[ivar[1..-1].to_sym] = val
-
end
-
end
-
-
30
def extend_with_plugin_classes(pl)
-
# extend request class
-
11083
if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
-
4135
@request_class = @request_class.dup
-
4135
SET_TEMPORARY_NAME[@request_class, pl]
-
4135
@request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
-
4135
@request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
-
end
-
# extend response class
-
11083
if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
-
3467
@response_class = @response_class.dup
-
3467
SET_TEMPORARY_NAME[@response_class, pl]
-
3467
@response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
-
3467
@response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
-
end
-
# extend headers class
-
11083
if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
-
234
@headers_class = @headers_class.dup
-
234
SET_TEMPORARY_NAME[@headers_class, pl]
-
234
@headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
-
234
@headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
-
end
-
# extend request body class
-
11083
if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
-
451
@request_body_class = @request_body_class.dup
-
451
SET_TEMPORARY_NAME[@request_body_class, pl]
-
451
@request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
-
451
@request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
-
end
-
# extend response body class
-
11083
if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
-
1272
@response_body_class = @response_body_class.dup
-
1272
SET_TEMPORARY_NAME[@response_body_class, pl]
-
1272
@response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
-
1272
@response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
-
end
-
# extend connection pool class
-
11083
if defined?(pl::PoolMethods)
-
983
@pool_class = @pool_class.dup
-
983
SET_TEMPORARY_NAME[@pool_class, pl]
-
983
@pool_class.__send__(:include, pl::PoolMethods)
-
end
-
# extend connection class
-
11083
if defined?(pl::ConnectionMethods)
-
4783
@connection_class = @connection_class.dup
-
4783
SET_TEMPORARY_NAME[@connection_class, pl]
-
4783
@connection_class.__send__(:include, pl::ConnectionMethods)
-
end
-
# extend http1 class
-
11083
if defined?(pl::HTTP1Methods)
-
635
@http1_class = @http1_class.dup
-
635
SET_TEMPORARY_NAME[@http1_class, pl]
-
635
@http1_class.__send__(:include, pl::HTTP1Methods)
-
end
-
# extend http2 class
-
11083
if defined?(pl::HTTP2Methods)
-
716
@http2_class = @http2_class.dup
-
716
SET_TEMPORARY_NAME[@http2_class, pl]
-
716
@http2_class.__send__(:include, pl::HTTP2Methods)
-
end
-
# extend native resolver class
-
11083
if defined?(pl::ResolverNativeMethods)
-
1719
@resolver_native_class = @resolver_native_class.dup
-
1719
SET_TEMPORARY_NAME[@resolver_native_class, pl]
-
1719
@resolver_native_class.__send__(:include, pl::ResolverNativeMethods)
-
end
-
# extend system resolver class
-
11083
if defined?(pl::ResolverSystemMethods)
-
740
@resolver_system_class = @resolver_system_class.dup
-
740
SET_TEMPORARY_NAME[@resolver_system_class, pl]
-
740
@resolver_system_class.__send__(:include, pl::ResolverSystemMethods)
-
end
-
# extend https resolver class
-
11083
if defined?(pl::ResolverHTTPSMethods)
-
123
@resolver_https_class = @resolver_https_class.dup
-
123
SET_TEMPORARY_NAME[@resolver_https_class, pl]
-
123
@resolver_https_class.__send__(:include, pl::ResolverHTTPSMethods)
-
end
-
-
11083
return unless defined?(pl::OptionsMethods)
-
-
# extend option class
-
# works around lack of initialize_dup callback
-
4980
@options_class = @options_class.dup
-
# (self.class.options_names)
-
4980
@options_class.__send__(:include, pl::OptionsMethods)
-
end
-
-
30
private
-
-
# number options
-
30
%i[
-
max_concurrent_requests max_requests window_size buffer_size
-
body_threshold_size debug_level
-
].each do |option|
-
180
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
6
# converts +v+ into an Integer before setting the +#{option}+ option.
-
6
private def option_#{option}(value) # private def option_max_requests(v)
-
value = Integer(value) unless value.respond_to?(:infinite?) && value.infinite?
-
6
raise TypeError, ":#{option} must be positive" unless value.positive? # raise TypeError, ":max_requests must be positive" unless value.positive?
-
-
value
-
end
-
OUT
-
end
-
-
# hashable options
-
30
%i[ssl http2_settings resolver_options pool_options].each do |option|
-
120
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
4
# converts +v+ into an Hash before setting the +#{option}+ option.
-
4
private def option_#{option}(value) # def option_ssl(v)
-
Hash[value]
-
end
-
OUT
-
end
-
-
30
%i[
-
request_class response_class headers_class request_body_class
-
response_body_class connection_class http1_class http2_class
-
resolver_native_class resolver_system_class resolver_https_class options_class pool_class
-
io fallback_protocol debug debug_redact
-
compress_request_body decompress_response_body
-
persistent close_on_fork
-
].each do |method_name|
-
630
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
21
# sets +v+ as the value of the +#{method_name}+ option
-
21
private def option_#{method_name}(v); v; end # private def option_smth(v); v; end
-
OUT
-
end
-
-
30
def option_origin(value)
-
786
URI(value)
-
end
-
-
30
def option_base_path(value)
-
36
String(value)
-
end
-
-
30
def option_headers(value)
-
11235
value = value.dup if value.frozen?
-
-
11235
headers_class.new(value)
-
end
-
-
30
def option_timeout(value)
-
12727
timeout_hash = Hash[value]
-
-
12727
default_timeouts = DEFAULT_OPTIONS[:timeout]
-
-
# Validate keys and values
-
12727
timeout_hash.each do |key, val|
-
87236
raise TypeError, "invalid timeout: :#{key}" unless default_timeouts.key?(key)
-
-
87227
next if val.nil?
-
-
67108
raise TypeError, ":#{key} must be numeric" unless val.is_a?(Numeric)
-
end
-
-
12709
timeout_hash
-
end
-
-
30
def option_supported_compression_formats(value)
-
10667
Array(value).map(&:to_s)
-
end
-
-
30
def option_transport(value)
-
56
transport = value.to_s
-
56
raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
-
-
56
transport
-
end
-
-
30
def option_addresses(value)
-
105
Array(value).map { |entry| Resolver::Entry.convert(entry) }
-
end
-
-
30
def option_ip_families(value)
-
210
Array(value)
-
end
-
-
30
def option_resolver_class(resolver_type)
-
10143
case resolver_type
-
when Symbol
-
7248
meth = :"resolver_#{resolver_type}_class"
-
-
7248
raise TypeError, ":resolver_class must be a supported type" unless respond_to?(meth)
-
-
7239
resolver_type
-
when Class
-
3915
raise TypeError, ":resolver_class must be a subclass of `#{Resolver::Resolver}`" unless resolver_type < Resolver::Resolver
-
-
3907
resolver_type
-
else
-
raise TypeError, ":resolver_class must be a supported type"
-
end
-
end
-
-
30
def option_resolver_cache(cache_type)
-
10622
if cache_type.is_a?(Symbol)
-
6800
raise TypeError, ":resolver_cache: #{cache_type} is invalid" unless RESOLVER_TYPES.include?(cache_type)
-
-
6800
require "httpx/resolver/cache/file" if cache_type == :file
-
-
else
-
3822
unless cache_type.respond_to?(:resolve) &&
-
cache_type.respond_to?(:get) &&
-
cache_type.respond_to?(:set) &&
-
cache_type.respond_to?(:evict)
-
raise TypeError, ":resolver_cache must be a compatible resolver cache and implement #resolve, #get, #set and #evict"
-
end
-
end
-
-
10622
cache_type
-
end
-
-
# called after all options are initialized
-
30
def do_initialize
-
6754
hs = @headers
-
-
# initialized default request headers
-
6754
hs["user-agent"] = USER_AGENT unless hs.key?("user-agent")
-
6754
hs["accept"] = "*/*" unless hs.key?("accept")
-
6754
if hs.key?("range")
-
9
hs.delete("accept-encoding")
-
else
-
6745
hs["accept-encoding"] = supported_compression_formats unless hs.key?("accept-encoding")
-
end
-
end
-
-
30
def access_option(obj, k, ivar_map)
-
case obj
-
when Hash
-
obj[ivar_map[k]]
-
else
-
obj.instance_variable_get(k)
-
end
-
end
-
-
# rubocop:disable Lint/UselessConstantScoping
-
# these really need to be defined at the end of the class
-
30
SET_TEMPORARY_NAME = ->(klass, pl = nil) do
-
20244
if klass.respond_to?(:set_temporary_name) # ruby 3.4 only
-
9153
name = klass.name || "#{klass.superclass.name}(plugin)"
-
9153
name = "#{name}/#{pl}" if pl
-
9153
klass.set_temporary_name(name)
-
end
-
end
-
-
2
DEFAULT_OPTIONS = {
-
28
:max_requests => Float::INFINITY,
-
:debug => nil,
-
30
:debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
-
:debug_redact => ENV.key?("HTTPX_DEBUG_REDACT"),
-
:ssl => EMPTY_HASH,
-
:http2_settings => { settings_enable_push: 0 }.freeze,
-
:fallback_protocol => "http/1.1",
-
:supported_compression_formats => %w[gzip deflate],
-
:decompress_response_body => true,
-
:compress_request_body => true,
-
:timeout => {
-
connect_timeout: CONNECT_TIMEOUT,
-
settings_timeout: SETTINGS_TIMEOUT,
-
close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
-
operation_timeout: OPERATION_TIMEOUT,
-
keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
-
read_timeout: READ_TIMEOUT,
-
write_timeout: WRITE_TIMEOUT,
-
request_timeout: REQUEST_TIMEOUT,
-
}.freeze,
-
:headers_class => Class.new(Headers, &SET_TEMPORARY_NAME),
-
:headers => EMPTY_HASH,
-
:window_size => WINDOW_SIZE,
-
:buffer_size => BUFFER_SIZE,
-
:body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
-
:request_class => Class.new(Request, &SET_TEMPORARY_NAME),
-
:response_class => Class.new(Response, &SET_TEMPORARY_NAME),
-
:request_body_class => Class.new(Request::Body, &SET_TEMPORARY_NAME),
-
:response_body_class => Class.new(Response::Body, &SET_TEMPORARY_NAME),
-
:pool_class => Class.new(Pool, &SET_TEMPORARY_NAME),
-
:connection_class => Class.new(Connection, &SET_TEMPORARY_NAME),
-
:http1_class => Class.new(Connection::HTTP1, &SET_TEMPORARY_NAME),
-
:http2_class => Class.new(Connection::HTTP2, &SET_TEMPORARY_NAME),
-
:resolver_native_class => Class.new(Resolver::Native, &SET_TEMPORARY_NAME),
-
:resolver_system_class => Class.new(Resolver::System, &SET_TEMPORARY_NAME),
-
:resolver_https_class => Class.new(Resolver::HTTPS, &SET_TEMPORARY_NAME),
-
:options_class => Class.new(self, &SET_TEMPORARY_NAME),
-
:transport => nil,
-
:addresses => nil,
-
:persistent => false,
-
30
:resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
-
30
:resolver_cache => (ENV["HTTPX_RESOLVER_CACHE"] || :memory).to_sym,
-
:resolver_options => { cache: true }.freeze,
-
:pool_options => EMPTY_HASH,
-
:ip_families => nil,
-
:close_on_fork => false,
-
}.each_value(&:freeze).freeze
-
# rubocop:enable Lint/UselessConstantScoping
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Parser
-
30
class Error < Error; end
-
-
30
class HTTP1
-
30
VERSIONS = %w[1.0 1.1].freeze
-
-
30
attr_reader :status_code, :http_version, :headers
-
-
30
def initialize(observer)
-
5852
@observer = observer
-
5852
@state = :idle
-
5852
@buffer = "".b
-
5852
@headers = {}
-
5852
@content_length = nil
-
5852
@_has_trailers = @upgrade = false
-
end
-
-
30
def <<(chunk)
-
8813
@buffer << chunk
-
8813
parse
-
end
-
-
30
def reset!
-
11460
@state = :idle
-
11460
@headers = {}
-
11460
@content_length = nil
-
11460
@_has_trailers = @upgrade = false
-
11460
@buffer = @buffer.to_s
-
end
-
-
30
def upgrade?
-
5612
@upgrade
-
end
-
-
30
def upgrade_data
-
37
@buffer.to_s
-
end
-
-
30
private
-
-
30
def parse
-
8813
loop do
-
18857
state = @state
-
17061
case @state
-
when :idle
-
6026
parse_headline
-
when :headers, :trailers
-
6125
parse_headers
-
when :data
-
6703
parse_data
-
end
-
13655
return if @buffer.empty? || state == @state
-
end
-
end
-
-
30
def parse_headline
-
#: @type ivar @buffer: String
-
6026
idx = @buffer.index("\n")
-
6026
return unless idx
-
-
6026
(m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
-
raise(Error, "wrong head line format")
-
6017
version, code, _ = m.captures
-
6017
raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
-
-
6008
@http_version = version.split(".").map(&:to_i)
-
6008
@status_code = code.to_i
-
6008
raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
-
-
5999
@buffer = @buffer.byteslice((idx + 1)..-1)
-
5999
nextstate(:headers)
-
end
-
-
30
def parse_headers
-
6128
headers = @headers
-
6128
buffer = @buffer
-
-
#: @type var buffer: String
-
-
48301
while (idx = buffer.index("\n"))
-
# @type var line: String
-
47090
line = buffer.byteslice(0..idx)
-
47090
raise Error, "wrong header format" if line.start_with?("\s", "\t")
-
-
47081
line.lstrip!
-
47081
buffer = @buffer = buffer.byteslice((idx + 1)..-1)
-
47081
if line.empty?
-
5434
case @state
-
when :headers
-
5981
prepare_data(headers)
-
5981
@observer.on_headers(headers)
-
5172
return unless @state == :headers
-
-
# state might have been reset
-
# in the :headers callback
-
5080
nextstate(:data)
-
5080
headers.clear
-
when :trailers
-
18
@observer.on_trailers(headers)
-
18
headers.clear
-
18
nextstate(:complete)
-
end
-
5089
return
-
end
-
41082
separator_index = line.index(":")
-
41082
raise Error, "wrong header format" unless separator_index
-
-
# @type var key: String
-
41073
key = line.byteslice(0..(separator_index - 1))
-
-
41073
key.rstrip! # was lstripped previously!
-
# @type var value: String
-
41073
value = line.byteslice((separator_index + 1)..-1)
-
41073
value.strip!
-
41073
raise Error, "wrong header format" if value.nil?
-
-
41073
(headers[key.downcase] ||= []) << value
-
end
-
end
-
-
30
def parse_data
-
6703
if @buffer.respond_to?(:each)
-
# @type ivar @buffer: Transcoder::Chunker::Decoder
-
221
@buffer.each do |chunk|
-
282
@observer.on_data(chunk)
-
end
-
6481
elsif @content_length
-
# @type ivar @buffer: String
-
6446
data = @buffer.byteslice(0, @content_length)
-
# @type var data: String
-
6446
@buffer = @buffer.byteslice(@content_length..-1) || "".b
-
5825
@content_length -= data.bytesize
-
6446
@observer.on_data(data)
-
6421
data.clear
-
else
-
# @type ivar @buffer: String
-
36
@observer.on_data(@buffer)
-
36
@buffer.clear
-
end
-
6669
return unless no_more_data?
-
-
4920
@buffer = @buffer.to_s
-
4920
if @_has_trailers
-
18
nextstate(:trailers)
-
else
-
4902
nextstate(:complete)
-
end
-
end
-
-
30
def prepare_data(headers)
-
5981
@upgrade = headers.key?("upgrade")
-
-
5981
@_has_trailers = headers.key?("trailer")
-
-
5981
if (tr_encodings = headers["transfer-encoding"])
-
165
tr_encodings.reverse_each do |tr_encoding|
-
165
tr_encoding.split(/ *, */).each do |encoding|
-
147
case encoding
-
when "chunked"
-
165
@buffer = Transcoder::Chunker::Decoder.new(@buffer.to_s, @_has_trailers)
-
end
-
end
-
end
-
else
-
5816
@content_length = headers["content-length"][0].to_i if headers.key?("content-length")
-
end
-
end
-
-
30
def no_more_data?
-
6669
if @content_length
-
6421
@content_length <= 0
-
247
elsif @buffer.respond_to?(:finished?)
-
# @type ivar @buffer: Transcoder::Chunker::Decoder
-
212
@buffer.finished?
-
else
-
36
false
-
end
-
end
-
-
30
def nextstate(state)
-
16017
@state = state
-
14517
case state
-
when :headers
-
5999
@observer.on_start
-
when :complete
-
4920
@observer.on_complete
-
606
reset!
-
606
nextstate(:idle) unless @buffer.empty?
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds a shim +authorization+ method to the session, which will fill
-
# the HTTP Authorization header, and another, +bearer_auth+, which fill the "Bearer " prefix
-
# in its value.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Auth#auth
-
#
-
23
module Auth
-
23
def self.subplugins
-
166
{
-
1331
retries: AuthRetries,
-
}
-
end
-
-
# adds support for the following options:
-
#
-
# :auth_header_value :: the token to use as a string, or a callable which returns a string when called.
-
# :auth_header_type :: the authentication type to use in the "authorization" header value (i.e. "Bearer", "Digest"...)
-
# :generate_auth_value_on_retry :: callable which returns whether the request should regenerate the auth_header_value
-
# when the request is retried (this option will only work if the session also loads the
-
# <tt>:retries</tt> plugin).
-
23
module OptionsMethods
-
23
def option_auth_header_value(value)
-
576
value
-
end
-
-
23
def option_auth_header_type(value)
-
504
value
-
end
-
-
23
def option_generate_auth_value_on_retry(value)
-
108
raise TypeError, "`:generate_auth_value_on_retry` must be a callable" unless value.respond_to?(:call)
-
-
108
value
-
end
-
end
-
-
23
module InstanceMethods
-
23
def initialize(*)
-
1698
super
-
-
1698
@auth_header_value = nil
-
1698
@auth_header_value_mtx = Thread::Mutex.new
-
1698
@skip_auth_header_value = false
-
end
-
-
23
def authorization(token = nil, auth_header_type: nil, &blk)
-
288
with(auth_header_type: auth_header_type, auth_header_value: token || blk)
-
end
-
-
23
def bearer_auth(token = nil, &blk)
-
18
authorization(token, auth_header_type: "Bearer", &blk)
-
end
-
-
23
def skip_auth_header
-
144
@skip_auth_header_value = true
-
144
yield
-
ensure
-
144
@skip_auth_header_value = false
-
end
-
-
23
def reset_auth_header_value!
-
18
@auth_header_value_mtx.synchronize do
-
18
@auth_header_value = nil
-
end
-
end
-
-
23
private
-
-
23
def send_request(request, *)
-
1612
return super if @skip_auth_header_value || request.authorized?
-
-
1282
auth_header_value = @auth_header_value_mtx.synchronize do
-
1282
@auth_header_value ||= generate_auth_token
-
end
-
-
1282
request.authorize(auth_header_value) if auth_header_value
-
-
1282
super
-
end
-
-
23
def generate_auth_token
-
580
return unless (auth_value = @options.auth_header_value)
-
-
360
auth_value = auth_value.call(self) if dynamic_auth_token?(auth_value)
-
-
360
auth_value
-
end
-
-
23
def dynamic_auth_token?(auth_header_value)
-
504
auth_header_value&.respond_to?(:call)
-
end
-
end
-
-
23
module RequestMethods
-
23
attr_reader :auth_token_value
-
-
23
def initialize(*)
-
1322
super
-
1322
@auth_token_value = @auth_header_value = nil
-
end
-
-
23
def authorized?
-
1468
!@auth_token_value.nil?
-
end
-
-
23
def unauthorize!
-
110
return unless (auth_value = @auth_header_value)
-
-
110
@headers.get("authorization").delete(auth_value)
-
-
110
@auth_token_value = @auth_header_value = nil
-
end
-
-
23
def authorize(auth_value)
-
1230
@auth_header_value = auth_value
-
1230
if (auth_type = @options.auth_header_type)
-
72
@auth_header_value = "#{auth_type} #{@auth_header_value}"
-
end
-
-
1230
@headers.add("authorization", @auth_header_value)
-
-
1230
@auth_token_value = auth_value
-
end
-
end
-
-
23
module AuthRetries
-
23
module InstanceMethods
-
23
private
-
-
23
def retryable_request?(request, response, options)
-
162
super || auth_error?(response, options)
-
end
-
-
23
def retryable_response?(response, options)
-
144
auth_error?(response, options) || super
-
end
-
-
23
def prepare_to_retry(request, response)
-
126
super
-
-
130
return unless auth_error?(response, request.options) ||
-
16
(@options.generate_auth_value_on_retry && @options.generate_auth_value_on_retry.call(response))
-
-
# regenerate token before retry, but only if it's the first request from batch failing.
-
# otherwise, it means that the first request already passed here, so this request should
-
# use whatever was generated for it.
-
108
@auth_header_value_mtx.synchronize do
-
108
@auth_header_value = generate_auth_token if request.auth_token_value == @auth_header_value
-
end
-
-
108
request.unauthorize!
-
end
-
-
23
def auth_error?(response, options)
-
288
response.is_a?(Response) && response.status == 401 && dynamic_auth_token?(options.auth_header_value)
-
end
-
end
-
end
-
end
-
23
register_plugin :auth, Auth
-
end
-
end
-
# frozen_string_literal: true
-
-
23
require "httpx/base64"
-
-
23
module HTTPX
-
23
module Plugins
-
23
module Authentication
-
23
class Basic
-
23
def initialize(user, password, **)
-
380
@user = user
-
380
@password = password
-
end
-
-
23
def authenticate(*)
-
359
"Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
23
require "time"
-
23
require "securerandom"
-
23
require "digest"
-
-
23
module HTTPX
-
23
module Plugins
-
23
module Authentication
-
23
class Digest
-
23
class Error < Error
-
end
-
-
23
def initialize(user, password, hashed: false, **)
-
198
@user = user
-
198
@password = password
-
198
@nonce = 0
-
198
@hashed = hashed
-
end
-
-
23
def can_authenticate?(authenticate)
-
180
authenticate && /Digest .*/.match?(authenticate)
-
end
-
-
23
def authenticate(request, authenticate)
-
180
"Digest #{generate_header(request.verb, request.path, authenticate)}"
-
end
-
-
23
private
-
-
23
def generate_header(meth, uri, authenticate)
-
# discard first token, it's Digest
-
180
auth_info = authenticate[/^(\w+) (.*)/, 2]
-
-
180
raise_format_error unless auth_info
-
-
180
s = StringScanner.new(auth_info)
-
-
180
params = {}
-
284
until s.eos?
-
936
k = s.scan_until(/=/)
-
936
raise_format_error unless k&.end_with?("=")
-
-
936
if s.peek(1) == "\""
-
720
s.skip("\"")
-
720
v = s.scan_until(/"/)
-
720
raise_format_error unless v&.end_with?("\"")
-
-
720
v = v[0..-2]
-
720
s.skip_until(/,/)
-
else
-
216
v = s.scan_until(/,|$/)
-
-
216
if v&.end_with?(",")
-
170
v = v[0..-2]
-
else
-
46
raise_format_error unless s.eos?
-
end
-
-
216
v = v[0..-2] if v&.end_with?(",")
-
end
-
832
params[k[0..-2]] = v
-
936
s.skip(/\s/)
-
end
-
-
180
nonce = params["nonce"]
-
180
nc = next_nonce
-
-
# verify qop
-
180
qop = params["qop"]
-
-
180
if qop
-
# some servers send multiple values wrapped in parentheses (i.e. "(qauth,)")
-
180
qop = qop[/\(?([^)]+)\)?/, 1]
-
360
qop = qop.split(",").map { |s| s.delete_prefix("'").delete_suffix("'") }.delete_if(&:empty?).map.first
-
end
-
-
180
if params["algorithm"] =~ /(.*?)(-sess)?$/
-
162
alg = Regexp.last_match(1)
-
162
algorithm = ::Digest.const_get(alg)
-
162
raise Error, "unknown algorithm \"#{alg}\"" unless algorithm
-
-
162
sess = Regexp.last_match(2)
-
else
-
18
algorithm = ::Digest::MD5
-
end
-
-
180
if qop || sess
-
180
cnonce = make_cnonce
-
180
nc = format("%<nonce>08x", nonce: nc)
-
end
-
-
180
a1 = if sess
-
4
[
-
36
(@hashed ? @password : algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}")),
-
nonce,
-
cnonce,
-
3
].join ":"
-
else
-
144
@hashed ? @password : "#{@user}:#{params["realm"]}:#{@password}"
-
end
-
-
180
ha1 = algorithm.hexdigest(a1)
-
180
ha2 = algorithm.hexdigest("#{meth}:#{uri}")
-
180
request_digest = [ha1, nonce]
-
180
request_digest.push(nc, cnonce, qop) if qop
-
180
request_digest << ha2
-
180
request_digest = request_digest.join(":")
-
-
40
header = [
-
160
%(username="#{@user}"),
-
20
%(nonce="#{nonce}"),
-
20
%(uri="#{uri}"),
-
20
%(response="#{algorithm.hexdigest(request_digest)}"),
-
]
-
180
header << %(realm="#{params["realm"]}") if params.key?("realm")
-
180
header << %(algorithm=#{params["algorithm"]}) if params.key?("algorithm")
-
180
header << %(cnonce="#{cnonce}") if cnonce
-
180
header << %(nc=#{nc})
-
180
header << %(qop=#{qop}) if qop
-
180
header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
-
180
header.join ", "
-
end
-
-
23
def make_cnonce
-
200
::Digest::MD5.hexdigest [
-
Time.now.to_i,
-
Process.pid,
-
SecureRandom.random_number(2**32),
-
].join ":"
-
end
-
-
23
def next_nonce
-
160
@nonce += 1
-
end
-
-
23
def raise_format_error
-
raise Error, "unsupported digest header format"
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
17
require "httpx/base64"
-
17
require "ntlm"
-
-
17
module HTTPX
-
17
module Plugins
-
17
module Authentication
-
17
class Ntlm
-
17
def initialize(user, password, domain: nil)
-
4
@user = user
-
4
@password = password
-
4
@domain = domain
-
end
-
-
17
def can_authenticate?(authenticate)
-
2
authenticate && /NTLM .*/.match?(authenticate)
-
end
-
-
17
def negotiate
-
4
"NTLM #{NTLM.negotiate(domain: @domain).to_base64}"
-
end
-
-
17
def authenticate(_req, www)
-
2
challenge = www[/NTLM (.*)/, 1]
-
-
2
challenge = Base64.decode64(challenge)
-
2
ntlm_challenge = NTLM.authenticate(challenge, @user, @domain, @password).to_base64
-
-
2
"NTLM #{ntlm_challenge}"
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
24
module HTTPX
-
24
module Plugins
-
24
module Authentication
-
24
class Socks5
-
24
def initialize(user, password, **)
-
54
@user = user
-
54
@password = password
-
end
-
-
24
def can_authenticate?(*)
-
54
@user && @password
-
end
-
-
24
def authenticate(*)
-
54
[0x01, @user.bytesize, @user, @password.bytesize, @password].pack("CCA*CA*")
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
-
#
-
# It requires the "aws-sdk-core" gem.
-
#
-
23
module AwsSdkAuthentication
-
# Mock configuration, to be used only when resolving credentials
-
23
class Configuration
-
23
attr_reader :profile
-
-
23
def initialize(profile)
-
36
@profile = profile
-
end
-
-
23
def respond_to_missing?(*)
-
18
true
-
end
-
-
23
def method_missing(*); end
-
end
-
-
#
-
# encapsulates access to an AWS SDK credentials store.
-
#
-
23
class Credentials
-
23
def initialize(aws_credentials)
-
18
@aws_credentials = aws_credentials
-
end
-
-
23
def username
-
18
@aws_credentials.access_key_id
-
end
-
-
23
def password
-
18
@aws_credentials.secret_access_key
-
end
-
-
23
def security_token
-
18
@aws_credentials.session_token
-
end
-
end
-
-
23
class << self
-
23
def load_dependencies(_klass)
-
18
require "aws-sdk-core"
-
end
-
-
23
def configure(klass)
-
18
klass.plugin(:aws_sigv4)
-
end
-
-
23
def extra_options(options)
-
18
options.merge(max_concurrent_requests: 1)
-
end
-
-
23
def credentials(profile)
-
18
mock_configuration = Configuration.new(profile)
-
18
Credentials.new(Aws::CredentialProviderChain.new(mock_configuration).resolve)
-
end
-
-
23
def region(profile)
-
# https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb#L62
-
18
keys = %w[AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION]
-
18
env_region = ENV.values_at(*keys).compact.first
-
18
env_region = nil if env_region == ""
-
18
cfg_region = Aws.shared_config.region(profile: profile)
-
18
env_region || cfg_region
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :aws_profile :: AWS account profile to retrieve credentials from.
-
23
module OptionsMethods
-
23
private
-
-
23
def option_aws_profile(value)
-
90
String(value)
-
end
-
end
-
-
23
module InstanceMethods
-
#
-
# aws_authentication
-
# aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
-
# aws_authentication()
-
#
-
23
def aws_sdk_authentication(
-
credentials: AwsSdkAuthentication.credentials(@options.aws_profile),
-
region: AwsSdkAuthentication.region(@options.aws_profile),
-
**options
-
)
-
18
aws_sigv4_authentication(
-
credentials: credentials,
-
region: region,
-
provider_prefix: "aws",
-
header_provider_field: "amz",
-
**options
-
)
-
end
-
23
alias_method :aws_auth, :aws_sdk_authentication
-
end
-
end
-
23
register_plugin :aws_sdk_authentication, AwsSdkAuthentication
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds AWS Sigv4 authentication.
-
#
-
# https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
-
#
-
# https://gitlab.com/os85/httpx/wikis/AWS-SigV4
-
#
-
23
module AWSSigV4
-
23
Credentials = Struct.new(:username, :password, :security_token)
-
-
# Signs requests using the AWS sigv4 signing.
-
23
class Signer
-
23
def initialize(
-
service:,
-
region:,
-
credentials: nil,
-
username: nil,
-
password: nil,
-
security_token: nil,
-
provider_prefix: "aws",
-
header_provider_field: "amz",
-
unsigned_headers: [],
-
apply_checksum_header: true,
-
algorithm: "SHA256"
-
)
-
171
@credentials = credentials || Credentials.new(username, password, security_token)
-
171
@service = service
-
171
@region = region
-
-
171
@unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
-
171
@unsigned_headers << "authorization"
-
171
@unsigned_headers << "x-amzn-trace-id"
-
171
@unsigned_headers << "expect"
-
-
171
@apply_checksum_header = apply_checksum_header
-
171
@provider_prefix = provider_prefix
-
171
@header_provider_field = header_provider_field
-
-
171
@algorithm = algorithm
-
end
-
-
23
def sign!(request)
-
171
lower_provider_prefix = "#{@provider_prefix}4"
-
171
upper_provider_prefix = lower_provider_prefix.upcase
-
-
171
downcased_algorithm = @algorithm.downcase
-
-
171
datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
-
171
date = datetime[0, 8]
-
-
171
content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
-
-
162
request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
-
162
request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
-
-
162
signature_headers = request.headers.each.reject do |k, _|
-
1107
@unsigned_headers.include?(k)
-
end
-
# aws sigv4 needs to declare the host, regardless of protocol version
-
162
signature_headers << ["host", request.authority] unless request.headers.key?("host")
-
162
signature_headers.sort_by!(&:first)
-
-
162
signed_headers = signature_headers.map(&:first).join(";")
-
-
162
canonical_headers = signature_headers.map do |k, v|
-
# eliminate whitespace between value fields, unless it's a quoted value
-
968
"#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
-
end.join
-
-
# canonical request
-
162
creq = "#{request.verb}" \
-
18
"\n#{request.canonical_path}" \
-
18
"\n#{request.canonical_query}" \
-
18
"\n#{canonical_headers}" \
-
18
"\n#{signed_headers}" \
-
18
"\n#{content_hashed}"
-
-
162
credential_scope = "#{date}" \
-
18
"/#{@region}" \
-
18
"/#{@service}" \
-
18
"/#{lower_provider_prefix}_request"
-
-
162
algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
-
# string to sign
-
162
sts = "#{algo_line}" \
-
18
"\n#{datetime}" \
-
18
"\n#{credential_scope}" \
-
18
"\n#{OpenSSL::Digest.new(@algorithm).hexdigest(creq)}"
-
-
# signature
-
162
k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
-
162
k_region = hmac(k_date, @region)
-
162
k_service = hmac(k_region, @service)
-
162
k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
-
162
sig = hexhmac(k_credentials, sts)
-
-
162
credential = "#{@credentials.username}/#{credential_scope}"
-
# apply signature
-
144
request.headers["authorization"] =
-
18
"#{algo_line} " \
-
18
"Credential=#{credential}, " \
-
18
"SignedHeaders=#{signed_headers}, " \
-
18
"Signature=#{sig}"
-
end
-
-
23
private
-
-
23
def hexdigest(value)
-
162
digest = OpenSSL::Digest.new(@algorithm)
-
-
162
if value.respond_to?(:read)
-
36
if value.respond_to?(:to_path)
-
# files, pathnames
-
9
digest.file(value.to_path).hexdigest
-
else
-
# gzipped request bodies
-
27
raise Error, "request body must be rewindable" unless value.respond_to?(:rewind)
-
-
27
buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
-
2
begin
-
27
IO.copy_stream(value, buffer)
-
27
buffer.flush
-
-
27
digest.file(buffer.to_path).hexdigest
-
ensure
-
27
value.rewind
-
27
buffer.close
-
27
buffer.unlink
-
end
-
end
-
else
-
# error on endless generators
-
126
raise Error, "hexdigest for endless enumerators is not supported" if value.unbounded_body?
-
-
117
mb_buffer = value.each.with_object("".b) do |chunk, b|
-
63
b << chunk
-
63
break if b.bytesize >= 1024 * 1024
-
end
-
-
117
digest.hexdigest(mb_buffer)
-
end
-
end
-
-
23
def hmac(key, value)
-
648
OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
-
end
-
-
23
def hexhmac(key, value)
-
162
OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
-
end
-
end
-
-
23
class << self
-
23
def load_dependencies(*)
-
171
require "set"
-
171
require "digest/sha2"
-
171
require "cgi/escape"
-
end
-
-
23
def configure(klass)
-
171
klass.plugin(:expect)
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :sigv4_signer :: instance of HTTPX::Plugins::AWSSigV4 used to sign requests.
-
23
module OptionsMethods
-
23
private
-
-
23
def option_sigv4_signer(value)
-
360
value.is_a?(Signer) ? value : Signer.new(value)
-
end
-
end
-
-
23
module InstanceMethods
-
23
def aws_sigv4_authentication(**options)
-
171
with(sigv4_signer: Signer.new(**options))
-
end
-
-
23
def build_request(*)
-
171
request = super
-
-
171
return request if request.headers.key?("authorization")
-
-
171
signer = request.options.sigv4_signer
-
-
171
return request unless signer
-
-
171
signer.sign!(request)
-
-
162
request
-
end
-
end
-
-
23
module RequestMethods
-
23
def canonical_path
-
162
path = uri.path.dup
-
162
path << "/" if path.empty?
-
198
path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
-
end
-
-
23
def canonical_query
-
198
params = query.split("&")
-
# params = params.map { |p| p.match(/=/) ? p : p + '=' }
-
# From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
-
# Sort the parameter names by character code point in ascending order.
-
# Parameters with duplicate names should be sorted by value.
-
#
-
# Default sort <=> in JRuby will swap members
-
# occasionally when <=> is 0 (considered still sorted), but this
-
# causes our normalized query string to not match the sent querystring.
-
# When names match, we then sort by their values. When values also
-
# match then we sort by their original order
-
198
params.each.with_index.sort do |a, b|
-
72
a, a_offset = a
-
72
b, b_offset = b
-
72
a_name, a_value = a.split("=", 2)
-
72
b_name, b_value = b.split("=", 2)
-
72
if a_name == b_name
-
36
if a_value == b_value
-
18
a_offset <=> b_offset
-
else
-
18
a_value <=> b_value
-
end
-
else
-
36
a_name <=> b_name
-
end
-
end.map(&:first).join("&")
-
end
-
end
-
end
-
23
register_plugin :aws_sigv4, AWSSigV4
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
-
#
-
# https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
-
#
-
23
module BasicAuth
-
23
class << self
-
23
def load_dependencies(_klass)
-
126
require_relative "auth/basic"
-
end
-
-
23
def configure(klass)
-
126
klass.plugin(:auth)
-
end
-
end
-
-
23
module InstanceMethods
-
23
def basic_auth(user, password)
-
144
authorization(Authentication::Basic.new(user, password).authenticate)
-
end
-
end
-
end
-
23
register_plugin :basic_auth, BasicAuth
-
end
-
end
-
# frozen_string_literal: true
-
-
22
module HTTPX
-
22
module Plugins
-
22
module Brotli
-
22
class Error < HTTPX::Error; end
-
-
22
class Deflater < Transcoder::Deflater
-
22
def initialize(body)
-
14
@compressor = ::Brotli::Compressor.new
-
14
super
-
end
-
-
22
def deflate(chunk)
-
42
return @compressor.process(chunk) << @compressor.flush if chunk
-
-
28
@compressor.finish
-
end
-
end
-
-
22
class Inflater
-
22
def initialize(bytesize)
-
15
@inflater = ::Brotli::Decompressor.new
-
15
@bytesize = bytesize
-
end
-
-
22
def call(chunk)
-
27
buffer = @inflater.process(chunk)
-
27
@bytesize -= chunk.bytesize
-
27
raise Error, "Unexpected end of compressed stream" if @bytesize <= 0 && !@inflater.finished?
-
-
27
buffer
-
end
-
end
-
-
22
module RequestBodyClassMethods
-
22
def initialize_deflater_body(body, encoding)
-
28
return Brotli.encode(body) if encoding == "br"
-
-
14
super
-
end
-
end
-
-
22
module ResponseBodyClassMethods
-
22
def initialize_inflater_by_encoding(encoding, response, **kwargs)
-
29
return Brotli.decode(response, **kwargs) if encoding == "br"
-
-
14
super
-
end
-
end
-
-
22
module_function
-
-
22
def load_dependencies(*)
-
29
gem "brotli", ">= 0.8.0"
-
29
require "brotli"
-
end
-
-
22
def self.extra_options(options)
-
29
supported_compression_formats = (%w[br] + options.supported_compression_formats).freeze
-
29
options.merge(
-
supported_compression_formats: supported_compression_formats,
-
headers: options.headers_class.new(options.headers.merge("accept-encoding" => supported_compression_formats))
-
)
-
end
-
-
22
def encode(body)
-
14
Deflater.new(body)
-
end
-
-
22
def decode(response, bytesize: nil)
-
15
bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
-
15
Inflater.new(bytesize)
-
end
-
end
-
22
register_plugin :brotli, Brotli
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Plugins
-
#
-
# This plugin adds suppoort for callbacks around the request/response lifecycle.
-
#
-
# https://gitlab.com/os85/httpx/-/wikis/Events
-
#
-
30
module Callbacks
-
30
CALLBACKS = %i[
-
connection_opened connection_closed
-
request_error
-
request_started request_body_chunk request_completed
-
response_started response_body_chunk response_completed
-
].freeze
-
-
# connection closed user-space errors happen after errors can be surfaced to requests,
-
# so they need to pierce through the scheduler, which is only possible by simulating an
-
# interrupt.
-
30
class CallbackError < Exception; end # rubocop:disable Lint/InheritException
-
-
30
module InstanceMethods
-
30
include HTTPX::Callbacks
-
-
30
CALLBACKS.each do |meth|
-
270
class_eval(<<-MOD, __FILE__, __LINE__ + 1)
-
9
def on_#{meth}(&blk) # def on_connection_opened(&blk)
-
9
on(:#{meth}, &blk) # on(:connection_opened, &blk)
-
self # self
-
end # end
-
MOD
-
end
-
-
30
def plugin(*args, &blk)
-
super(*args).tap do |sess|
-
CALLBACKS.each do |cb|
-
next unless callbacks_for?(cb)
-
-
sess.callbacks(cb).concat(callbacks(cb))
-
end
-
-
sess.wrap(&blk) if blk
-
end
-
end
-
-
30
private
-
-
30
def branch(options, &blk)
-
36
super(options).tap do |sess|
-
36
CALLBACKS.each do |cb|
-
324
next unless callbacks_for?(cb)
-
-
18
sess.callbacks(cb).concat(callbacks(cb))
-
end
-
36
sess.wrap(&blk) if blk
-
end
-
end
-
-
30
def do_init_connection(connection, selector)
-
253
super
-
253
connection.on(:open) do
-
219
next unless connection.current_session == self
-
-
219
emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
-
end
-
253
connection.on(:callback_connection_closed) do
-
245
next unless connection.current_session == self
-
-
245
emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
-
end
-
-
253
connection
-
end
-
-
30
def set_request_callbacks(request)
-
255
super
-
-
255
request.on(:headers) do
-
201
emit_or_callback_error(:request_started, request)
-
end
-
255
request.on(:body_chunk) do |chunk|
-
18
emit_or_callback_error(:request_body_chunk, request, chunk)
-
end
-
255
request.on(:done) do
-
183
emit_or_callback_error(:request_completed, request)
-
end
-
-
255
request.on(:response_started) do |res|
-
201
if res.is_a?(Response)
-
165
emit_or_callback_error(:response_started, request, res)
-
147
res.on(:chunk_received) do |chunk|
-
181
emit_or_callback_error(:response_body_chunk, request, res, chunk)
-
end
-
else
-
36
emit_or_callback_error(:request_error, request, res.error)
-
end
-
end
-
255
request.on(:response) do |res|
-
147
emit_or_callback_error(:response_completed, request, res) if res.is_a?(Response)
-
end
-
end
-
-
30
def emit_or_callback_error(*args)
-
1359
emit(*args)
-
rescue StandardError => e
-
153
ex = CallbackError.new(e.message)
-
153
ex.set_backtrace(e.backtrace)
-
153
raise ex
-
end
-
-
30
def receive_requests(*)
-
255
super
-
rescue CallbackError => e
-
135
raise e.cause
-
end
-
-
30
def close(*)
-
253
super
-
rescue CallbackError => e
-
9
raise e.cause
-
end
-
end
-
-
30
module ConnectionMethods
-
30
private
-
-
30
def disconnect
-
263
return if @exhausted
-
-
263
return unless @current_session && @current_selector
-
-
245
emit(:callback_connection_closed)
-
-
218
super
-
end
-
end
-
end
-
30
register_plugin :callbacks, Callbacks
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin implements a circuit breaker around connection errors.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Circuit-Breaker
-
#
-
23
module CircuitBreaker
-
23
using URIExtensions
-
-
23
def self.load_dependencies(*)
-
63
require_relative "circuit_breaker/circuit"
-
63
require_relative "circuit_breaker/circuit_store"
-
end
-
-
23
def self.extra_options(options)
-
63
options.merge(
-
circuit_breaker_max_attempts: 3,
-
circuit_breaker_reset_attempts_in: 60,
-
circuit_breaker_break_in: 60,
-
circuit_breaker_half_open_drip_rate: 1
-
)
-
end
-
-
23
module InstanceMethods
-
23
include HTTPX::Callbacks
-
-
23
def initialize(*)
-
63
super
-
63
@circuit_store = CircuitStore.new(@options)
-
end
-
-
23
%i[circuit_open].each do |meth|
-
23
class_eval(<<-MOD, __FILE__, __LINE__ + 1)
-
1
def on_#{meth}(&blk) # def on_circuit_open(&blk)
-
1
on(:#{meth}, &blk) # on(:circuit_open, &blk)
-
self # self
-
end # end
-
MOD
-
end
-
-
23
private
-
-
23
def send_requests(*requests)
-
# @type var short_circuit_responses: Array[response]
-
252
short_circuit_responses = []
-
-
# run all requests through the circuit breaker, see if the circuit is
-
# open for any of them.
-
252
real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
-
252
short_circuit_response = @circuit_store.try_respond(req)
-
252
if short_circuit_response.nil?
-
198
real_reqs << req
-
198
next
-
end
-
48
short_circuit_responses[idx] = short_circuit_response
-
end
-
-
# run requests for the remainder
-
252
unless real_requests.empty?
-
198
responses = super(*real_requests)
-
-
198
real_requests.each_with_index do |request, idx|
-
176
short_circuit_responses[requests.index(request)] = responses[idx]
-
end
-
end
-
-
252
short_circuit_responses
-
end
-
-
23
def set_request_callbacks(request)
-
252
super
-
252
request.on(:response) do |response|
-
198
emit(:circuit_open, request) if try_circuit_open(request, response)
-
end
-
end
-
-
23
def try_circuit_open(request, response)
-
198
if response.is_a?(ErrorResponse)
-
128
case response.error
-
when RequestTimeoutError
-
90
@circuit_store.try_open(request.uri, response)
-
else
-
54
@circuit_store.try_open(request.origin, response)
-
end
-
54
elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
-
18
@circuit_store.try_open(request.uri, response)
-
else
-
36
@circuit_store.try_close(request.uri)
-
16
nil
-
end
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :circuit_breaker_max_attempts :: the number of attempts the circuit allows, before it is opened (defaults to <tt>3</tt>).
-
# :circuit_breaker_reset_attempts_in :: the time a circuit stays open at most, before it resets (defaults to <tt>60</tt>).
-
# :circuit_breaker_break_on :: callable defining an alternative rule for a response to break
-
# (i.e. <tt>->(res) { res.status == 429 } </tt>)
-
# :circuit_breaker_break_in :: the time that must elapse before an open circuit can transit to the half-open state
-
# (defaults to <tt><60</tt>).
-
# :circuit_breaker_half_open_drip_rate :: the rate of requests a circuit allows to be performed when in an half-open state
-
# (defaults to <tt>1</tt>).
-
23
module OptionsMethods
-
23
private
-
-
23
def option_circuit_breaker_max_attempts(value)
-
126
attempts = Integer(value)
-
126
raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
-
-
126
attempts
-
end
-
-
23
def option_circuit_breaker_reset_attempts_in(value)
-
72
timeout = Float(value)
-
72
raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
-
-
72
timeout
-
end
-
-
23
def option_circuit_breaker_break_in(value)
-
99
timeout = Float(value)
-
99
raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
-
-
99
timeout
-
end
-
-
23
def option_circuit_breaker_half_open_drip_rate(value)
-
99
ratio = Float(value)
-
99
raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
-
-
99
ratio
-
end
-
-
23
def option_circuit_breaker_break_on(value)
-
18
raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
-
-
18
value
-
end
-
end
-
end
-
23
register_plugin :circuit_breaker, CircuitBreaker
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins::CircuitBreaker
-
#
-
# A circuit is assigned to a given absoolute url or origin.
-
#
-
# It sets +max_attempts+, the number of attempts the circuit allows, before it is opened.
-
# It sets +reset_attempts_in+, the time a circuit stays open at most, before it resets.
-
# It sets +break_in+, the time that must elapse before an open circuit can transit to the half-open state.
-
# It sets +circuit_breaker_half_open_drip_rate+, the rate of requests a circuit allows to be performed when in an half-open state.
-
#
-
23
class Circuit
-
23
def initialize(max_attempts, reset_attempts_in, break_in, circuit_breaker_half_open_drip_rate)
-
63
@max_attempts = max_attempts
-
63
@reset_attempts_in = reset_attempts_in
-
63
@break_in = break_in
-
63
@circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
-
63
@attempts = 0
-
-
63
total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
-
63
@drip_factor = (@max_attempts / total_real_attempts).round
-
63
@state = :closed
-
end
-
-
23
def respond
-
252
try_close
-
-
224
case @state
-
when :closed
-
68
nil
-
when :half_open
-
56
@attempts += 1
-
-
# do real requests while drip rate valid
-
63
if (@real_attempts % @drip_factor).zero?
-
40
@real_attempts += 1
-
40
return
-
end
-
-
18
@response
-
when :open
-
-
36
@response
-
end
-
end
-
-
23
def try_open(response)
-
144
case @state
-
when :closed
-
135
now = Utils.now
-
-
135
if @attempts.positive?
-
# reset if error happened long ago
-
54
@attempts = 0 if now - @attempted_at > @reset_attempts_in
-
else
-
81
@attempted_at = now
-
end
-
-
120
@attempts += 1
-
-
135
return unless @attempts >= @max_attempts
-
-
72
@state = :open
-
72
@opened_at = now
-
72
@response = response
-
when :half_open
-
# open immediately
-
-
27
@state = :open
-
27
@attempted_at = @opened_at = Utils.now
-
27
@response = response
-
end
-
end
-
-
23
def try_close
-
256
case @state
-
when :closed
-
68
nil
-
when :half_open
-
-
# do not close circuit unless attempts exhausted
-
54
return unless @attempts >= @max_attempts
-
-
# reset!
-
18
@attempts = 0
-
18
@opened_at = @attempted_at = @response = nil
-
18
@state = :closed
-
-
when :open
-
81
if Utils.elapsed_time(@opened_at) > @break_in
-
45
@state = :half_open
-
45
@attempts = 0
-
45
@real_attempts = 0
-
end
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX::Plugins::CircuitBreaker
-
23
using HTTPX::URIExtensions
-
-
23
class CircuitStore
-
23
def initialize(options)
-
63
@circuits = Hash.new do |h, k|
-
56
h[k] = Circuit.new(
-
options.circuit_breaker_max_attempts,
-
options.circuit_breaker_reset_attempts_in,
-
options.circuit_breaker_break_in,
-
options.circuit_breaker_half_open_drip_rate
-
)
-
end
-
63
@circuits_mutex = Thread::Mutex.new
-
end
-
-
23
def try_open(uri, response)
-
324
circuit = @circuits_mutex.synchronize { get_circuit_for_uri(uri) }
-
-
162
circuit.try_open(response)
-
end
-
-
23
def try_close(uri)
-
36
circuit = @circuits_mutex.synchronize do
-
36
return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
-
-
36
get_circuit_for_uri(uri)
-
end
-
-
36
circuit.try_close
-
end
-
-
# if circuit is open, it'll respond with the stored response.
-
# if not, nil.
-
23
def try_respond(request)
-
504
circuit = @circuits_mutex.synchronize { get_circuit_for_uri(request.uri) }
-
-
252
circuit.respond
-
end
-
-
23
private
-
-
23
def get_circuit_for_uri(uri)
-
450
if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
-
324
@circuits[uri.origin]
-
else
-
126
@circuits[uri.to_s]
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds `Content-Digest` headers to requests
-
# and can validate these headers on responses
-
#
-
# https://datatracker.ietf.org/doc/html/rfc9530
-
#
-
23
module ContentDigest
-
23
class Error < HTTPX::Error; end
-
-
# Error raised on response "content-digest" header validation.
-
23
class ValidationError < Error
-
23
attr_reader :response
-
-
23
def initialize(message, response)
-
54
super(message)
-
54
@response = response
-
end
-
end
-
-
23
class MissingContentDigestError < ValidationError; end
-
23
class InvalidContentDigestError < ValidationError; end
-
-
2
SUPPORTED_ALGORITHMS = {
-
21
"sha-256" => OpenSSL::Digest::SHA256,
-
"sha-512" => OpenSSL::Digest::SHA512,
-
}.freeze
-
-
23
class << self
-
23
def extra_options(options)
-
234
options.merge(encode_content_digest: true, validate_content_digest: false, content_digest_algorithm: "sha-256")
-
end
-
end
-
-
# add support for the following options:
-
#
-
# :content_digest_algorithm :: the digest algorithm to use. Currently supports `sha-256` and `sha-512`. (defaults to `sha-256`)
-
# :encode_content_digest :: whether a <tt>Content-Digest</tt> header should be computed for the request;
-
# can also be a callable object (i.e. <tt>->(req) { ... }</tt>, defaults to <tt>true</tt>)
-
# :validate_content_digest :: whether a <tt>Content-Digest</tt> header in the response should be validated;
-
# can also be a callable object (i.e. <tt>->(res) { ... }</tt>, defaults to <tt>false</tt>)
-
23
module OptionsMethods
-
23
private
-
-
23
def option_content_digest_algorithm(value)
-
243
raise TypeError, ":content_digest_algorithm must be one of 'sha-256', 'sha-512'" unless SUPPORTED_ALGORITHMS.key?(value)
-
-
243
value
-
end
-
-
23
def option_encode_content_digest(value)
-
234
value
-
end
-
-
23
def option_validate_content_digest(value)
-
162
value
-
end
-
end
-
-
23
module ResponseBodyMethods
-
23
attr_reader :content_digest_buffer
-
-
23
def initialize(response, options)
-
198
super
-
-
198
return unless response.headers.key?("content-digest")
-
-
144
should_validate = options.validate_content_digest
-
144
should_validate = should_validate.call(response) if should_validate.respond_to?(:call)
-
-
144
return unless should_validate
-
-
126
@content_digest_buffer = Response::Buffer.new(
-
threshold_size: @options.body_threshold_size,
-
bytesize: @length,
-
encoding: @encoding
-
)
-
end
-
-
23
def write(chunk)
-
326
@content_digest_buffer.write(chunk) if @content_digest_buffer
-
326
super
-
end
-
-
23
def close
-
126
if @content_digest_buffer
-
126
@content_digest_buffer.close
-
126
@content_digest_buffer = nil
-
end
-
126
super
-
end
-
end
-
-
23
module InstanceMethods
-
23
def build_request(*)
-
252
request = super
-
-
252
return request if request.empty?
-
-
54
return request if request.headers.key?("content-digest")
-
-
54
perform_encoding = @options.encode_content_digest
-
54
perform_encoding = perform_encoding.call(request) if perform_encoding.respond_to?(:call)
-
-
54
return request unless perform_encoding
-
-
45
digest = base64digest(request.body)
-
45
request.headers.add("content-digest", "#{@options.content_digest_algorithm}=:#{digest}:")
-
-
45
request
-
end
-
-
23
private
-
-
23
def fetch_response(request, _, _)
-
396
response = super
-
396
return response unless response.is_a?(Response)
-
-
198
perform_validation = @options.validate_content_digest
-
198
perform_validation = perform_validation.call(response) if perform_validation.respond_to?(:call)
-
-
198
validate_content_digest(response) if perform_validation
-
-
144
response
-
rescue ValidationError => e
-
54
ErrorResponse.new(request, e)
-
end
-
-
23
def validate_content_digest(response)
-
144
content_digest_header = response.headers["content-digest"]
-
-
144
raise MissingContentDigestError.new("response is missing a `content-digest` header", response) unless content_digest_header
-
-
126
digests = extract_content_digests(content_digest_header)
-
-
126
included_algorithms = SUPPORTED_ALGORITHMS.keys & digests.keys
-
-
126
raise MissingContentDigestError.new("unsupported algorithms: #{digests.keys.join(", ")}", response) if included_algorithms.empty?
-
-
126
content_buffer = response.body.content_digest_buffer
-
-
126
included_algorithms.each do |algorithm|
-
126
digest = SUPPORTED_ALGORITHMS.fetch(algorithm).new
-
126
digest_received = digests[algorithm]
-
14
digest_computed =
-
125
if content_buffer.respond_to?(:to_path)
-
18
content_buffer.flush
-
18
digest.file(content_buffer.to_path).base64digest
-
else
-
108
digest.base64digest(content_buffer.to_s)
-
end
-
-
4
raise InvalidContentDigestError.new("#{algorithm} digest does not match content",
-
125
response) unless digest_received == digest_computed
-
end
-
end
-
-
23
def extract_content_digests(header)
-
126
header.split(",").to_h do |entry|
-
144
algorithm, digest = entry.split("=", 2)
-
144
raise Error, "#{entry} is an invalid digest format" unless algorithm && digest
-
-
144
[algorithm, digest.byteslice(1..-2)]
-
end
-
end
-
-
23
def base64digest(body)
-
45
digest = SUPPORTED_ALGORITHMS.fetch(@options.content_digest_algorithm).new
-
-
45
if body.respond_to?(:read)
-
36
if body.respond_to?(:to_path)
-
9
digest.file(body.to_path).base64digest
-
else
-
27
raise ContentDigestError, "request body must be rewindable" unless body.respond_to?(:rewind)
-
-
27
buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
-
2
begin
-
27
IO.copy_stream(body, buffer)
-
27
buffer.flush
-
-
27
digest.file(buffer.to_path).base64digest
-
ensure
-
27
body.rewind
-
27
buffer.close
-
27
buffer.unlink
-
end
-
end
-
else
-
9
raise ContentDigestError, "base64digest for endless enumerators is not supported" if body.unbounded_body?
-
-
9
buffer = "".b
-
18
body.each { |chunk| buffer << chunk }
-
-
9
digest.base64digest(buffer)
-
end
-
end
-
end
-
end
-
23
register_plugin :content_digest, ContentDigest
-
end
-
end
-
# frozen_string_literal: true
-
-
23
require "forwardable"
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin implements a persistent cookie jar for the duration of a session.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Cookies
-
#
-
23
module Cookies
-
23
def self.load_dependencies(*)
-
216
require "httpx/plugins/cookies/jar"
-
216
require "httpx/plugins/cookies/cookie"
-
216
require "httpx/plugins/cookies/set_cookie_parser"
-
end
-
-
23
module InstanceMethods
-
23
extend Forwardable
-
-
23
def_delegator :@options, :cookies
-
-
23
def initialize(options = {}, &blk)
-
378
super({ cookies: Jar.new }.merge(options), &blk)
-
end
-
-
23
def wrap
-
18
return super unless block_given?
-
-
18
super do |session|
-
18
old_cookies_jar = @options.cookies.dup
-
1
begin
-
18
yield session
-
ensure
-
18
@options = @options.merge(cookies: old_cookies_jar)
-
end
-
end
-
end
-
-
23
def build_request(*)
-
360
request = super
-
360
request.headers.set_cookie(request.options.cookies[request.uri])
-
360
request
-
end
-
-
# factory method to return a Jar to the user, which can then manipulate
-
# externally to the session.
-
23
def make_jar(*args)
-
18
Jar.new(*args)
-
end
-
-
23
private
-
-
23
def set_request_callbacks(request)
-
360
super
-
360
request.on(:response) do |response|
-
360
next unless response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
-
-
72
log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
-
-
72
@options.cookies.parse(set_cookie)
-
end
-
end
-
end
-
-
23
module HeadersMethods
-
23
def set_cookie(cookies)
-
360
return if cookies.empty?
-
-
306
header_value = cookies.sort.join("; ")
-
-
306
add("cookie", header_value)
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :cookies :: cookie jar for the session (can be a Hash, an Array, an instance of HTTPX::Plugins::Cookies::CookieJar)
-
23
module OptionsMethods
-
23
private
-
-
23
def option_headers(*)
-
414
value = super
-
-
414
merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
-
-
414
value
-
end
-
-
23
def option_cookies(value)
-
594
jar = value.is_a?(Jar) ? value : Jar.new(value)
-
-
594
merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
-
-
594
jar
-
end
-
-
23
def merge_cookie_in_jar(cookies, jar)
-
18
cookies.each do |ck|
-
18
ck.split(/ *; */).each do |cookie|
-
36
name, value = cookie.split("=", 2)
-
36
jar.set(name, value)
-
end
-
end
-
end
-
end
-
end
-
23
register_plugin :cookies, Cookies
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins::Cookies
-
# The HTTP Cookie.
-
#
-
# Contains the single cookie info: name, value and attributes.
-
23
class Cookie
-
23
include Comparable
-
-
# Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
-
# least)
-
23
MAX_LENGTH = 4096
-
-
23
attr_reader :domain, :path, :name, :value, :created_at
-
-
# assigns a new +path+ to this cookie.
-
23
def path=(path)
-
207
path = String(path)
-
207
@for_domain = false
-
207
@path = path.start_with?("/") ? path : "/"
-
end
-
-
# assigns a new +domain+ to this cookie.
-
23
def domain=(domain)
-
45
domain = String(domain)
-
-
45
if domain.start_with?(".")
-
18
@for_domain = true
-
18
domain = domain[1..-1]
-
end
-
-
45
return if domain.empty?
-
-
45
@domain_name = DomainName.new(domain)
-
# RFC 6265 5.3 5.
-
45
@for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
-
-
45
@domain = @domain_name.hostname
-
end
-
-
# checks whether +other+ is the same cookie, i.e. name, value, domain and path are
-
# the same.
-
23
def ==(other)
-
135
@name == other.name && @value == other.value &&
-
@path == other.path && @domain == other.domain
-
end
-
-
# Compares the cookie with another. When there are many cookies with
-
# the same name for a URL, the value of the smallest must be used.
-
23
def <=>(other)
-
# RFC 6265 5.4
-
# Precedence: 1. longer path 2. older creation
-
782
(@name <=> other.name).nonzero? ||
-
67
(other.path.length <=> @path.length).nonzero? ||
-
39
(@created_at <=> other.created_at).nonzero? || 0
-
end
-
-
23
def match?(name_or_options)
-
152
case name_or_options
-
when String
-
72
@name == name_or_options
-
when Hash, Array
-
268
name_or_options.all? { |k, v| respond_to?(k) && send(k) == v }
-
else
-
false
-
end
-
end
-
-
23
class << self
-
23
def new(cookie, *args)
-
632
case cookie
-
when self
-
cookie
-
when Array, Hash
-
36
options = Hash[cookie] #: cookie_attributes
-
36
super(options[:name], options[:value], options)
-
else
-
-
675
super
-
end
-
end
-
-
# Tests if +target_path+ is under +base_path+ as described in RFC
-
# 6265 5.1.4. +base_path+ must be an absolute path.
-
# +target_path+ may be empty, in which case it is treated as the
-
# root path.
-
#
-
# e.g.
-
#
-
# path_match?('/admin/', '/admin/index') == true
-
# path_match?('/admin/', '/Admin/index') == false
-
# path_match?('/admin/', '/admin/') == true
-
# path_match?('/admin/', '/admin') == false
-
#
-
# path_match?('/admin', '/admin') == true
-
# path_match?('/admin', '/Admin') == false
-
# path_match?('/admin', '/admins') == false
-
# path_match?('/admin', '/admin/') == true
-
# path_match?('/admin', '/admin/index') == true
-
23
def path_match?(base_path, target_path)
-
1521
base_path.start_with?("/") || (return false)
-
# RFC 6265 5.1.4
-
1521
bsize = base_path.size
-
1521
tsize = target_path.size
-
1521
return bsize == 1 if tsize.zero? # treat empty target_path as "/"
-
1521
return false unless target_path.start_with?(base_path)
-
1512
return true if bsize == tsize || base_path.end_with?("/")
-
-
18
target_path[bsize] == "/"
-
end
-
end
-
-
23
def initialize(arg, value, attrs = nil)
-
711
@created_at = Time.now
-
-
711
@name = arg
-
711
@value = value
-
711
attr_hash = Hash.try_convert(attrs)
-
-
34
attr_hash.each do |key, val|
-
369
key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
-
-
328
case key
-
when :domain, :path
-
229
__send__(:"#{key}=", val)
-
else
-
117
instance_variable_set(:"@#{key}", val)
-
end
-
710
end if attr_hash
-
-
711
@path ||= "/"
-
711
raise ArgumentError, "name must be specified" if @name.nil?
-
-
711
@name = @name.to_s
-
end
-
-
23
def expires
-
855
@expires || (@created_at && @max_age ? @created_at + @max_age : nil)
-
end
-
-
23
def expired?(time = Time.now)
-
819
return false unless expires
-
-
36
expires <= time
-
end
-
-
# Returns a string for use in the Cookie header, i.e. `name=value`
-
# or `name="value"`.
-
23
def cookie_value
-
552
"#{@name}=#{Scanner.quote(@value.to_s)}"
-
end
-
23
alias_method :to_s, :cookie_value
-
-
# Tests if it is OK to send this cookie to a given `uri`. A
-
# RuntimeError is raised if the cookie's domain is unknown.
-
23
def valid_for_uri?(uri)
-
801
uri = URI(uri)
-
# RFC 6265 5.4
-
-
801
return false if @secure && uri.scheme != "https"
-
-
792
acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
-
end
-
-
23
private
-
-
# Tests if it is OK to accept this cookie if it is sent from a given
-
# URI/URL, `uri`.
-
23
def acceptable_from_uri?(uri)
-
828
uri = URI(uri)
-
-
828
host = DomainName.new(uri.host)
-
-
# RFC 6265 5.3
-
828
if host.hostname == @domain
-
18
true
-
809
elsif @for_domain # !host-only-flag
-
36
host.cookie_domain?(@domain_name)
-
else
-
774
@domain.nil?
-
end
-
end
-
-
23
module Scanner
-
23
RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
-
-
23
module_function
-
-
23
def quote(s)
-
621
return s unless s.match(RE_BAD_CHAR)
-
-
9
"\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins::Cookies
-
# The Cookie Jar
-
#
-
# It stores and manages cookies for a session, such as i.e. evicting when expired, access methods, or
-
# initialization from parsing `Set-Cookie` HTTP header values.
-
#
-
# It closely follows the [CookieStore API](https://developer.mozilla.org/en-US/docs/Web/API/CookieStore),
-
# by implementing the same methods, with a few specific conveniences for this non-browser manipulation use-case.
-
#
-
23
class Jar
-
23
using URIExtensions
-
-
23
include Enumerable
-
-
23
def initialize_dup(orig)
-
243
super
-
243
@mtx = orig.instance_variable_get(:@mtx).dup
-
243
@cookies = orig.instance_variable_get(:@cookies).dup
-
end
-
-
# initializes the cookie store, either empty, or with whatever is passed as +cookies+, which
-
# can be an array of HTTPX::Plugins::Cookies::Cookie objects or hashes-or-tuples of cookie attributes.
-
23
def initialize(cookies = nil)
-
774
@mtx = Thread::Mutex.new
-
774
@cookies = []
-
-
162
cookies.each do |elem|
-
270
cookie = case elem
-
when Cookie
-
27
elem
-
when Array
-
225
Cookie.new(*elem)
-
else
-
18
Cookie.new(elem)
-
end
-
-
270
@cookies << cookie
-
773
end if cookies
-
end
-
-
# parses the `Set-Cookie` header value as +set_cookie+ and does the corresponding updates.
-
23
def parse(set_cookie)
-
162
SetCookieParser.call(set_cookie) do |name, value, attrs|
-
234
set(Cookie.new(name, value, attrs))
-
end
-
end
-
-
# returns the first HTTPX::Plugins::Cookie::Cookie instance in the store which matches either the name
-
# (when String) or all attributes (when a Hash or array of tuples) passed to +name_or_options+
-
23
def get(name_or_options)
-
108
each.find { |ck| ck.match?(name_or_options) }
-
end
-
-
# returns all HTTPX::Plugins::Cookie::Cookie instances in the store which match either the name
-
# (when String) or all attributes (when a Hash or array of tuples) passed to +name_or_options+
-
23
def get_all(name_or_options)
-
135
each.select { |ck| ck.match?(name_or_options) } # rubocop:disable Style/SelectByRegexp
-
end
-
-
# when +name+ is a HTTPX::Plugins::Cookie::Cookie, it stores it internally; when +name+ is a String,
-
# it creates a cookie with it and the value-or-attributes passed to +value_or_options+.
-
-
# optionally, +name+ can also be the attributes hash-or-array as long it contains a <tt>:name</tt> field).
-
23
def set(name, value_or_options = nil)
-
576
cookie = case name
-
when Cookie
-
504
raise ArgumentError, "there should not be a second argument" if value_or_options
-
-
495
name
-
when Array, Hash
-
18
raise ArgumentError, "there should not be a second argument" if value_or_options
-
-
9
Cookie.new(name)
-
else
-
54
raise ArgumentError, "the second argument is required" unless value_or_options
-
-
45
Cookie.new(name, value_or_options)
-
end
-
-
549
synchronize do
-
# If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
-
# as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
-
1008
@cookies.delete_if { |ck| ck.name == cookie.name && ck.domain == cookie.domain && ck.path == cookie.path }
-
-
549
@cookies << cookie
-
end
-
end
-
-
# @deprecated
-
23
def add(cookie, path = nil)
-
9
warn "DEPRECATION WARNING: calling `##{__method__}` is deprecated. Use `#set` instead."
-
9
c = cookie.dup
-
9
c.path = path if path && c.path == "/"
-
9
set(c)
-
end
-
-
# deletes all cookies in the store which match either the name (when String) or all attributes (when a Hash
-
# or array of tuples) passed to +name_or_options+.
-
#
-
# alternatively, of +name_or_options+ is an instance of HTTPX::Plugins::Cookies::Cookiem, it deletes it from the store.
-
23
def delete(name_or_options)
-
36
synchronize do
-
32
case name_or_options
-
when Cookie
-
9
@cookies.delete(name_or_options)
-
else
-
54
@cookies.delete_if { |ck| ck.match?(name_or_options) }
-
end
-
end
-
end
-
-
# returns the list of valid cookies which matdh the domain and path from the URI object passed to +uri+.
-
23
def [](uri)
-
531
each(uri).sort
-
end
-
-
# enumerates over all stored cookies. if +uri+ is passed, it'll filter out expired cookies and
-
# only yield cookies which match its domain and path.
-
23
def each(uri = nil, &blk)
-
1764
return enum_for(__method__, uri) unless blk
-
-
1431
return synchronize { @cookies.each(&blk) } unless uri
-
-
531
now = Time.now
-
531
tpath = uri.path
-
-
531
synchronize do
-
531
@cookies.delete_if do |cookie|
-
819
if cookie.expired?(now)
-
18
true
-
else
-
801
yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
-
801
false
-
end
-
end
-
end
-
end
-
-
23
def merge(other)
-
225
jar_dup = dup
-
-
225
other.each do |elem|
-
243
cookie = case elem
-
when Cookie
-
225
elem
-
when Array
-
9
Cookie.new(*elem)
-
else
-
9
Cookie.new(elem)
-
end
-
-
243
jar_dup.set(cookie)
-
end
-
-
225
jar_dup
-
end
-
-
23
private
-
-
23
def synchronize(&block)
-
1566
return yield if @mtx.owned?
-
-
1566
@mtx.synchronize(&block)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
23
require "strscan"
-
23
require "time"
-
-
23
module HTTPX
-
23
module Plugins::Cookies
-
23
module SetCookieParser
-
# Whitespace.
-
23
RE_WSP = /[ \t]+/.freeze
-
-
# A pattern that matches a cookie name or attribute name which may
-
# be empty, capturing trailing whitespace.
-
23
RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
-
-
23
RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
-
-
# A pattern that matches the comma in a (typically date) value.
-
23
RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
-
-
23
module_function
-
-
23
def scan_dquoted(scanner)
-
18
s = +""
-
-
24
until scanner.eos?
-
72
break if scanner.skip(/"/)
-
-
54
if scanner.skip(/\\/)
-
18
s << scanner.getch
-
35
elsif scanner.scan(/[^"\\]+/)
-
36
s << scanner.matched
-
end
-
end
-
-
18
s
-
end
-
-
23
def scan_value(scanner, comma_as_separator = false)
-
495
value = +""
-
-
553
until scanner.eos?
-
855
if scanner.scan(/[^,;"]+/)
-
486
value << scanner.matched
-
368
elsif scanner.skip(/"/)
-
# RFC 6265 2.2
-
# A cookie-value may be DQUOTE'd.
-
18
value << scan_dquoted(scanner)
-
350
elsif scanner.check(/;/)
-
261
break
-
89
elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
-
72
break
-
else
-
18
value << scanner.getch
-
end
-
end
-
-
495
value.rstrip!
-
495
value
-
end
-
-
23
def scan_name_value(scanner, comma_as_separator = false)
-
495
name = scanner.scan(RE_NAME)
-
495
name.rstrip! if name
-
-
495
if scanner.skip(/=/)
-
486
value = scan_value(scanner, comma_as_separator)
-
else
-
9
scan_value(scanner, comma_as_separator)
-
9
value = nil
-
end
-
495
[name, value]
-
end
-
-
23
def call(set_cookie)
-
162
scanner = StringScanner.new(set_cookie)
-
-
# RFC 6265 4.1.1 & 5.2
-
188
until scanner.eos?
-
234
start = scanner.pos
-
234
len = nil
-
-
234
scanner.skip(RE_WSP)
-
-
234
name, value = scan_name_value(scanner, true)
-
234
value = nil if name && name.empty?
-
-
234
attrs = {}
-
-
263
until scanner.eos?
-
333
if scanner.skip(/,/)
-
# The comma is used as separator for concatenating multiple
-
# values of a header.
-
72
len = (scanner.pos - 1) - start
-
72
break
-
260
elsif scanner.skip(/;/)
-
261
scanner.skip(RE_WSP)
-
-
261
aname, avalue = scan_name_value(scanner, true)
-
-
261
next if (aname.nil? || aname.empty?) || value.nil?
-
-
261
aname.downcase!
-
-
232
case aname
-
when "expires"
-
18
next unless avalue
-
-
# RFC 6265 5.2.1
-
18
(avalue = Time.parse(avalue)) || next
-
when "max-age"
-
9
next unless avalue
-
# RFC 6265 5.2.2
-
9
next unless /\A-?\d+\z/.match?(avalue)
-
-
9
avalue = Integer(avalue)
-
when "domain"
-
# RFC 6265 5.2.3
-
# An empty value SHOULD be ignored.
-
27
next if avalue.nil? || avalue.empty?
-
when "path"
-
# RFC 6265 5.2.4
-
# A relative path must be ignored rather than normalizing it
-
# to "/".
-
198
next unless avalue && avalue.start_with?("/")
-
when "secure", "httponly"
-
# RFC 6265 5.2.5, 5.2.6
-
8
avalue = true
-
end
-
232
attrs[aname] = avalue
-
end
-
end
-
-
234
len ||= scanner.pos - start
-
-
234
next if len > Cookie::MAX_LENGTH
-
-
234
yield(name, value, attrs) if name && !name.empty? && value
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
-
#
-
# https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
-
#
-
23
module DigestAuth
-
23
class << self
-
23
def extra_options(options)
-
180
options.merge(max_concurrent_requests: 1)
-
end
-
-
23
def load_dependencies(klass)
-
180
require_relative "auth/digest"
-
180
klass.plugin(:auth)
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :digest :: instance of HTTPX::Plugins::Authentication::Digest, used to authenticate requests in the session.
-
23
module OptionsMethods
-
23
private
-
-
23
def option_digest(value)
-
360
raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)
-
-
360
value
-
end
-
end
-
-
23
module InstanceMethods
-
23
def digest_auth(user, password, hashed: false)
-
180
with(digest: Authentication::Digest.new(user, password, hashed: hashed))
-
end
-
-
23
private
-
-
23
def send_requests(*requests)
-
216
requests.flat_map do |request|
-
216
digest = request.options.digest
-
-
216
next super(request) unless digest
-
-
360
probe_response = wrap { super(request).first }
-
-
180
return [probe_response] * requests.size unless probe_response.is_a?(Response)
-
-
180
if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
-
162
request.transition(:idle)
-
162
request.authorize(digest.authenticate(request, probe_response.headers["www-authenticate"]))
-
162
super(request)
-
else
-
18
probe_response
-
end
-
end
-
end
-
end
-
end
-
-
23
register_plugin :digest_auth, DigestAuth
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
-
#
-
# https://gitlab.com/os85/httpx/wikis/Expect#expect
-
#
-
23
module Expect
-
23
EXPECT_TIMEOUT = 2
-
23
NOEXPECT_STORE_MUTEX = Thread::Mutex.new
-
-
23
class Store
-
23
def initialize
-
10
@store = []
-
10
@mutex = Thread::Mutex.new
-
end
-
-
23
def include?(host)
-
310
@mutex.synchronize { @store.include?(host) }
-
end
-
-
23
def add(host)
-
36
@mutex.synchronize { @store << host }
-
end
-
-
23
def delete(host)
-
18
@mutex.synchronize { @store.delete(host) }
-
end
-
end
-
-
23
class << self
-
23
def no_expect_store
-
182
return Ractor.store_if_absent(:httpx_no_expect_store) { Store.new } if Utils.in_ractor?
-
-
182
@no_expect_store ||= NOEXPECT_STORE_MUTEX.synchronize do
-
10
@no_expect_store || Store.new
-
end
-
end
-
-
23
def extra_options(options)
-
227
options.merge(expect_timeout: EXPECT_TIMEOUT)
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :expect_timeout :: time (in seconds) to wait for a 100-expect response,
-
# before retrying without the Expect header (defaults to <tt>2</tt>).
-
# :expect_threshold_size :: min threshold (in bytes) of the request payload to enable the 100-continue negotiation on.
-
23
module OptionsMethods
-
23
private
-
-
23
def option_expect_timeout(value)
-
416
seconds = Float(value)
-
416
raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
-
-
416
seconds
-
end
-
-
23
def option_expect_threshold_size(value)
-
18
bytes = Integer(value)
-
18
raise TypeError, ":expect_threshold_size must be positive" unless bytes.positive?
-
-
18
bytes
-
end
-
end
-
-
23
module RequestMethods
-
23
def initialize(*)
-
263
super
-
-
263
@informational_status = nil
-
-
263
return if @body.empty?
-
-
173
threshold = @options.expect_threshold_size
-
173
return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
-
-
155
return if Expect.no_expect_store.include?(origin)
-
-
130
@headers["expect"] = "100-continue"
-
end
-
-
23
def response=(response)
-
159
if response.is_a?(Response) &&
-
response.status == 100 &&
-
!@headers.key?("expect") &&
-
4
(@state == :body || @state == :done)
-
-
# if we're past this point, this means that we just received a 100-Continue response,
-
# but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
-
#
-
# this means that expect was deactivated for this request too soon, i.e. response took longer.
-
#
-
# so we have to reactivate it again.
-
8
@headers["expect"] = "100-continue"
-
9
@informational_status = 100
-
9
Expect.no_expect_store.delete(origin)
-
end
-
159
super
-
end
-
end
-
-
23
module ConnectionMethods
-
23
def send_request_to_parser(request)
-
94
super
-
-
94
return unless request.headers["expect"] == "100-continue"
-
-
74
expect_timeout = request.options.expect_timeout
-
-
74
return if expect_timeout.nil? || expect_timeout.infinite?
-
-
74
set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
-
# expect timeout expired
-
18
if request.state == :expect && !request.expects?
-
18
Expect.no_expect_store.add(request.origin)
-
18
request.headers.delete("expect")
-
18
consume
-
end
-
end
-
end
-
end
-
-
23
module InstanceMethods
-
23
def fetch_response(request, selector, options)
-
188
response = super
-
-
188
return unless response
-
-
94
if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
-
2
response.close
-
2
request.headers.delete("expect")
-
2
request.transition(:idle)
-
2
send_request(request, selector, options)
-
-
# recalling itself, in case an error was triggered by the above, and we can
-
# verify retriability again.
-
2
return fetch_response(request, selector, options)
-
end
-
-
92
response
-
end
-
end
-
end
-
23
register_plugin :expect, Expect
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Plugins
-
# This plugin makes a session reuse the same selector across all fibers in a given thread.
-
#
-
# This enables integration with fiber scheduler implementations such as [async](https://github.com/async).
-
#
-
# # https://gitlab.com/os85/httpx/wikis/Fiber-Concurrency
-
#
-
30
module FiberConcurrency
-
30
def self.subplugins
-
124
{
-
2072
h2c: FiberConcurrencyH2C,
-
stream: FiberConcurrencyStream,
-
}
-
end
-
-
30
module InstanceMethods
-
30
private
-
-
30
def send_request(request, *)
-
846
request.set_context!
-
-
846
super
-
end
-
-
30
def get_current_selector
-
697
super(&nil) || begin
-
598
return unless block_given?
-
-
598
default = yield
-
-
598
set_current_selector(default)
-
-
598
default
-
end
-
end
-
end
-
-
30
module RequestMethods
-
# the execution context (fiber) this request was sent on.
-
30
attr_reader :context
-
-
30
def initialize(*)
-
737
super
-
737
@context = nil
-
end
-
-
# sets the execution context for this request. the default is the current fiber.
-
30
def set_context!
-
1706
@context ||= Fiber.current # rubocop:disable Naming/MemoizedInstanceVariableName
-
end
-
-
# checks whether the current execution context is the one where the request was created.
-
30
def current_context?
-
4328
@context == Fiber.current
-
end
-
-
30
def complete!(response = @response)
-
706
@context = nil
-
706
super
-
end
-
end
-
-
30
module ConnectionMethods
-
30
def current_context?
-
667
@pending.any?(&:current_context?) || (
-
149
@sibling && @sibling.pending.any?(&:current_context?)
-
)
-
end
-
-
30
def interests
-
12118
return if connecting? && @pending.none?(&:current_context?)
-
-
12046
super
-
end
-
-
30
def send(request)
-
# DoH requests bypass the session, so context needs to be set here.
-
860
request.set_context!
-
-
860
super
-
end
-
end
-
-
30
module HTTP1Methods
-
30
def interests
-
1448
request = @request || @requests.first
-
-
1448
return unless request
-
-
1428
return unless request.current_context? || @requests.any?(&:current_context?) || @pending.any?(&:current_context?)
-
-
1419
super
-
end
-
end
-
-
30
module HTTP2Methods
-
30
def initialize(*)
-
457
super
-
1388
@contexts = Hash.new { |hs, k| hs[k] = Set.new }
-
end
-
-
30
def interests
-
9336
if @connection.state == :connected && @handshake_completed && !@contexts.key?(Fiber.current)
-
561
return :w unless @pings.empty?
-
-
492
return
-
end
-
-
8775
super
-
end
-
-
30
def send(request, *)
-
1026
add_to_context(request)
-
-
1026
super
-
end
-
-
30
private
-
-
30
def on_close(_, error, _)
-
20
if error == :http_1_1_required
-
# remove all pending requests context
-
@pending.each do |req|
-
clear_from_context(req)
-
end
-
end
-
-
20
super
-
end
-
-
30
def on_stream_close(_, request, error)
-
517
clear_from_context(request) if error != :stream_closed && @streams.key?(request)
-
-
517
super
-
end
-
-
30
def teardown(request = nil)
-
509
super
-
-
509
if request
-
489
clear_from_context(request)
-
else
-
20
@contexts.clear
-
end
-
end
-
-
30
def add_to_context(request)
-
1026
@contexts[request.context] << request
-
end
-
-
30
def clear_from_context(request)
-
978
requests = @contexts[request.context]
-
-
978
requests.delete(request)
-
-
978
@contexts.delete(request.context) if requests.empty?
-
end
-
end
-
-
30
module ResolverNativeMethods
-
30
def calculate_interests
-
578
return if @queries.empty?
-
-
538
return unless @queries.values.any?(&:current_context?) || @connections.any?(&:current_context?)
-
-
518
super
-
end
-
end
-
-
30
module ResolverSystemMethods
-
30
def interests
-
return unless @queries.any? { |_, conn| conn.current_context? }
-
-
super
-
end
-
end
-
-
30
module FiberConcurrencyH2C
-
30
module HTTP2Methods
-
30
def upgrade(request, *)
-
@contexts[request.context] << request
-
-
super
-
end
-
end
-
end
-
-
30
module FiberConcurrencyStream
-
30
module StreamResponseMethods
-
30
def close
-
9
unless @request.current_context?
-
9
@request.close
-
-
9
return
-
end
-
-
super
-
end
-
end
-
end
-
end
-
-
30
register_plugin :fiber_concurrency, FiberConcurrency
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
class InsecureRedirectError < Error
-
end
-
-
30
module Plugins
-
#
-
# This plugin adds support for automatically following redirect (status 30X) responses.
-
#
-
# It has a default upper bound of followed redirects (see *MAX_REDIRECTS* and the *max_redirects* option),
-
# after which it will return the last redirect response. It will **not** raise an exception.
-
#
-
# It doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
-
#
-
# It doesn't propagate authorization related headers to requests redirecting to different origins
-
# (see *allow_auth_to_other_origins*) to override.
-
#
-
# It allows customization of when to redirect via the *redirect_on* callback option).
-
#
-
# https://gitlab.com/os85/httpx/wikis/Follow-Redirects
-
#
-
30
module FollowRedirects
-
30
MAX_REDIRECTS = 3
-
30
REDIRECT_STATUS = (300..399).freeze
-
30
REQUEST_BODY_HEADERS = %w[transfer-encoding content-encoding content-type content-length content-language content-md5 trailer].freeze
-
-
30
using URIExtensions
-
-
# adds support for the following options:
-
#
-
# :max_redirects :: max number of times a request will be redirected (defaults to <tt>3</tt>).
-
# :follow_insecure_redirects :: whether redirects to an "http://" URI, when coming from an "https//", are allowed
-
# (defaults to <tt>false</tt>).
-
# :allow_auth_to_other_origins :: whether auth-related headers, such as "Authorization", are propagated on redirection
-
# (defaults to <tt>false</tt>).
-
# :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
-
30
module OptionsMethods
-
30
private
-
-
30
def option_max_redirects(value)
-
520
num = Integer(value)
-
520
raise TypeError, ":max_redirects must be positive" if num.negative?
-
-
520
num
-
end
-
-
30
def option_follow_insecure_redirects(value)
-
27
value
-
end
-
-
30
def option_allow_auth_to_other_origins(value)
-
27
value
-
end
-
-
30
def option_redirect_on(value)
-
54
raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
-
-
54
value
-
end
-
end
-
-
30
module InstanceMethods
-
# returns a session with the *max_redirects* option set to +n+
-
30
def max_redirects(n)
-
54
with(max_redirects: n.to_i)
-
end
-
-
30
private
-
-
30
def fetch_response(request, selector, options)
-
1248
redirect_request = request.redirect_request
-
1248
response = super(redirect_request, selector, options)
-
1248
return unless response
-
-
640
max_redirects = redirect_request.max_redirects
-
-
640
return response unless response.is_a?(Response)
-
622
return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
-
407
return response unless max_redirects.positive?
-
-
371
redirect_uri = __get_location_from_response(response)
-
-
371
if options.redirect_on
-
36
redirect_allowed = options.redirect_on.call(redirect_uri)
-
36
return response unless redirect_allowed
-
end
-
-
# build redirect request
-
353
request_body = redirect_request.body
-
353
redirect_method = "GET"
-
353
redirect_params = {}
-
-
353
if response.status == 305 && options.respond_to?(:proxy)
-
9
request_body.rewind
-
# The requested resource MUST be accessed through the proxy given by
-
# the Location field. The Location field gives the URI of the proxy.
-
9
redirect_options = options.merge(headers: redirect_request.headers,
-
proxy: { uri: redirect_uri },
-
max_redirects: max_redirects - 1)
-
-
8
redirect_params[:body] = request_body
-
9
redirect_uri = redirect_request.uri
-
9
options = redirect_options
-
else
-
344
redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
-
344
redirect_opts = Hash[options]
-
307
redirect_params[:max_redirects] = max_redirects - 1
-
-
344
unless request_body.empty?
-
27
if response.status == 307
-
# The method and the body of the original request are reused to perform the redirected request.
-
9
redirect_method = redirect_request.verb
-
9
request_body.rewind
-
8
redirect_params[:body] = request_body
-
else
-
# redirects are **ALWAYS** GET, so remove body-related headers
-
18
REQUEST_BODY_HEADERS.each do |h|
-
126
redirect_headers.delete(h)
-
end
-
16
redirect_params[:body] = nil
-
end
-
end
-
-
344
options = options.class.new(redirect_opts.merge(headers: redirect_headers.to_h))
-
end
-
-
353
redirect_uri = Utils.to_uri(redirect_uri)
-
-
353
if !options.follow_insecure_redirects &&
-
response.uri.scheme == "https" &&
-
redirect_uri.scheme == "http"
-
9
error = InsecureRedirectError.new(redirect_uri.to_s)
-
9
error.set_backtrace(caller)
-
8
return ErrorResponse.new(request, error)
-
end
-
-
344
retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
-
-
344
request.redirect_request = retry_request
-
-
344
redirect_after = response.headers["retry-after"]
-
-
344
if redirect_after
-
# Servers send the "Retry-After" header field to indicate how long the
-
# user agent ought to wait before making a follow-up request.
-
# When sent with any 3xx (Redirection) response, Retry-After indicates
-
# the minimum time that the user agent is asked to wait before issuing
-
# the redirected request.
-
#
-
36
redirect_after = Utils.parse_retry_after(redirect_after)
-
-
36
retry_start = Utils.now
-
36
log { "redirecting after #{redirect_after} secs..." }
-
36
selector.after(redirect_after) do
-
36
if (response = request.response)
-
18
response.finish!
-
18
retry_request.response = response
-
# request has terminated abruptly meanwhile
-
18
retry_request.emit_response(response)
-
else
-
18
log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-
18
send_request(retry_request, selector, options)
-
end
-
end
-
else
-
308
send_request(retry_request, selector, options)
-
-
# recalling itself, in case an error was triggered by the above, and we can
-
# verify retriability again.
-
308
return fetch_response(request, selector, options)
-
end
-
16
nil
-
end
-
-
# :nodoc:
-
30
def redirect_request_headers(original_uri, redirect_uri, headers, options)
-
344
headers = headers.dup
-
-
344
return headers if options.allow_auth_to_other_origins
-
-
335
return headers unless headers.key?("authorization")
-
-
9
return headers if original_uri.origin == redirect_uri.origin
-
-
9
headers.delete("authorization")
-
-
9
headers
-
end
-
-
# :nodoc:
-
30
def __get_location_from_response(response)
-
# @type var location_uri: http_uri
-
371
location_uri = URI(response.headers["location"])
-
371
location_uri = response.uri.merge(location_uri) if location_uri.relative?
-
371
location_uri
-
end
-
end
-
-
30
module RequestMethods
-
# returns the top-most original HTTPX::Request from the redirect chain
-
30
attr_accessor :root_request
-
-
30
def initialize(*)
-
640
super
-
640
@redirect_request = nil
-
end
-
-
30
def on_response_arrived=(cb)
-
1184
@redirect_request.on_response_arrived = cb if @redirect_request
-
-
1184
super
-
end
-
-
# returns the follow-up redirect request, or itself
-
30
def redirect_request
-
1248
@redirect_request || self
-
end
-
-
# sets the follow-up redirect request
-
30
def redirect_request=(req)
-
344
@redirect_request = req
-
344
req.root_request = @root_request || self
-
344
req.on_response_arrived = @on_response_arrived
-
344
@response = nil
-
end
-
-
30
def response
-
3517
return super unless @redirect_request && @response.nil?
-
-
86
@redirect_request.response
-
end
-
-
30
def max_redirects
-
640
@options.max_redirects || MAX_REDIRECTS
-
end
-
end
-
-
30
module ConnectionMethods
-
30
private
-
-
30
def set_request_request_timeout(request)
-
608
return unless request.root_request.nil?
-
-
289
super
-
end
-
end
-
end
-
30
register_plugin :follow_redirects, FollowRedirects
-
end
-
end
-
# frozen_string_literal: true
-
-
21
module HTTPX
-
21
GRPCError = Class.new(Error) do
-
21
attr_reader :status, :details, :metadata
-
-
21
def initialize(status, details, metadata)
-
28
@status = status
-
28
@details = details
-
28
@metadata = metadata
-
28
super("GRPC error, code=#{status}, details=#{details}, metadata=#{metadata}")
-
end
-
end
-
-
21
module Plugins
-
#
-
# This plugin adds DSL to build GRPC interfaces.
-
#
-
# https://gitlab.com/os85/httpx/wikis/GRPC
-
#
-
21
module GRPC
-
21
unless String.method_defined?(:underscore)
-
21
module StringExtensions
-
21
refine String do
-
21
def underscore
-
364
s = dup # Avoid mutating the argument, as it might be frozen.
-
364
s.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
-
364
s.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
-
364
s.tr!("-", "_")
-
364
s.downcase!
-
364
s
-
end
-
end
-
end
-
21
using StringExtensions
-
end
-
-
21
DEADLINE = 60
-
21
MARSHAL_METHOD = :encode
-
21
UNMARSHAL_METHOD = :decode
-
21
HEADERS = {
-
"content-type" => "application/grpc",
-
"te" => "trailers",
-
"accept" => "application/grpc",
-
# metadata fits here
-
# ex "foo-bin" => base64("bar")
-
}.freeze
-
-
21
class << self
-
21
def load_dependencies(*)
-
161
require "stringio"
-
161
require "httpx/plugins/grpc/grpc_encoding"
-
161
require "httpx/plugins/grpc/message"
-
161
require "httpx/plugins/grpc/call"
-
end
-
-
21
def configure(klass)
-
161
klass.plugin(:persistent)
-
161
klass.plugin(:stream)
-
end
-
-
21
def extra_options(options)
-
161
options.merge(
-
fallback_protocol: "h2",
-
grpc_rpcs: {}.freeze,
-
grpc_compression: false,
-
grpc_deadline: DEADLINE
-
)
-
end
-
end
-
-
21
module OptionsMethods
-
21
private
-
-
21
def option_grpc_service(value)
-
140
String(value)
-
end
-
-
21
def option_grpc_compression(value)
-
350
case value
-
when true, false
-
322
value
-
else
-
28
value.to_s
-
end
-
end
-
-
21
def option_grpc_rpcs(value)
-
1463
Hash[value]
-
end
-
-
21
def option_grpc_deadline(value)
-
1099
raise TypeError, ":grpc_deadline must be positive" unless value.positive?
-
-
1099
value
-
end
-
-
21
def option_call_credentials(value)
-
21
raise TypeError, ":call_credentials must respond to #call" unless value.respond_to?(:call)
-
-
21
value
-
end
-
end
-
-
21
module ResponseMethods
-
21
attr_reader :trailing_metadata
-
-
21
def merge_headers(trailers)
-
133
@trailing_metadata = Hash[trailers]
-
133
super
-
end
-
end
-
-
21
module RequestBodyMethods
-
21
def initialize(*, **)
-
147
super
-
-
147
if (compression = @headers["grpc-encoding"])
-
14
deflater_body = self.class.initialize_deflater_body(@body, compression)
-
14
@body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
-
else
-
133
@body = Transcoder::GRPCEncoding.encode(@body, compressed: false)
-
end
-
end
-
end
-
-
21
module InstanceMethods
-
21
def with_channel_credentials(ca_path, key = nil, cert = nil, **ssl_opts)
-
# @type var ssl_params: ::Hash[::Symbol, untyped]
-
84
ssl_params = {
-
**ssl_opts,
-
ca_file: ca_path,
-
}
-
84
if key
-
84
key = File.read(key) if File.file?(key)
-
84
ssl_params[:key] = OpenSSL::PKey.read(key)
-
end
-
-
84
if cert
-
84
cert = File.read(cert) if File.file?(cert)
-
84
ssl_params[:cert] = OpenSSL::X509::Certificate.new(cert)
-
end
-
-
84
with(ssl: ssl_params)
-
end
-
-
21
def rpc(rpc_name, input, output, **opts)
-
364
rpc_name = rpc_name.to_s
-
364
raise Error, "rpc #{rpc_name} already defined" if @options.grpc_rpcs.key?(rpc_name)
-
-
rpc_opts = {
-
364
deadline: @options.grpc_deadline,
-
}.merge(opts)
-
-
364
local_rpc_name = rpc_name.underscore
-
-
364
session_class = Class.new(self.class) do
-
# define rpc method with ruby style name
-
364
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
def #{local_rpc_name}(input, **opts) # def grpc_action(input, **opts)
-
rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
-
end # end
-
OUT
-
-
# define rpc method with original name
-
364
unless local_rpc_name == rpc_name
-
14
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
def #{rpc_name}(input, **opts) # def grpcAction(input, **opts)
-
rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
-
end # end
-
OUT
-
end
-
end
-
-
364
session_class.new(@options.merge(
-
grpc_rpcs: @options.grpc_rpcs.merge(
-
local_rpc_name => [rpc_name, input, output, rpc_opts]
-
).freeze
-
))
-
end
-
-
21
def build_stub(origin, service: nil, compression: false)
-
161
scheme = @options.ssl.empty? ? "http" : "https"
-
-
161
origin = URI.parse("#{scheme}://#{origin}")
-
-
161
session = self
-
-
161
if service && service.respond_to?(:rpc_descs)
-
# it's a grpc generic service
-
70
service.rpc_descs.each do |rpc_name, rpc_desc|
-
rpc_opts = {
-
350
marshal_method: rpc_desc.marshal_method,
-
unmarshal_method: rpc_desc.unmarshal_method,
-
}
-
-
350
input = rpc_desc.input
-
350
input = input.type if input.respond_to?(:type)
-
-
350
output = rpc_desc.output
-
350
if output.respond_to?(:type)
-
140
rpc_opts[:stream] = true
-
140
output = output.type
-
end
-
-
350
session = session.rpc(rpc_name, input, output, **rpc_opts)
-
end
-
-
70
service = service.service_name
-
end
-
-
161
session.with(origin: origin, grpc_service: service, grpc_compression: compression)
-
end
-
-
21
def execute(rpc_method, input,
-
deadline: DEADLINE,
-
metadata: nil,
-
**opts)
-
147
grpc_request = build_grpc_request(rpc_method, input, deadline: deadline, metadata: metadata, **opts)
-
147
response = request(grpc_request, **opts)
-
147
response.raise_for_status unless opts[:stream]
-
133
GRPC::Call.new(response)
-
end
-
-
21
private
-
-
21
def rpc_execute(rpc_name, input, **opts)
-
70
rpc_name, input_enc, output_enc, rpc_opts = @options.grpc_rpcs[rpc_name]
-
-
70
exec_opts = rpc_opts.merge(opts)
-
-
70
marshal_method ||= exec_opts.delete(:marshal_method) || MARSHAL_METHOD
-
70
unmarshal_method ||= exec_opts.delete(:unmarshal_method) || UNMARSHAL_METHOD
-
-
70
messages = if input.respond_to?(:each)
-
28
Enumerator.new do |y|
-
28
input.each do |message|
-
56
y << input_enc.__send__(marshal_method, message)
-
end
-
end
-
else
-
42
input_enc.__send__(marshal_method, input)
-
end
-
-
70
call = execute(rpc_name, messages, **exec_opts)
-
-
70
call.decoder = output_enc.method(unmarshal_method)
-
-
70
call
-
end
-
-
21
def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **opts)
-
147
uri = @options.origin.dup
-
147
rpc_method = "/#{rpc_method}" unless rpc_method.start_with?("/")
-
147
rpc_method = "/#{@options.grpc_service}#{rpc_method}" if @options.grpc_service
-
147
uri.path = rpc_method
-
-
147
headers = HEADERS.merge(
-
"grpc-accept-encoding" => ["identity", *@options.supported_compression_formats]
-
)
-
147
unless deadline == Float::INFINITY
-
# convert to milliseconds
-
147
deadline = (deadline * 1000.0).to_i
-
147
headers["grpc-timeout"] = "#{deadline}m"
-
end
-
-
147
headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
-
-
# prepare compressor
-
147
compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
-
-
147
headers["grpc-encoding"] = compression if compression
-
-
147
headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
-
-
147
build_request("POST", uri, headers: headers, body: input, **opts)
-
end
-
end
-
end
-
21
register_plugin :grpc, GRPC
-
end
-
end
-
# frozen_string_literal: true
-
-
21
module HTTPX
-
21
module Plugins
-
21
module GRPC
-
# Encapsulates call information
-
21
class Call
-
21
attr_writer :decoder
-
-
21
def initialize(response)
-
133
@response = response
-
182
@decoder = ->(z) { z }
-
133
@consumed = false
-
133
@grpc_response = nil
-
end
-
-
21
def inspect
-
"#{self.class}(#{grpc_response})"
-
end
-
-
21
def to_s
-
77
grpc_response.to_s
-
end
-
-
21
def metadata
-
response.headers
-
end
-
-
21
def trailing_metadata
-
84
return unless @consumed
-
-
56
@response.trailing_metadata
-
end
-
-
21
private
-
-
21
def grpc_response
-
217
@grpc_response ||= if @response.respond_to?(:each)
-
28
Enumerator.new do |y|
-
28
Message.stream(@response).each do |message|
-
56
y << @decoder.call(message)
-
end
-
28
@consumed = true
-
end
-
else
-
105
@consumed = true
-
105
@decoder.call(Message.unary(@response))
-
end
-
end
-
-
21
def respond_to_missing?(meth, *args, &blk)
-
28
grpc_response.respond_to?(meth, *args) || super
-
end
-
-
21
def method_missing(meth, *args, &blk)
-
56
return grpc_response.__send__(meth, *args, &blk) if grpc_response.respond_to?(meth)
-
-
super
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
21
module HTTPX
-
21
module Transcoder
-
21
module GRPCEncoding
-
21
class Deflater
-
21
extend Forwardable
-
-
21
attr_reader :content_type
-
-
21
def initialize(body, compressed:)
-
147
@content_type = body.content_type
-
147
@body = BodyReader.new(body)
-
147
@compressed = compressed
-
end
-
-
21
def bytesize
-
525
return @body.bytesize if @body.respond_to?(:bytesize)
-
-
Float::INFINITY
-
end
-
-
21
def read(length = nil, outbuf = nil)
-
322
buf = @body.read(length, outbuf)
-
-
294
return unless buf
-
-
161
compressed_flag = @compressed ? 1 : 0
-
-
161
buf = outbuf if outbuf
-
-
161
buf = buf.b if buf.frozen?
-
-
161
buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
-
161
buf
-
end
-
end
-
-
21
class Inflater
-
21
def initialize(response)
-
105
@response = response
-
105
@grpc_encodings = nil
-
end
-
-
21
def call(message, &blk)
-
133
data = "".b
-
-
133
until message.empty?
-
133
compressed, size = message.unpack("CL>")
-
-
133
encoded_data = message.byteslice(5..(size + 5 - 1))
-
-
133
if compressed == 1
-
14
grpc_encodings.reverse_each do |encoding|
-
14
decoder = @response.body.class.initialize_inflater_by_encoding(encoding, @response, bytesize: encoded_data.bytesize)
-
14
encoded_data = decoder.call(encoded_data)
-
-
14
blk.call(encoded_data) if blk
-
-
14
data << encoded_data
-
end
-
else
-
119
blk.call(encoded_data) if blk
-
-
119
data << encoded_data
-
end
-
-
133
message = message.byteslice((size + 5)..-1)
-
end
-
-
133
data
-
end
-
-
21
private
-
-
21
def grpc_encodings
-
14
@grpc_encodings ||= @response.headers.get("grpc-encoding")
-
end
-
end
-
-
21
def self.encode(*args, **kwargs)
-
147
Deflater.new(*args, **kwargs)
-
end
-
-
21
def self.decode(response)
-
105
Inflater.new(response)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
21
module HTTPX
-
21
module Plugins
-
21
module GRPC
-
# Encoding module for GRPC responses
-
#
-
# Can encode and decode grpc messages.
-
21
module Message
-
21
module_function
-
-
# decodes a unary grpc response
-
21
def unary(response)
-
105
verify_status(response)
-
-
77
decoder = Transcoder::GRPCEncoding.decode(response)
-
-
77
decoder.call(response.to_s)
-
end
-
-
# lazy decodes a grpc stream response
-
21
def stream(response, &block)
-
56
return enum_for(__method__, response) unless block
-
-
28
decoder = Transcoder::GRPCEncoding.decode(response)
-
-
28
response.each do |frame|
-
56
decoder.call(frame, &block)
-
end
-
-
28
verify_status(response)
-
end
-
-
21
def cancel(request)
-
request.emit(:refuse, :client_cancellation)
-
end
-
-
# interprets the grpc call trailing metadata, and raises an
-
# exception in case of error code
-
21
def verify_status(response)
-
# return standard errors if need be
-
133
response.raise_for_status
-
-
133
status = Integer(response.headers["grpc-status"])
-
133
message = response.headers["grpc-message"]
-
-
133
return if status.zero?
-
-
28
response.close
-
28
raise GRPCError.new(status, message, response.trailing_metadata)
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Plugins
-
#
-
# This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
-
# (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
-
#
-
# https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
-
#
-
30
module H2C
-
30
VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
-
-
30
class << self
-
30
def load_dependencies(klass)
-
18
klass.plugin(:upgrade)
-
end
-
-
30
def call(connection, request, response)
-
18
connection.upgrade_to_h2c(request, response)
-
end
-
-
30
def extra_options(options)
-
20
options.merge(
-
18
h2c_class: Class.new(options.http2_class) { include(H2CParser) },
-
max_concurrent_requests: 1,
-
upgrade_handlers: options.upgrade_handlers.merge("h2c" => self),
-
)
-
end
-
end
-
-
30
module OptionsMethods
-
30
def option_h2c_class(value)
-
18
value
-
end
-
end
-
-
30
module RequestMethods
-
30
def valid_h2c_verb?
-
18
VALID_H2C_VERBS.include?(@verb)
-
end
-
end
-
-
30
module ConnectionMethods
-
30
using URIExtensions
-
-
30
def initialize(*)
-
18
super
-
18
@h2c_handshake = false
-
end
-
-
30
def send(request)
-
45
return super if @h2c_handshake
-
-
18
return super unless request.valid_h2c_verb? && request.scheme == "http"
-
-
18
return super if @upgrade_protocol == "h2c"
-
-
18
@h2c_handshake = true
-
-
# build upgrade request
-
18
request.headers.add("connection", "upgrade")
-
18
request.headers.add("connection", "http2-settings")
-
16
request.headers["upgrade"] = "h2c"
-
16
request.headers["http2-settings"] = ::HTTP2::Client.settings_header(request.options.http2_settings)
-
-
18
super
-
end
-
-
30
def upgrade_to_h2c(request, response)
-
18
enqueue_pending_requests_from_parser(@parser)
-
-
18
@parser = request.options.h2c_class.new(@write_buffer, @options)
-
18
set_parser_callbacks(@parser)
-
16
@inflight += 1 # request is being completed below
-
18
@parser.upgrade(request, response)
-
18
@upgrade_protocol = "h2c"
-
end
-
-
30
private
-
-
30
def send_request_to_parser(request)
-
63
super
-
-
63
return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
-
-
18
max_concurrent_requests = parser.max_concurrent_requests
-
-
18
return if max_concurrent_requests == 1
-
-
parser.max_concurrent_requests = 1
-
request.once(:response) do
-
parser.max_concurrent_requests = max_concurrent_requests
-
end
-
end
-
end
-
-
30
module H2CParser
-
30
def upgrade(request, response)
-
# skip checks, it is assumed that this is the first
-
# request in the connection
-
18
stream = @connection.upgrade
-
-
# on_settings
-
18
handle_stream(stream, request)
-
16
@streams[request] = stream
-
-
# clean up data left behind in the buffer, if the server started
-
# sending frames
-
18
data = response.read
-
18
@connection << data
-
end
-
end
-
end
-
30
register_plugin(:h2c, H2C)
-
end
-
end
-
# frozen_string_literal: true
-
-
17
module HTTPX
-
17
module Plugins
-
#
-
# https://gitlab.com/os85/httpx/wikis/Auth#ntlm-auth
-
#
-
17
module NTLMAuth
-
17
class << self
-
17
def load_dependencies(klass)
-
2
require_relative "auth/ntlm"
-
2
klass.plugin(:auth)
-
end
-
-
17
def extra_options(options)
-
2
options.merge(max_concurrent_requests: 1)
-
end
-
end
-
-
17
module OptionsMethods
-
17
private
-
-
17
def option_ntlm(value)
-
8
raise TypeError, ":ntlm must be a #{Authentication::Ntlm}" unless value.is_a?(Authentication::Ntlm)
-
-
8
value
-
end
-
end
-
-
17
module InstanceMethods
-
17
def ntlm_auth(user, password, domain = nil)
-
4
with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
-
end
-
-
17
private
-
-
17
def send_requests(*requests)
-
8
requests.flat_map do |request|
-
8
ntlm = request.options.ntlm
-
-
8
if ntlm
-
4
request.authorize(ntlm.negotiate)
-
8
probe_response = wrap { super(request).first }
-
-
4
return probe_response unless probe_response.is_a?(Response)
-
-
4
if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
-
2
request.transition(:idle)
-
2
request.unauthorize!
-
2
request.authorize(ntlm.authenticate(request, probe_response.headers["www-authenticate"]).encode("utf-8"))
-
2
super(request)
-
else
-
2
probe_response
-
end
-
else
-
4
super(request)
-
end
-
end
-
end
-
end
-
end
-
17
register_plugin :ntlm_auth, NTLMAuth
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds support for managing an OAuth Session associated with the given session.
-
#
-
# The scope of OAuth support is limited to the `client_crendentials` and `refresh_token` grants.
-
#
-
# https://gitlab.com/os85/httpx/wikis/OAuth
-
#
-
23
module OAuth
-
23
class << self
-
23
def load_dependencies(klass)
-
270
require_relative "auth/basic"
-
270
klass.plugin(:auth)
-
end
-
-
23
def subplugins
-
60
{
-
479
retries: OAuthRetries,
-
}
-
end
-
-
23
def extra_options(options)
-
270
options.merge(auth_header_type: "Bearer")
-
end
-
end
-
-
23
SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
-
23
SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
-
-
# Implements the bulk of functionality and maintains the state associated with the
-
# management of the the lifecycle of an OAuth session.
-
23
class OAuthSession
-
23
attr_reader :access_token, :refresh_token
-
-
23
def initialize(
-
issuer:,
-
client_id:,
-
client_secret:,
-
access_token: nil,
-
refresh_token: nil,
-
scope: nil,
-
audience: nil,
-
token_endpoint: nil,
-
grant_type: nil,
-
token_endpoint_auth_method: nil
-
)
-
270
@issuer = URI(issuer)
-
270
@client_id = client_id
-
270
@client_secret = client_secret
-
270
@token_endpoint = URI(token_endpoint) if token_endpoint
-
270
@scope = case scope
-
when String
-
162
scope.split
-
when Array
-
36
scope
-
end
-
270
@audience = audience
-
270
@access_token = access_token
-
270
@refresh_token = refresh_token
-
270
@token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
-
270
@grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
-
270
@access_token = access_token
-
270
@refresh_token = refresh_token
-
-
270
unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
-
18
raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
-
end
-
-
252
return if SUPPORTED_GRANT_TYPES.include?(@grant_type)
-
-
18
raise Error, "#{@grant_type} is not a supported grant type"
-
end
-
-
# returns the URL where to request access and refresh tokens from.
-
23
def token_endpoint
-
252
@token_endpoint || "#{@issuer}/token"
-
end
-
-
# returns the oauth-documented authorization method to use when requesting a token.
-
23
def token_endpoint_auth_method
-
360
@token_endpoint_auth_method || "client_secret_basic"
-
end
-
-
23
def reset!
-
36
@access_token = nil
-
end
-
-
# when not available, it uses the +http+ object to request new access and refresh tokens.
-
23
def fetch_access_token(http)
-
126
return access_token if access_token
-
-
108
load(http)
-
-
# always prefer refresh token grant if a refresh token is available
-
108
grant_type = @refresh_token ? "refresh_token" : @grant_type
-
-
108
headers = {} # : Hash[String ,String]
-
24
form_post = {
-
84
"grant_type" => @grant_type,
-
"scope" => Array(@scope).join(" "),
-
"audience" => @audience,
-
}.compact
-
-
# auth
-
96
case token_endpoint_auth_method
-
when "client_secret_post"
-
16
form_post["client_id"] = @client_id
-
16
form_post["client_secret"] = @client_secret
-
when "client_secret_basic"
-
80
headers["authorization"] = Authentication::Basic.new(@client_id, @client_secret).authenticate
-
end
-
-
96
case grant_type
-
when "client_credentials"
-
# do nothing
-
when "refresh_token"
-
18
raise Error, "cannot use the `\"refresh_token\"` grant type without a refresh token" unless refresh_token
-
-
16
form_post["refresh_token"] = refresh_token
-
end
-
-
# POST /token
-
108
token_request = http.build_request("POST", token_endpoint, headers: headers, form: form_post)
-
-
108
token_request.headers.delete("authorization") unless token_endpoint_auth_method == "client_secret_basic"
-
-
216
token_response = http.skip_auth_header { http.request(token_request) }
-
-
11
begin
-
108
token_response.raise_for_status
-
rescue HTTPError => e
-
@refresh_token = nil if e.response.status == 401 && (grant_type == "refresh_token")
-
raise e
-
end
-
-
108
payload = token_response.json
-
-
108
@refresh_token = payload["refresh_token"] || @refresh_token
-
108
@access_token = payload["access_token"]
-
end
-
-
# TODO: remove this after deprecating the `:oauth_session` option
-
23
def merge(other)
-
obj = dup
-
-
case other
-
when OAuthSession
-
other.instance_variables.each do |ivar|
-
val = other.instance_variable_get(ivar)
-
next unless val
-
-
obj.instance_variable_set(ivar, val)
-
end
-
when Hash
-
other.each do |k, v|
-
obj.instance_variable_set(:"@#{k}", v) if obj.instance_variable_defined?(:"@#{k}")
-
end
-
end
-
obj
-
end
-
-
23
private
-
-
# uses +http+ to fetch for the oauth server metadata.
-
23
def load(http)
-
126
return if @grant_type && @scope
-
-
72
metadata = http.skip_auth_header { http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json }
-
-
36
@token_endpoint = metadata["token_endpoint"]
-
36
@scope = metadata["scopes_supported"]
-
144
@grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
-
36
@token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
-
36
SUPPORTED_AUTH_METHODS.include?(am)
-
end
-
16
nil
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :oauth_options :: an hash of options to be used during session management.
-
# check the parameters to initialize the OAuthSession class.
-
23
module OptionsMethods
-
23
private
-
-
23
def option_oauth_session(value)
-
36
warn "DEPRECATION WARNING: option `:oauth_session` is deprecated. " \
-
"Use `:oauth_options` instead."
-
-
32
case value
-
when Hash
-
18
OAuthSession.new(**value)
-
when OAuthSession
-
18
value
-
else
-
raise TypeError, ":oauth_session must be a #{OAuthSession}"
-
end
-
end
-
-
23
def option_oauth_options(value)
-
450
value = Hash[value] unless value.is_a?(Hash)
-
432
value
-
end
-
end
-
-
23
module InstanceMethods
-
23
attr_reader :oauth_session
-
23
protected :oauth_session
-
-
23
def initialize(*)
-
468
super
-
-
468
@oauth_session = if @options.oauth_options
-
252
OAuthSession.new(**@options.oauth_options)
-
215
elsif @options.oauth_session
-
18
@oauth_session = @options.oauth_session.dup
-
end
-
end
-
-
23
def initialize_dup(other)
-
18
super
-
18
@oauth_session = other.instance_variable_get(:@oauth_session).dup
-
end
-
-
23
def oauth_auth(**args)
-
18
warn "DEPRECATION WARNING: `#{__method__}` is deprecated. " \
-
"Use `with(oauth_options: options)` instead."
-
-
18
with(oauth_options: args)
-
end
-
-
# will eagerly negotiate new oauth tokens with the issuer
-
23
def refresh_oauth_tokens!
-
18
return unless @oauth_session
-
-
18
@oauth_session.reset!
-
18
@oauth_session.fetch_access_token(self)
-
end
-
-
# TODO: deprecate
-
23
def with_access_token
-
18
warn "DEPRECATION WARNING: `#{__method__}` is deprecated. " \
-
"The session will automatically handle token lifecycles for you."
-
-
18
other_session = dup # : instance
-
18
oauth_session = other_session.oauth_session
-
18
oauth_session.fetch_access_token(other_session)
-
18
other_session
-
end
-
-
23
private
-
-
23
def generate_auth_token
-
54
return unless @oauth_session
-
-
54
@oauth_session.fetch_access_token(self)
-
end
-
-
23
def dynamic_auth_token?(_)
-
36
@oauth_session
-
end
-
end
-
-
23
module OAuthRetries
-
23
module InstanceMethods
-
23
private
-
-
23
def prepare_to_retry(_request, response)
-
18
@oauth_session.reset! if @oauth_session
-
-
18
super
-
end
-
end
-
end
-
end
-
23
register_plugin :oauth, OAuth
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Plugins
-
# This plugin implements a session that persists connections over the duration of the process.
-
#
-
# This will improve connection reuse in a long-running process.
-
#
-
# One important caveat to note is, although this session might not close connections,
-
# other sessions from the same process that don't have this plugin turned on might.
-
#
-
# This session will still be able to work with it, as if, when expecting a connection
-
# terminated by a different session, it will just retry on a new one and keep it open.
-
#
-
# This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
-
# So when you use this, make sure that you don't fall into this trap.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Persistent
-
#
-
30
module Persistent
-
30
class << self
-
30
def load_dependencies(klass)
-
597
klass.plugin(:fiber_concurrency)
-
-
597
max_retries = if klass.default_options.respond_to?(:max_retries)
-
9
[klass.default_options.max_retries, 1].max
-
else
-
588
1
-
end
-
597
klass.plugin(:retries, max_retries: max_retries)
-
end
-
end
-
-
30
def self.extra_options(options)
-
597
options.merge(persistent: true)
-
end
-
-
30
module InstanceMethods
-
30
def close(*)
-
342
super
-
-
# traverse other threads and unlink respective selector
-
# WARNING: this is not thread safe, make sure that the session isn't being
-
# used anymore, or all non-main threads are stopped.
-
342
Thread.list.each do |th|
-
4392
store = thread_selector_store(th)
-
-
4392
next unless store && store.key?(self)
-
-
334
selector = store.delete(self)
-
-
334
selector_close(selector)
-
end
-
end
-
-
30
private
-
-
30
def retryable_request?(request, response, *)
-
725
super || begin
-
226
return false unless response && response.is_a?(ErrorResponse)
-
-
32
error = response.error
-
-
352
Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
-
end
-
end
-
-
30
def retryable_error?(ex, options)
-
135
super &&
-
# under the persistent plugin rules, requests are only retried for connection related errors,
-
# which do not include request timeout related errors. This only gets overriden if the end user
-
# manually changed +:max_retries+ to something else, which means it is aware of the
-
# consequences.
-
119
(!ex.is_a?(RequestTimeoutError) || options.max_retries != 1)
-
end
-
end
-
end
-
30
register_plugin :persistent, Persistent
-
end
-
end
-
# frozen_string_literal: true
-
-
24
module HTTPX
-
24
class ProxyError < ConnectionError; end
-
-
24
module Plugins
-
#
-
# This plugin adds support for proxies. It ships with support for:
-
#
-
# * HTTP proxies
-
# * HTTPS proxies
-
# * Socks4/4a proxies
-
# * Socks5 proxies
-
#
-
# https://gitlab.com/os85/httpx/wikis/Proxy
-
#
-
24
module Proxy
-
24
class ProxyConnectionError < ProxyError; end
-
-
24
PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
-
-
24
class << self
-
24
def configure(klass)
-
405
klass.plugin(:"proxy/http")
-
405
klass.plugin(:"proxy/socks4")
-
405
klass.plugin(:"proxy/socks5")
-
end
-
-
24
def extra_options(options)
-
405
options.merge(supported_proxy_protocols: [])
-
end
-
-
24
def subplugins
-
178
{
-
1462
retries: ProxyRetries,
-
}
-
end
-
end
-
-
24
class Parameters
-
24
attr_reader :uri, :username, :password, :scheme, :no_proxy
-
-
24
def initialize(uri: nil, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
-
443
@no_proxy = Array(no_proxy) if no_proxy
-
443
@uris = Array(uri)
-
443
uri = @uris.first
-
-
443
@username = username
-
443
@password = password
-
-
443
@ns = 0
-
-
443
if uri
-
398
@uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
-
398
@username ||= @uri.user
-
398
@password ||= @uri.password
-
end
-
-
443
@scheme = scheme
-
-
443
return unless @uri && @username && @password
-
-
254
@authenticator = nil
-
254
@scheme ||= infer_default_auth_scheme(@uri)
-
-
254
return unless @scheme
-
-
200
@authenticator = load_authenticator(@scheme, @username, @password, **extra)
-
end
-
-
24
def shift
-
# TODO: this operation must be synchronized
-
16
@ns += 1
-
18
@uri = @uris[@ns]
-
-
18
return unless @uri
-
-
18
@uri = URI(@uri) unless @uri.is_a?(URI::Generic)
-
-
18
scheme = infer_default_auth_scheme(@uri)
-
-
18
return unless scheme != @scheme
-
-
18
@scheme = scheme
-
18
@username = username || @uri.user
-
18
@password = password || @uri.password
-
18
@authenticator = load_authenticator(scheme, @username, @password)
-
end
-
-
24
def can_authenticate?(*args)
-
234
return false unless @authenticator && @authenticator.respond_to?(:can_authenticate?)
-
-
72
@authenticator.can_authenticate?(*args)
-
end
-
-
24
def authenticate(*args)
-
197
return unless @authenticator
-
-
197
@authenticator.authenticate(*args)
-
end
-
-
24
def ==(other)
-
476
case other
-
when Parameters
-
488
@uri == other.uri &&
-
@username == other.username &&
-
@password == other.password &&
-
@scheme == other.scheme
-
when URI::Generic, String
-
27
proxy_uri = @uri.dup
-
27
proxy_uri.user = @username
-
27
proxy_uri.password = @password
-
27
other_uri = other.is_a?(URI::Generic) ? other : URI.parse(other)
-
27
proxy_uri == other_uri
-
else
-
18
super
-
end
-
end
-
-
24
private
-
-
24
def infer_default_auth_scheme(uri)
-
227
case uri.scheme
-
when "socks5"
-
54
uri.scheme
-
when "http", "https"
-
131
"basic"
-
end
-
end
-
-
24
def load_authenticator(scheme, username, password, **extra)
-
218
auth_scheme = scheme.to_s.capitalize
-
-
218
require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
-
-
218
Authentication.const_get(auth_scheme).new(username, password, **extra)
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :proxy :: proxy options defining *:uri*, *:username*, *:password* or
-
# *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
-
24
module OptionsMethods
-
24
private
-
-
24
def option_proxy(value)
-
808
value.is_a?(Parameters) ? value : Parameters.new(**Hash[value])
-
end
-
-
24
def option_supported_proxy_protocols(value)
-
2041
raise TypeError, ":supported_proxy_protocols must be an Array" unless value.is_a?(Array)
-
-
2041
value.map(&:to_s)
-
end
-
end
-
-
24
module InstanceMethods
-
24
def find_connection(request_uri, selector, options)
-
511
return super unless options.respond_to?(:proxy)
-
-
511
if (next_proxy = request_uri.find_proxy)
-
4
return super(request_uri, selector, options.merge(proxy: Parameters.new(uri: next_proxy)))
-
end
-
-
507
proxy = options.proxy
-
-
507
return super unless proxy
-
-
496
next_proxy = proxy.uri
-
-
496
raise ProxyError, "Failed to connect to proxy" unless next_proxy
-
-
1
raise ProxyError,
-
478
"#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
-
-
469
if (no_proxy = proxy.no_proxy)
-
18
no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
-
-
# TODO: setting proxy to nil leaks the connection object in the pool
-
18
return super(request_uri, selector, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
-
next_proxy.port, no_proxy)
-
end
-
-
460
super(request_uri, selector, options.merge(proxy: proxy))
-
end
-
-
24
private
-
-
24
def fetch_response(request, selector, options)
-
994
response = request.response # in case it goes wrong later
-
-
98
begin
-
994
response = super
-
-
994
if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
-
18
options.proxy.shift
-
-
# return last error response if no more proxies to try
-
18
return response if options.proxy.uri.nil?
-
-
18
log { "failed connecting to proxy, trying next..." }
-
18
request.transition(:idle)
-
18
send_request(request, selector, options)
-
-
# recalling itself, in case an error was triggered by the above, and we can
-
# verify retriability again.
-
18
return fetch_response(request, selector, options)
-
end
-
976
response
-
rescue ProxyError
-
# may happen if coupled with retries, and there are no more proxies to try, in which case
-
# it'll end up here
-
response
-
end
-
end
-
-
24
def proxy_error?(_request, response, options)
-
190
return false unless options.proxy
-
-
189
error = response.error
-
168
case error
-
when NativeResolveError
-
18
proxy_uri = URI(options.proxy.uri)
-
-
18
unresolved_host = error.host
-
-
# failed resolving proxy domain
-
18
unresolved_host == proxy_uri.host
-
when ResolveError
-
proxy_uri = URI(options.proxy.uri)
-
-
error.message.end_with?(proxy_uri.to_s)
-
when ProxyConnectionError
-
# timeout errors connecting to proxy
-
true
-
else
-
171
false
-
end
-
end
-
end
-
-
24
module ConnectionMethods
-
24
using URIExtensions
-
-
24
def initialize(*)
-
458
super
-
458
return unless @options.proxy
-
-
# redefining the connection origin as the proxy's URI,
-
# as this will be used as the tcp peer ip.
-
438
@proxy_uri = URI(@options.proxy.uri)
-
end
-
-
24
def peer
-
1096
@proxy_uri || super
-
end
-
-
24
def connecting?
-
7062
return super unless @options.proxy
-
-
6846
super || @state == :connecting || @state == :connected
-
end
-
-
24
def call
-
1626
super
-
-
1626
return unless @options.proxy
-
-
1456
case @state
-
when :connecting
-
418
consume
-
end
-
rescue *PROXY_ERRORS => e
-
if connecting?
-
error = ProxyConnectionError.new(e.message)
-
error.set_backtrace(e.backtrace)
-
raise error
-
end
-
-
raise e
-
end
-
-
24
def reset
-
511
return super unless @options.proxy
-
-
492
@state = :open
-
-
492
super
-
# emit(:close)
-
end
-
-
24
private
-
-
24
def initialize_type(uri, options)
-
458
return super unless options.proxy
-
-
438
"tcp"
-
end
-
-
24
def connect
-
1318
return super unless @options.proxy
-
-
1148
case @state
-
when :idle
-
869
transition(:connecting)
-
when :connected
-
411
transition(:open)
-
end
-
end
-
-
24
def handle_transition(nextstate)
-
2715
return super unless @options.proxy
-
-
2342
case nextstate
-
when :closing
-
# this is a hack so that we can use the super method
-
# and it'll think that the current state is open
-
492
@state = :open if @state == :connecting
-
end
-
2618
super
-
end
-
-
24
def purge_after_closed
-
538
super
-
-
550
while @io.respond_to?(:proxy_io)
-
115
@io = @io.proxy_io
-
-
115
super
-
end
-
end
-
end
-
-
24
module ProxyRetries
-
24
module InstanceMethods
-
24
private
-
-
24
def retryable_error?(ex, *)
-
62
super || ex.is_a?(ProxyConnectionError)
-
end
-
end
-
end
-
end
-
24
register_plugin :proxy, Proxy
-
end
-
-
24
class ProxySSL < SSL
-
24
attr_reader :proxy_io
-
-
24
def initialize(tcp, request_uri, options)
-
116
@proxy_io = tcp
-
116
@io = tcp.to_io
-
116
super(request_uri, tcp.addresses, options)
-
116
@hostname = request_uri.host
-
116
@state = :connected
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
24
module HTTPX
-
24
module Plugins
-
24
module Proxy
-
24
module HTTP
-
24
class << self
-
24
def extra_options(options)
-
405
options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[http])
-
end
-
end
-
-
24
module InstanceMethods
-
24
def with_proxy_basic_auth(opts)
-
9
with(proxy: opts.merge(scheme: "basic"))
-
end
-
-
24
def with_proxy_digest_auth(opts)
-
27
with(proxy: opts.merge(scheme: "digest"))
-
end
-
-
24
def with_proxy_ntlm_auth(opts)
-
9
with(proxy: opts.merge(scheme: "ntlm"))
-
end
-
-
24
def fetch_response(request, selector, options)
-
994
response = super
-
-
994
if response &&
-
response.is_a?(Response) &&
-
response.status == 407 &&
-
!request.headers.key?("proxy-authorization") &&
-
response.headers.key?("proxy-authenticate") && options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
-
9
request.transition(:idle)
-
8
request.headers["proxy-authorization"] =
-
options.proxy.authenticate(request, response.headers["proxy-authenticate"])
-
9
send_request(request, selector, options)
-
-
# recalling itself, in case an error was triggered by the above, and we can
-
# verify retriability again.
-
8
return fetch_response(request, selector, options)
-
end
-
-
985
response
-
end
-
end
-
-
24
module ConnectionMethods
-
24
def force_close(*)
-
if @state == :connecting
-
# proxy connect related requests should not be reenqueed
-
@parser.reset
-
@inflight -= @parser.pending.size
-
@parser.pending.clear
-
end
-
-
super
-
end
-
-
24
private
-
-
24
def handle_transition(nextstate)
-
3027
return super unless @options.proxy && @options.proxy.uri.scheme == "http"
-
-
1494
case nextstate
-
when :connecting
-
410
return unless @state == :idle
-
-
410
@io.connect
-
410
return unless @io.connected?
-
-
205
@parser || begin
-
196
@parser = parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
-
196
parser.extend(ProxyParser)
-
196
parser.on(:response, &method(:__http_on_connect))
-
196
parser.on(:close) do
-
89
next unless @parser
-
-
18
reset
-
end
-
196
parser.on(:reset) do
-
27
if parser.pending.empty? && parser.empty?
-
18
reset
-
else
-
9
enqueue_pending_requests_from_parser(parser)
-
-
9
initial_state = @state
-
-
9
reset
-
-
9
if @pending.empty?
-
@parser = nil
-
next
-
end
-
# keep parser state around due to proxy auth protocol;
-
# intermediate authenticated request is already inside
-
# the parser
-
9
connect_request = parser = nil
-
-
9
if initial_state == :connecting
-
9
parser = @parser
-
9
@parser.reset
-
9
if @pending.first.is_a?(ConnectRequest)
-
9
connect_request = @pending.shift # this happened when reenqueing
-
end
-
end
-
-
9
idling
-
-
9
@parser = parser
-
9
if connect_request
-
8
@inflight += 1
-
9
parser.send(connect_request)
-
end
-
9
transition(:connecting)
-
end
-
end
-
196
__http_proxy_connect(parser)
-
end
-
205
return if @state == :connected
-
when :connected
-
178
return unless @state == :idle || @state == :connecting
-
-
160
case @state
-
when :connecting
-
71
parser = @parser
-
71
@parser = nil
-
71
parser.close
-
when :idle
-
107
@parser.callbacks.clear
-
107
set_parser_callbacks(@parser)
-
end
-
end
-
1351
super
-
end
-
-
24
def __http_proxy_connect(parser)
-
196
req = @pending.first
-
196
if req && req.uri.scheme == "https"
-
# if the first request after CONNECT is to an https address, it is assumed that
-
# all requests in the queue are not only ALL HTTPS, but they also share the certificate,
-
# and therefore, will share the connection.
-
#
-
89
connect_request = ConnectRequest.new(req.uri, @options)
-
80
@inflight += 1
-
89
parser.send(connect_request)
-
else
-
107
handle_transition(:connected)
-
end
-
end
-
-
24
def __http_on_connect(request, response)
-
88
@inflight -= 1
-
98
if response.is_a?(Response) && response.status == 200
-
71
req = @pending.first
-
71
request_uri = req.uri
-
71
@io = ProxySSL.new(@io, request_uri, @options)
-
71
transition(:connected)
-
71
throw(:called)
-
26
elsif response.is_a?(Response) &&
-
response.status == 407 &&
-
!request.headers.key?("proxy-authorization") &&
-
@options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
-
-
9
request.transition(:idle)
-
8
request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
-
9
@parser.send(request)
-
8
@inflight += 1
-
else
-
18
pending = @pending + @parser.pending
-
48
while (req = pending.shift)
-
18
response.finish!
-
18
req.response = response
-
18
req.emit_response(response)
-
end
-
18
reset
-
end
-
end
-
end
-
-
24
module ProxyParser
-
24
def join_headline(request)
-
196
return super if request.verb == "CONNECT"
-
-
88
"#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
-
end
-
-
24
def set_protocol_headers(request)
-
205
extra_headers = super
-
-
205
proxy_params = @options.proxy
-
205
if proxy_params.scheme == "basic"
-
# opt for basic auth
-
113
extra_headers["proxy-authorization"] = proxy_params.authenticate(extra_headers)
-
end
-
205
extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
-
205
extra_headers
-
end
-
end
-
-
24
class ConnectRequest < Request
-
24
def initialize(uri, options)
-
89
super("CONNECT", uri, options)
-
89
@headers.delete("accept")
-
end
-
-
24
def path
-
96
"#{@uri.hostname}:#{@uri.port}"
-
end
-
end
-
end
-
end
-
24
register_plugin :"proxy/http", Proxy::HTTP
-
end
-
end
-
# frozen_string_literal: true
-
-
24
require "resolv"
-
24
require "ipaddr"
-
-
24
module HTTPX
-
24
class Socks4Error < ProxyError; end
-
-
24
module Plugins
-
24
module Proxy
-
24
module Socks4
-
24
VERSION = 4
-
24
CONNECT = 1
-
24
GRANTED = 0x5A
-
24
PROTOCOLS = %w[socks4 socks4a].freeze
-
-
24
Error = Socks4Error
-
-
24
class << self
-
24
def extra_options(options)
-
405
options.merge(supported_proxy_protocols: options.supported_proxy_protocols + PROTOCOLS)
-
end
-
end
-
-
24
module ConnectionMethods
-
24
def interests
-
4938
if @state == :connecting
-
return @write_buffer.empty? ? :r : :w
-
end
-
-
4938
super
-
end
-
-
24
private
-
-
24
def handle_transition(nextstate)
-
3099
return super unless @options.proxy && PROTOCOLS.include?(@options.proxy.uri.scheme)
-
-
440
case nextstate
-
when :connecting
-
144
return unless @state == :idle
-
-
144
@io.connect
-
144
return unless @io.connected?
-
-
72
req = @pending.first
-
72
return unless req
-
-
72
request_uri = req.uri
-
72
@write_buffer << Packet.connect(@options.proxy, request_uri)
-
72
__socks4_proxy_connect
-
when :connected
-
54
return unless @state == :connecting
-
-
54
@parser = nil
-
end
-
421
log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
-
421
super
-
end
-
-
24
def __socks4_proxy_connect
-
72
@parser = SocksParser.new(@write_buffer, @options)
-
72
@parser.once(:packet, &method(:__socks4_on_packet))
-
end
-
-
24
def __socks4_on_packet(packet)
-
72
_version, status, _port, _ip = packet.unpack("CCnN")
-
72
if status == GRANTED
-
54
req = @pending.first
-
54
request_uri = req.uri
-
54
@io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
-
54
transition(:connected)
-
54
throw(:called)
-
else
-
18
on_socks4_error("socks error: #{status}")
-
end
-
end
-
-
24
def on_socks4_error(message)
-
18
ex = Error.new(message)
-
18
ex.set_backtrace(caller)
-
18
on_error(ex)
-
18
throw(:called)
-
end
-
end
-
-
24
class SocksParser
-
24
include HTTPX::Callbacks
-
-
24
def initialize(buffer, options)
-
72
@buffer = buffer
-
72
@options = options
-
end
-
-
24
def close; end
-
-
24
def consume(*); end
-
-
24
def empty?
-
true
-
end
-
-
24
def <<(packet)
-
72
emit(:packet, packet)
-
end
-
end
-
-
24
module Packet
-
24
module_function
-
-
24
def connect(parameters, uri)
-
72
packet = [VERSION, CONNECT, uri.port].pack("CCn")
-
-
64
case parameters.uri.scheme
-
when "socks4"
-
54
socks_host = uri.host
-
5
begin
-
108
ip = IPAddr.new(socks_host)
-
54
packet << ip.hton
-
rescue IPAddr::InvalidAddressError
-
54
socks_host = Resolv.getaddress(socks_host)
-
54
retry
-
end
-
54
packet << [parameters.username].pack("Z*")
-
when "socks4a"
-
18
packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
-
end
-
72
packet
-
end
-
end
-
end
-
end
-
24
register_plugin :"proxy/socks4", Proxy::Socks4
-
end
-
end
-
# frozen_string_literal: true
-
-
24
module HTTPX
-
24
class Socks5Error < ProxyError; end
-
-
24
module Plugins
-
24
module Proxy
-
24
module Socks5
-
24
VERSION = 5
-
24
NOAUTH = 0
-
24
PASSWD = 2
-
24
NONE = 0xff
-
24
CONNECT = 1
-
24
IPV4 = 1
-
24
DOMAIN = 3
-
24
IPV6 = 4
-
24
SUCCESS = 0
-
-
24
Error = Socks5Error
-
-
24
class << self
-
24
def load_dependencies(*)
-
405
require_relative "../auth/socks5"
-
end
-
-
24
def extra_options(options)
-
405
options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[socks5])
-
end
-
end
-
-
24
module ConnectionMethods
-
24
def call
-
1626
super
-
-
1626
return unless @options.proxy && @options.proxy.uri.scheme == "socks5"
-
-
388
case @state
-
when :connecting,
-
:negotiating,
-
:authenticating
-
190
consume
-
end
-
end
-
-
24
def connecting?
-
7062
super || @state == :authenticating || @state == :negotiating
-
end
-
-
24
def interests
-
8161
if @state == :connecting || @state == :authenticating || @state == :negotiating
-
2997
return @write_buffer.empty? ? :r : :w
-
end
-
-
4938
super
-
end
-
-
24
private
-
-
24
def handle_transition(nextstate)
-
3423
return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
-
-
1040
case nextstate
-
when :connecting
-
324
return unless @state == :idle
-
-
324
@io.connect
-
324
return unless @io.connected?
-
-
162
@write_buffer << Packet.negotiate(@options.proxy)
-
162
__socks5_proxy_connect
-
when :authenticating
-
54
return unless @state == :connecting
-
-
54
@write_buffer << Packet.authenticate(@options.proxy)
-
when :negotiating
-
216
return unless @state == :connecting || @state == :authenticating
-
-
54
req = @pending.first
-
54
request_uri = req.uri
-
54
@write_buffer << Packet.connect(request_uri)
-
when :connected
-
36
return unless @state == :negotiating
-
-
36
@parser = nil
-
end
-
846
log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
-
846
super
-
end
-
-
24
def __socks5_proxy_connect
-
162
@parser = SocksParser.new(@write_buffer, @options)
-
162
@parser.on(:packet, &method(:__socks5_on_packet))
-
162
transition(:negotiating)
-
end
-
-
24
def __socks5_on_packet(packet)
-
240
case @state
-
when :connecting
-
162
version, method = packet.unpack("CC")
-
162
__socks5_check_version(version)
-
144
case method
-
when PASSWD
-
54
transition(:authenticating)
-
24
nil
-
when NONE
-
90
__on_socks5_error("no supported authorization methods")
-
else
-
18
transition(:negotiating)
-
end
-
when :authenticating
-
54
_, status = packet.unpack("CC")
-
54
return transition(:negotiating) if status == SUCCESS
-
-
18
__on_socks5_error("socks authentication error: #{status}")
-
when :negotiating
-
54
version, reply, = packet.unpack("CC")
-
54
__socks5_check_version(version)
-
54
__on_socks5_error("socks5 negotiation error: #{reply}") unless reply == SUCCESS
-
36
req = @pending.first
-
36
request_uri = req.uri
-
36
@io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
-
36
transition(:connected)
-
36
throw(:called)
-
end
-
end
-
-
24
def __socks5_check_version(version)
-
216
__on_socks5_error("invalid SOCKS version (#{version})") if version != 5
-
end
-
-
24
def __on_socks5_error(message)
-
126
ex = Error.new(message)
-
126
ex.set_backtrace(caller)
-
126
on_error(ex)
-
126
throw(:called)
-
end
-
end
-
-
24
class SocksParser
-
24
include HTTPX::Callbacks
-
-
24
def initialize(buffer, options)
-
162
@buffer = buffer
-
162
@options = options
-
end
-
-
24
def close; end
-
-
24
def consume(*); end
-
-
24
def empty?
-
true
-
end
-
-
24
def <<(packet)
-
270
emit(:packet, packet)
-
end
-
end
-
-
24
module Packet
-
24
module_function
-
-
24
def negotiate(parameters)
-
162
methods = [NOAUTH]
-
162
methods << PASSWD if parameters.can_authenticate?
-
162
methods.unshift(methods.size)
-
162
methods.unshift(VERSION)
-
162
methods.pack("C*")
-
end
-
-
24
def authenticate(parameters)
-
54
parameters.authenticate
-
end
-
-
24
def connect(uri)
-
54
packet = [VERSION, CONNECT, 0].pack("C*")
-
5
begin
-
54
ip = IPAddr.new(uri.host)
-
-
18
ipcode = ip.ipv6? ? IPV6 : IPV4
-
-
18
packet << [ipcode].pack("C") << ip.hton
-
rescue IPAddr::InvalidAddressError
-
36
packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
-
end
-
54
packet << [uri.port].pack("n")
-
54
packet
-
end
-
end
-
end
-
end
-
24
register_plugin :"proxy/socks5", Proxy::Socks5
-
end
-
end
-
# frozen_string_literal: true
-
-
21
require "httpx/plugins/proxy"
-
-
21
module HTTPX
-
21
module Plugins
-
21
module Proxy
-
21
module SSH
-
21
class << self
-
21
def load_dependencies(*)
-
14
require "net/ssh/gateway"
-
end
-
end
-
-
21
module OptionsMethods
-
21
private
-
-
21
def option_proxy(value)
-
28
Hash[value]
-
end
-
end
-
-
21
module InstanceMethods
-
21
def request(*args, **options)
-
14
raise ArgumentError, "must perform at least one request" if args.empty?
-
-
14
requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
-
-
14
request = requests.first or return super
-
-
14
request_options = request.options
-
-
14
return super unless request_options.proxy
-
-
14
ssh_options = request_options.proxy
-
14
ssh_uris = ssh_options.delete(:uri)
-
14
ssh_uri = URI.parse(ssh_uris.shift)
-
-
14
return super unless ssh_uri.scheme == "ssh"
-
-
14
ssh_username = ssh_options.delete(:username)
-
14
ssh_options[:port] ||= ssh_uri.port || 22
-
14
if request_options.debug
-
ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
-
end
-
-
14
request_uri = URI(requests.first.uri)
-
14
@_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
-
begin
-
14
@_gateway.open(request_uri.host, request_uri.port) do |local_port|
-
14
io = build_gateway_socket(local_port, request_uri, request_options)
-
14
super(*args, **options.merge(io: io))
-
end
-
ensure
-
14
@_gateway.shutdown!
-
end
-
end
-
-
21
private
-
-
21
def build_gateway_socket(port, request_uri, options)
-
14
case request_uri.scheme
-
when "https"
-
7
ctx = OpenSSL::SSL::SSLContext.new
-
7
ctx_options = SSL::TLS_OPTIONS.merge(options.ssl)
-
7
ctx.set_params(ctx_options) unless ctx_options.empty?
-
7
sock = TCPSocket.open("localhost", port)
-
7
io = OpenSSL::SSL::SSLSocket.new(sock, ctx)
-
7
io.hostname = request_uri.host
-
7
io.sync_close = true
-
7
io.connect
-
7
io.post_connection_check(request_uri.host) if ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
-
7
io
-
when "http"
-
7
TCPSocket.open("localhost", port)
-
else
-
raise TypeError, "unexpected scheme: #{request_uri.scheme}"
-
end
-
end
-
end
-
-
21
module ConnectionMethods
-
# should not coalesce connections here, as the IP is the IP of the proxy
-
21
def coalescable?(*)
-
return super unless @options.proxy
-
-
false
-
end
-
end
-
end
-
end
-
21
register_plugin :"proxy/ssh", Proxy::SSH
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds support for HTTP/2 Push responses.
-
#
-
# In order to benefit from this, requests are sent one at a time, so that
-
# no push responses are received after corresponding request has been sent.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Server-Push
-
#
-
23
module PushPromise
-
23
def self.extra_options(options)
-
18
options.merge(http2_settings: { settings_enable_push: 1 },
-
max_concurrent_requests: 1)
-
end
-
-
23
module ResponseMethods
-
23
def pushed?
-
18
@__pushed
-
end
-
-
23
def mark_as_pushed!
-
9
@__pushed = true
-
end
-
end
-
-
23
module InstanceMethods
-
23
private
-
-
23
def promise_headers
-
18
@promise_headers ||= {}
-
end
-
-
23
def on_promise(parser, stream)
-
18
stream.on(:promise_headers) do |h|
-
18
__on_promise_request(parser, stream, h)
-
end
-
18
stream.on(:headers) do |h|
-
9
__on_promise_response(parser, stream, h)
-
end
-
end
-
-
23
def __on_promise_request(parser, stream, h)
-
18
log(level: 1, color: :yellow) do
-
skipped
# :nocov:
-
skipped
h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
-
skipped
# :nocov:
-
end
-
18
headers = @options.headers_class.new(h)
-
18
path = headers[":path"]
-
18
authority = headers[":authority"]
-
-
27
request = parser.pending.find { |r| r.authority == authority && r.path == path }
-
18
if request
-
9
request.merge_headers(headers)
-
8
promise_headers[stream] = request
-
9
parser.pending.delete(request)
-
8
parser.streams[request] = stream
-
9
request.transition(:done)
-
else
-
9
stream.refuse
-
end
-
end
-
-
23
def __on_promise_response(parser, stream, h)
-
9
request = promise_headers.delete(stream)
-
9
return unless request
-
-
9
parser.__send__(:on_stream_headers, stream, request, h)
-
9
response = request.response
-
9
response.mark_as_pushed!
-
9
stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
-
9
stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
-
end
-
end
-
end
-
23
register_plugin(:push_promise, PushPromise)
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds support for using the experimental QUERY HTTP method
-
#
-
# https://gitlab.com/os85/httpx/wikis/Query
-
23
module Query
-
23
def self.subplugins
-
3
{
-
23
retries: QueryRetries,
-
}
-
end
-
-
23
module InstanceMethods
-
23
def query(*uri, **options)
-
18
request("QUERY", uri, **options)
-
end
-
end
-
-
23
module QueryRetries
-
23
module InstanceMethods
-
23
private
-
-
23
def retryable_request?(request, *)
-
27
super || request.verb == "QUERY"
-
end
-
end
-
end
-
end
-
-
23
register_plugin :query, Query
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds support for retrying requests when the request:
-
#
-
# * is rate limited;
-
# * when the server is unavailable (503);
-
# * when a 3xx request comes with a "retry-after" value
-
#
-
# https://gitlab.com/os85/httpx/wikis/Rate-Limiter
-
#
-
23
module RateLimiter
-
23
RATE_LIMIT_CODES = [429, 503].freeze
-
-
23
class << self
-
23
def load_dependencies(klass)
-
72
klass.plugin(:retries)
-
end
-
end
-
-
23
module InstanceMethods
-
23
private
-
-
23
def retryable_request?(request, response, options)
-
144
super || rate_limit_error?(response)
-
end
-
-
23
def retryable_response?(response, options)
-
144
rate_limit_error?(response) || super
-
end
-
-
23
def rate_limit_error?(response)
-
144
response.is_a?(Response) && RATE_LIMIT_CODES.include?(response.status)
-
end
-
-
# Servers send the "Retry-After" header field to indicate how long the
-
# user agent ought to wait before making a follow-up request. When
-
# sent with a 503 (Service Unavailable) response, Retry-After indicates
-
# how long the service is expected to be unavailable to the client.
-
# When sent with any 3xx (Redirection) response, Retry-After indicates
-
# the minimum time that the user agent is asked to wait before issuing
-
# the redirected request.
-
#
-
23
def when_to_retry(_, response, options)
-
72
return super unless response.is_a?(Response)
-
-
72
retry_after = response.headers["retry-after"]
-
-
72
return super unless retry_after
-
-
36
Utils.parse_retry_after(retry_after)
-
end
-
end
-
end
-
-
23
register_plugin :rate_limiter, RateLimiter
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds support for retrying requests when certain errors happen.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Response-Cache
-
#
-
23
module ResponseCache
-
23
CACHEABLE_VERBS = %w[GET HEAD].freeze
-
23
CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
-
23
SUPPORTED_VARY_HEADERS = %w[accept accept-encoding accept-language cookie origin].sort.freeze
-
23
private_constant :CACHEABLE_VERBS
-
23
private_constant :CACHEABLE_STATUS_CODES
-
-
23
class << self
-
23
def load_dependencies(*)
-
270
require_relative "response_cache/store"
-
270
require_relative "response_cache/file_store"
-
end
-
-
# whether the +response+ can be stored in the response cache.
-
# (i.e. has a cacheable body, does not contain directives prohibiting storage, etc...)
-
23
def cacheable_response?(response)
-
153
response.is_a?(Response) &&
-
(
-
153
response.cache_control.nil? ||
-
# TODO: !response.cache_control.include?("private") && is shared cache
-
!response.cache_control.include?("no-store")
-
) &&
-
CACHEABLE_STATUS_CODES.include?(response.status) &&
-
# RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
-
# 410 MAY be stored by a cache and used in reply to a subsequent
-
# request, subject to the expiration mechanism, unless a cache-control
-
# directive prohibits caching. However, a cache that does not support
-
# the Range and Content-Range headers MUST NOT cache 206 (Partial
-
# Content) responses.
-
response.status != 206
-
end
-
-
# whether the +response+
-
23
def not_modified?(response)
-
189
response.is_a?(Response) && response.status == 304
-
end
-
-
23
def extra_options(options)
-
270
options.merge(
-
supported_vary_headers: SUPPORTED_VARY_HEADERS,
-
response_cache_store: :store,
-
)
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :supported_vary_headers :: array of header values that will be considered for a "vary" header based cache validation
-
# (defaults to {SUPPORTED_VARY_HEADERS}).
-
# :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
-
# cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
-
# abides by the Cache Store Interface
-
#
-
# The Cache Store Interface requires implementation of the following methods:
-
#
-
# * +#get(request) -> response or nil+
-
# * +#set(request, response) -> void+
-
# * +#clear() -> void+)
-
#
-
23
module OptionsMethods
-
23
private
-
-
23
def option_response_cache_store(value)
-
408
case value
-
when :store
-
288
Store.new
-
when :file_store
-
18
FileStore.new
-
else
-
153
value
-
end
-
end
-
-
23
def option_supported_vary_headers(value)
-
270
Array(value).sort
-
end
-
end
-
-
23
module InstanceMethods
-
# wipes out all cached responses from the cache store.
-
23
def clear_response_cache
-
171
@options.response_cache_store.clear
-
end
-
-
23
def build_request(*)
-
558
request = super
-
558
return request unless cacheable_request?(request)
-
-
540
prepare_cache(request)
-
-
540
request
-
end
-
-
23
private
-
-
23
def send_request(request, *)
-
189
return request if request.response
-
-
171
super
-
end
-
-
23
def fetch_response(request, *)
-
360
response = super
-
-
360
return unless response
-
-
189
if ResponseCache.not_modified?(response)
-
36
log { "returning cached response for #{request.uri}" }
-
-
36
response.copy_from_cached!
-
152
elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
-
126
unless response.cached?
-
108
log { "caching response for #{request.uri}..." }
-
108
request.options.response_cache_store.set(request, response)
-
end
-
end
-
-
189
response
-
end
-
-
# will either assign a still-fresh cached response to +request+, or set up its HTTP
-
# cache invalidation headers in case it's not fresh anymore.
-
23
def prepare_cache(request)
-
792
cached_response = request.options.response_cache_store.get(request)
-
-
792
return unless cached_response && match_by_vary?(request, cached_response)
-
-
342
cached_response.body.rewind
-
-
342
if cached_response.fresh?
-
72
cached_response = cached_response.dup
-
72
cached_response.mark_as_cached!
-
72
request.response = cached_response
-
72
request.emit_response(cached_response)
-
64
return
-
end
-
-
270
request.cached_response = cached_response
-
-
270
if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
-
36
request.headers.add("if-modified-since", last_modified)
-
end
-
-
270
if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"])
-
198
request.headers.add("if-none-match", etag)
-
end
-
end
-
-
23
def cacheable_request?(request)
-
558
request.cacheable_verb? &&
-
(
-
540
!request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
-
)
-
end
-
-
# whether the +response+ complies with the directives set by the +request+ "vary" header
-
# (true when none is available).
-
23
def match_by_vary?(request, response)
-
342
vary = response.vary
-
-
342
return true unless vary
-
-
108
original_request = response.original_request
-
-
108
if vary == %w[*]
-
36
request.options.supported_vary_headers.each do |field|
-
180
return false unless request.headers[field] == original_request.headers[field]
-
end
-
-
32
return true
-
end
-
-
72
vary.all? do |field|
-
72
!original_request.headers.key?(field) || request.headers[field] == original_request.headers[field]
-
end
-
end
-
end
-
-
23
module RequestMethods
-
# points to a previously cached Response corresponding to this request.
-
23
attr_accessor :cached_response
-
-
23
def initialize(*)
-
756
super
-
756
@cached_response = nil
-
end
-
-
23
def merge_headers(*)
-
369
super
-
369
@response_cache_key = nil
-
end
-
-
# returns whether this request is cacheable as per HTTP caching rules.
-
23
def cacheable_verb?
-
711
CACHEABLE_VERBS.include?(@verb)
-
end
-
-
# returns a unique cache key as a String identifying this request
-
23
def response_cache_key
-
1575
@response_cache_key ||= begin
-
567
keys = [@verb, @uri.merge(path)]
-
-
567
@options.supported_vary_headers.each do |field|
-
2835
value = @headers[field]
-
-
2835
keys << value if value
-
end
-
567
Digest::SHA1.hexdigest("httpx-response-cache-#{keys.join("-")}")
-
end
-
end
-
end
-
-
23
module ResponseMethods
-
23
attr_writer :original_request, :revalidated_at
-
-
23
def initialize(*)
-
603
super
-
603
@cached = false
-
603
@revalidated_at = nil
-
end
-
-
# a copy of the request this response was originally cached from
-
23
def original_request
-
108
@original_request || @request
-
end
-
-
# whether this Response was duplicated from a previously {RequestMethods#cached_response}.
-
23
def cached?
-
690
@cached
-
end
-
-
# sets this Response as being duplicated from a previously cached response.
-
23
def mark_as_cached!
-
270
@cached = true
-
end
-
-
# eager-copies the response headers and body from {RequestMethods#cached_response}.
-
23
def copy_from_cached!
-
36
cached_response = @request.cached_response
-
-
36
return unless cached_response
-
-
# 304 responses do not have content-type, which are needed for decoding.
-
36
@headers = @headers.class.new(cached_response.headers.merge(@headers))
-
-
36
@body = cached_response.body.dup
-
-
36
@body.rewind
-
-
36
cached_response.revalidated_at = date
-
end
-
-
# A response is fresh if its age has not yet exceeded its freshness lifetime.
-
# other (#cache_control} directives may influence the outcome, as per the rules
-
# from the {rfc}[https://www.rfc-editor.org/rfc/rfc7234]
-
23
def fresh?
-
342
if cache_control
-
126
return false if cache_control.include?("no-cache")
-
-
90
return true if cache_control.include?("immutable")
-
-
# check age: max-age
-
216
max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
-
-
216
max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
-
-
90
max_age = max_age[/age=(\d+)/, 1] if max_age
-
-
90
max_age = max_age.to_i if max_age
-
-
90
return max_age > age if max_age
-
end
-
-
# check age: expires
-
216
if @headers.key?("expires")
-
5
begin
-
54
expires = Time.httpdate(@headers["expires"])
-
rescue ArgumentError
-
18
return false
-
end
-
-
32
return (expires - Time.now).to_i.positive?
-
end
-
-
162
false
-
end
-
-
# returns the "cache-control" directives as an Array of String(s).
-
23
def cache_control
-
972
return @cache_control if defined?(@cache_control)
-
-
40
@cache_control = begin
-
360
@headers["cache-control"].split(/ *, */) if @headers.key?("cache-control")
-
end
-
end
-
-
# returns the "vary" header value as an Array of (String) headers.
-
23
def vary
-
360
return @vary if defined?(@vary)
-
-
31
@vary = begin
-
279
@headers["vary"].split(/ *, */).map(&:downcase) if @headers.key?("vary")
-
end
-
end
-
-
23
private
-
-
# returns the value of the "age" header as an Integer (time since epoch).
-
# if no "age" of header exists, it returns the number of seconds since {#date}.
-
23
def age
-
108
if (revalidated_at = @revalidated_at)
-
(Time.now - revalidated_at).to_i
-
else
-
108
return @headers["age"].to_i if @headers.key?("age")
-
-
108
(Time.now - date).to_i
-
end
-
end
-
-
# returns the value of the "date" header as a Time object
-
23
def date
-
144
@date ||= Time.httpdate(@headers["date"])
-
rescue NoMethodError, ArgumentError
-
18
Time.now
-
end
-
end
-
-
23
module ResponseBodyMethods
-
23
def decode_chunk(chunk)
-
564
return chunk if @response.cached?
-
-
352
super
-
end
-
end
-
end
-
23
register_plugin :response_cache, ResponseCache
-
end
-
end
-
# frozen_string_literal: true
-
-
23
require "pathname"
-
-
23
module HTTPX::Plugins
-
23
module ResponseCache
-
# Implementation of a file system based cache store.
-
#
-
# It stores cached responses in a file under a directory pointed by the +dir+
-
# variable (defaults to the default temp directory from the OS), in a custom
-
# format (similar but different from HTTP/1.1 request/response framing).
-
23
class FileStore
-
23
CRLF = HTTPX::Connection::HTTP1::CRLF
-
-
23
attr_reader :dir
-
-
23
def initialize(dir = Dir.tmpdir)
-
99
@dir = Pathname.new(dir).join("httpx-response-cache")
-
-
99
FileUtils.mkdir_p(@dir)
-
end
-
-
23
def clear
-
81
FileUtils.rm_rf(@dir)
-
end
-
-
23
def get(request)
-
378
path = file_path(request)
-
-
378
return unless File.exist?(path)
-
-
189
File.open(path, mode: File::RDONLY | File::BINARY) do |f|
-
189
f.flock(File::Constants::LOCK_SH)
-
-
189
read_from_file(request, f)
-
end
-
end
-
-
23
def set(request, response)
-
117
path = file_path(request)
-
-
117
file_exists = File.exist?(path)
-
-
117
mode = file_exists ? File::RDWR : File::CREAT | File::Constants::WRONLY
-
-
117
File.open(path, mode: mode | File::BINARY) do |f|
-
117
f.flock(File::Constants::LOCK_EX)
-
-
117
if file_exists
-
9
cached_response = read_from_file(request, f)
-
-
9
if cached_response
-
9
next if cached_response == request.cached_response
-
-
9
cached_response.close
-
-
9
f.truncate(0)
-
-
9
f.rewind
-
end
-
end
-
# cache the request headers
-
117
f << request.verb << CRLF
-
117
f << request.uri << CRLF
-
-
117
request.headers.each do |field, value|
-
351
f << field << ":" << value << CRLF
-
end
-
-
117
f << CRLF
-
-
# cache the response
-
117
f << response.status << CRLF
-
117
f << response.version << CRLF
-
-
117
response.headers.each do |field, value|
-
333
f << field << ":" << value << CRLF
-
end
-
-
117
f << CRLF
-
-
117
response.body.rewind
-
-
117
IO.copy_stream(response.body, f)
-
end
-
end
-
-
23
private
-
-
23
def file_path(request)
-
495
@dir.join(request.response_cache_key)
-
end
-
-
23
def read_from_file(request, f)
-
# if it's an empty file
-
198
return if f.eof?
-
-
# read request data
-
198
verb = f.readline.delete_suffix!(CRLF)
-
198
uri = f.readline.delete_suffix!(CRLF)
-
-
198
request_headers = {}
-
880
while (line = f.readline) != CRLF
-
594
line.delete_suffix!(CRLF)
-
594
sep_index = line.index(":")
-
-
594
field = line.byteslice(0..(sep_index - 1))
-
594
value = line.byteslice((sep_index + 1)..-1)
-
-
528
request_headers[field] = value
-
end
-
-
198
status = f.readline.delete_suffix!(CRLF)
-
198
version = f.readline.delete_suffix!(CRLF)
-
-
198
response_headers = {}
-
856
while (line = f.readline) != CRLF
-
567
line.delete_suffix!(CRLF)
-
567
sep_index = line.index(":")
-
-
567
field = line.byteslice(0..(sep_index - 1))
-
567
value = line.byteslice((sep_index + 1)..-1)
-
-
504
response_headers[field] = value
-
end
-
-
198
original_request = request.options.request_class.new(verb, uri, request.options)
-
198
original_request.merge_headers(request_headers)
-
-
198
response = request.options.response_class.new(request, status, version, response_headers)
-
198
response.original_request = original_request
-
198
response.finish!
-
198
response.mark_as_cached!
-
-
198
IO.copy_stream(f, response.body)
-
-
198
response
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX::Plugins
-
23
module ResponseCache
-
# Implementation of a thread-safe in-memory cache store.
-
23
class Store
-
23
def initialize
-
360
@store = {}
-
360
@store_mutex = Thread::Mutex.new
-
end
-
-
23
def clear
-
180
@store_mutex.synchronize { @store.clear }
-
end
-
-
23
def get(request)
-
549
@store_mutex.synchronize do
-
549
@store[request.response_cache_key]
-
end
-
end
-
-
23
def set(request, response)
-
234
@store_mutex.synchronize do
-
234
cached_response = @store[request.response_cache_key]
-
-
234
cached_response.close if cached_response
-
-
208
@store[request.response_cache_key] = response
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Plugins
-
#
-
# This plugin adds support for retrying requests when errors happen.
-
#
-
# It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
-
# after which it will return the last response, error or not. It will **not** raise an exception.
-
#
-
# It does not retry which are not considered idempotent (see *retry_change_requests* to override).
-
#
-
# https://gitlab.com/os85/httpx/wikis/Retries
-
#
-
30
module Retries
-
30
MAX_RETRIES = 3
-
# TODO: pass max_retries in a configure/load block
-
-
30
IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
-
-
# subset of retryable errors which are safe to retry when reconnecting
-
2
RECONNECTABLE_ERRORS = [
-
28
IOError,
-
EOFError,
-
Errno::ECONNRESET,
-
Errno::ECONNABORTED,
-
Errno::EPIPE,
-
Errno::EINVAL,
-
Errno::ETIMEDOUT,
-
ConnectionError,
-
TLSError,
-
Connection::HTTP2::Error,
-
].freeze
-
-
30
RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
-
Parser::Error,
-
TimeoutError,
-
]).freeze
-
-
30
DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }.freeze
-
-
# list of supported backoff algorithms
-
30
BACKOFF_ALGORITHMS = %i[exponential_backoff polynomial_backoff].freeze
-
-
30
class << self
-
30
if ENV.key?("HTTPX_NO_JITTER")
-
30
def extra_options(options)
-
1063
options.merge(max_retries: MAX_RETRIES)
-
end
-
else
-
def extra_options(options)
-
options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
-
end
-
end
-
-
# returns the time to wait before resending +request+ as per the polynomial backoff retry strategy.
-
30
def retry_after_polynomial_backoff(request, _)
-
36
offset = request.options.max_retries - request.retries
-
36
2 * (offset - 1)
-
end
-
-
# returns the time to wait before resending +request+ as per the exponential backoff retry strategy.
-
30
def retry_after_exponential_backoff(request, _)
-
36
offset = request.options.max_retries - request.retries
-
36
(offset - 1) * 2
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
-
# :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
-
# :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
-
# or the name of a supported backoff algorithm (i.e. <tt>:exponential_backoff</tt>).
-
# :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
-
# :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
-
# (i.e. <tt>->(res) { ... }</tt>).
-
30
module OptionsMethods
-
30
private
-
-
30
def option_retry_after(value)
-
270
if value.respond_to?(:call)
-
126
value1 = value
-
126
value1 = value1.method(:call) unless value1.respond_to?(:arity)
-
-
# allow ->(*) arity as well, which is < 0
-
126
raise TypeError, "`:retry_after` proc has invalid number of parameters" unless value1.arity.negative? || value1.arity.between?(
-
1, 2
-
)
-
-
else
-
128
case value
-
when Symbol
-
36
raise TypeError, "`retry_after`: `#{value}` is not a supported backoff algorithm" unless BACKOFF_ALGORITHMS.include?(value)
-
-
36
value = Retries.method(:"retry_after_#{value}")
-
-
else
-
108
value = Float(value)
-
108
raise TypeError, "`:retry_after` must be positive" unless value.positive?
-
end
-
end
-
-
270
value
-
end
-
-
30
def option_retry_jitter(value)
-
# return early if callable
-
54
raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
-
-
54
value
-
end
-
-
30
def option_max_retries(value)
-
3477
num = Integer(value)
-
3477
raise TypeError, ":max_retries must be positive" unless num >= 0
-
-
3477
num
-
end
-
-
30
def option_retry_change_requests(v)
-
101
v
-
end
-
-
30
def option_retry_on(value)
-
414
raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
-
-
414
value
-
end
-
end
-
-
30
module InstanceMethods
-
# returns a `:retries` plugin enabled session with +n+ maximum retries per request setting.
-
30
def max_retries(n)
-
162
with(max_retries: n)
-
end
-
-
30
private
-
-
30
def fetch_response(request, selector, options)
-
4407
response = super
-
-
4407
if response &&
-
request.retries.positive? &&
-
retryable_request?(request, response, options) &&
-
retryable_response?(response, options)
-
969
try_partial_retry(request, response)
-
969
log { "failed to get response, #{request.retries} tries to go..." }
-
969
prepare_to_retry(request, response)
-
-
969
if (retry_after = when_to_retry(request, response, options)) && retry_after.positive?
-
-
144
retry_start = Utils.now
-
144
log { "retrying after #{retry_after} secs..." }
-
144
selector.after(retry_after) do
-
144
if (response = request.response)
-
response.finish!
-
# request has terminated abruptly meanwhile
-
request.emit_response(response)
-
else
-
144
log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-
144
send_request(request, selector, options)
-
end
-
end
-
-
128
return
-
else
-
825
send_request(request, selector, options)
-
-
# recalling itself, in case an error was triggered by the above, and we can
-
# verify retriability again.
-
742
return fetch_response(request, selector, options)
-
end
-
end
-
3438
response
-
end
-
-
# returns whether +request+ can be retried.
-
30
def retryable_request?(request, _, options)
-
1772
IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
-
end
-
-
30
def retryable_response?(response, options)
-
1362
(response.is_a?(ErrorResponse) && retryable_error?(response.error, options)) || options.retry_on&.call(response)
-
end
-
-
# returns whether the +ex+ exception happend for a retriable request.
-
30
def retryable_error?(ex, _)
-
7665
RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
-
end
-
-
30
def proxy_error?(request, response, _)
-
72
super && !request.retries.positive?
-
end
-
-
30
def prepare_to_retry(request, _response)
-
969
request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
-
969
request.transition(:idle)
-
end
-
-
30
def when_to_retry(request, response, options)
-
933
retry_after = options.retry_after
-
933
retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
-
# apply jitter
-
933
if (jitter = request.options.retry_jitter)
-
18
retry_after = jitter.call(retry_after)
-
end
-
933
retry_after
-
end
-
-
#
-
# Attempt to set the request to perform a partial range request.
-
# This happens if the peer server accepts byte-range requests, and
-
# the last response contains some body payload.
-
#
-
30
def try_partial_retry(request, response)
-
969
response = response.response if response.is_a?(ErrorResponse)
-
-
969
return unless response
-
-
403
unless response.headers.key?("accept-ranges") &&
-
response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
-
18
(original_body = response.body)
-
385
response.body.close
-
343
return
-
end
-
-
18
request.partial_response = response
-
-
18
size = original_body.bytesize
-
-
16
request.headers["range"] = "bytes=#{size}-"
-
end
-
end
-
-
30
module RequestMethods
-
# number of retries left.
-
30
attr_accessor :retries
-
-
# a response partially received before.
-
30
attr_writer :partial_response
-
-
# initializes the request instance, sets the number of retries for the request.
-
30
def initialize(*args)
-
1330
super
-
1330
@retries = @options.max_retries
-
1330
@partial_response = nil
-
end
-
-
30
def response=(response)
-
2345
if (partial_response = @partial_response)
-
18
if response.is_a?(Response) && response.status == 206
-
18
response.from_partial_response(partial_response)
-
else
-
partial_response.close
-
end
-
18
@partial_response = nil
-
end
-
-
2345
super
-
end
-
end
-
-
30
module ResponseMethods
-
30
def from_partial_response(response)
-
18
@status = response.status
-
18
@headers = response.headers
-
18
@body = response.body
-
end
-
end
-
end
-
30
register_plugin :retries, Retries
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
class ServerSideRequestForgeryError < Error; end
-
-
23
module Plugins
-
#
-
# This plugin adds support for preventing Server-Side Request Forgery attacks.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
-
#
-
23
module SsrfFilter
-
23
module IPAddrExtensions
-
23
refine IPAddr do
-
23
def prefixlen
-
368
mask_addr = @mask_addr
-
368
raise "Invalid mask" if mask_addr.zero?
-
-
627
mask_addr >>= 1 while mask_addr.nobits?(0x1)
-
-
368
length = 0
-
621
while mask_addr & 0x1 == 0x1
-
5566
length += 1
-
5566
mask_addr >>= 1
-
end
-
-
368
length
-
end
-
end
-
end
-
-
23
using IPAddrExtensions
-
-
# https://en.wikipedia.org/wiki/Reserved_IP_addresses
-
2
IPV4_BLACKLIST = [
-
21
IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
-
IPAddr.new("10.0.0.0/8"), # Private network
-
IPAddr.new("100.64.0.0/10"), # Shared Address Space
-
IPAddr.new("127.0.0.0/8"), # Loopback
-
IPAddr.new("169.254.0.0/16"), # Link-local
-
IPAddr.new("172.16.0.0/12"), # Private network
-
IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
-
IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
-
IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
-
IPAddr.new("192.168.0.0/16"), # Private network
-
IPAddr.new("198.18.0.0/15"), # Network benchmark tests
-
IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
-
IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
-
IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
-
IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
-
IPAddr.new("255.255.255.255"), # Broadcast
-
].freeze
-
-
3
IPV6_BLACKLIST = ([
-
21
IPAddr.new("::1/128"), # Loopback
-
IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
-
IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
-
IPAddr.new("2001::/32"), # Teredo tunneling
-
IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
-
IPAddr.new("2001:20::/28"), # ORCHIDv2
-
IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
-
IPAddr.new("2002::/16"), # 6to4
-
IPAddr.new("fc00::/7"), # Unique local address
-
IPAddr.new("fe80::/10"), # Link-local address
-
IPAddr.new("ff00::/8"), # Multicast
-
] + IPV4_BLACKLIST.flat_map do |ipaddr|
-
368
prefixlen = ipaddr.prefixlen
-
-
368
ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
-
368
ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
-
-
368
[ipv4_compatible, ipv4_mapped]
-
end).freeze
-
-
23
class << self
-
23
def extra_options(options)
-
79
options.merge(allowed_schemes: %w[https http])
-
end
-
-
23
def unsafe_ip_address?(ipaddr)
-
88
range = ipaddr.to_range
-
88
return true if range.first != range.last
-
-
106
return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
-
-
760
IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
-
23
module OptionsMethods
-
23
private
-
-
23
def option_allowed_schemes(value)
-
88
Array(value)
-
end
-
end
-
-
23
module InstanceMethods
-
23
def send_requests(*requests)
-
97
responses = requests.map do |request|
-
97
next if @options.allowed_schemes.include?(request.uri.scheme)
-
-
9
error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
-
9
error.set_backtrace(caller)
-
9
response = ErrorResponse.new(request, error)
-
9
request.response = response
-
9
request.emit_response(response)
-
9
response
-
end
-
194
allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
-
97
allowed_responses = super(*allowed_requests)
-
97
allowed_responses.each_with_index do |res, idx|
-
88
req = allowed_requests[idx]
-
78
responses[requests.index(req)] = res
-
end
-
-
97
responses
-
end
-
end
-
-
23
module ConnectionMethods
-
23
def initialize(*)
-
begin
-
88
super
-
8
rescue ServerSideRequestForgeryError => e
-
# may raise when IPs are passed as options via :addresses
-
18
throw(:resolve_error, e)
-
end
-
end
-
-
23
def addresses=(addrs)
-
88
addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
-
-
88
raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
-
-
18
super
-
end
-
end
-
end
-
-
23
register_plugin :ssrf_filter, SsrfFilter
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
class StreamResponse
-
30
attr_reader :request
-
-
30
def initialize(request, session)
-
261
@request = request
-
261
@options = @request.options
-
261
@session = session
-
261
@response_enum = nil
-
261
@buffered_chunks = []
-
end
-
-
30
def each(&block)
-
437
return enum_for(__method__) unless block
-
-
279
if (response_enum = @response_enum)
-
18
@response_enum = nil
-
# streaming already started, let's finish it
-
-
48
while (chunk = @buffered_chunks.shift)
-
18
block.call(chunk)
-
end
-
-
# consume enum til the end
-
1
begin
-
59
while (chunk = response_enum.next)
-
30
block.call(chunk)
-
end
-
rescue StopIteration
-
18
return
-
end
-
end
-
-
261
@request.stream = self
-
-
22
begin
-
261
@on_chunk = block
-
-
261
response = @session.request(@request)
-
-
234
response.raise_for_status
-
ensure
-
234
@on_chunk = nil
-
end
-
end
-
-
30
def each_line
-
122
return enum_for(__method__) unless block_given?
-
-
61
line = "".b
-
-
61
each do |chunk|
-
55
line << chunk
-
-
156
while (idx = line.index("\n"))
-
61
yield line.byteslice(0..(idx - 1))
-
-
61
line = line.byteslice((idx + 1)..-1)
-
end
-
end
-
-
25
yield line unless line.empty?
-
end
-
-
# This is a ghost method. It's to be used ONLY internally, when processing streams
-
30
def on_chunk(chunk)
-
569
raise NoMethodError unless @on_chunk
-
-
569
@on_chunk.call(chunk)
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id}>"
-
skipped
end
-
skipped
# :nocov:
-
-
30
def to_s
-
18
if @request.response
-
@request.response.to_s
-
else
-
18
@buffered_chunks.join
-
end
-
end
-
-
30
private
-
-
30
def response
-
469
@request.response || begin
-
50
response_enum = each
-
90
while (chunk = response_enum.next)
-
45
@buffered_chunks << chunk
-
45
break if @request.response
-
end
-
45
@response_enum = response_enum
-
45
@request.response
-
end
-
end
-
-
30
def respond_to_missing?(meth, include_private)
-
37
if (response = @request.response)
-
response.respond_to_missing?(meth, include_private)
-
else
-
37
@options.response_class.method_defined?(meth) || (include_private && @options.response_class.private_method_defined?(meth))
-
end || super
-
end
-
-
30
def method_missing(meth, *args, **kwargs, &block)
-
237
return super unless response.respond_to?(meth)
-
-
232
response.__send__(meth, *args, **kwargs, &block)
-
end
-
end
-
-
30
module Plugins
-
#
-
# This plugin adds support for streaming a response (useful for i.e. "text/event-stream" payloads).
-
#
-
# https://gitlab.com/os85/httpx/wikis/Stream
-
#
-
30
module Stream
-
30
STREAM_REQUEST_OPTIONS = { timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 }.freeze }.freeze
-
-
30
def self.extra_options(options)
-
457
options.merge(
-
stream: false,
-
timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 },
-
stream_response_class: Class.new(StreamResponse, &Options::SET_TEMPORARY_NAME).freeze
-
)
-
end
-
-
# adds support for the following options:
-
#
-
# :stream :: whether the request to process should be handled as a stream (defaults to <tt>false</tt>).
-
# :stream_response_class :: Class used to build the stream response object.
-
30
module OptionsMethods
-
30
def option_stream(val)
-
363
val
-
end
-
-
30
def option_stream_response_class(value)
-
1103
value
-
end
-
-
30
def extend_with_plugin_classes(pl)
-
274
return super unless defined?(pl::StreamResponseMethods)
-
-
166
@stream_response_class = @stream_response_class.dup
-
166
Options::SET_TEMPORARY_NAME[@stream_response_class, pl]
-
166
@stream_response_class.__send__(:include, pl::StreamResponseMethods) if defined?(pl::StreamResponseMethods)
-
-
166
super
-
end
-
end
-
-
30
module InstanceMethods
-
30
def request(*args, **options)
-
686
if args.first.is_a?(Request)
-
471
requests = args
-
-
471
request = requests.first
-
-
471
unless request.options.stream && !request.stream
-
380
if options[:stream]
-
warn "passing `stream: true` with a request object is not supported anymore. " \
-
"You can instead build the request object with `stream :true`"
-
end
-
380
return super
-
end
-
else
-
215
return super unless options[:stream]
-
-
188
requests = build_requests(*args, options)
-
-
188
request = requests.first
-
end
-
-
279
raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
-
-
261
@options.stream_response_class.new(request, self)
-
end
-
-
30
def build_request(verb, uri, params = EMPTY_HASH, options = @options)
-
477
return super unless params[:stream]
-
-
306
super(verb, uri, params, options.merge(STREAM_REQUEST_OPTIONS.merge(stream: true)))
-
end
-
end
-
-
30
module RequestMethods
-
30
attr_accessor :stream
-
end
-
-
30
module ResponseMethods
-
30
def stream
-
423
request = @request.root_request if @request.respond_to?(:root_request)
-
423
request ||= @request
-
-
423
request.stream
-
end
-
end
-
-
30
module ResponseBodyMethods
-
30
def initialize(*)
-
423
super
-
423
@stream = @response.stream
-
end
-
-
30
def write(chunk)
-
777
return super unless @stream
-
-
666
return 0 if chunk.empty?
-
-
569
chunk = decode_chunk(chunk)
-
-
569
@stream.on_chunk(chunk.dup)
-
-
542
chunk.bytesize
-
end
-
-
30
private
-
-
30
def transition(*)
-
167
return if @stream
-
-
139
super
-
end
-
end
-
end
-
30
register_plugin :stream, Stream
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds support for bidirectional HTTP/2 streams.
-
#
-
# https://gitlab.com/os85/httpx/wikis/StreamBidi
-
#
-
# It is required that the request body allows chunk to be buffered, (i.e., responds to +#<<(chunk)+).
-
23
module StreamBidi
-
# Extension of the Connection::HTTP2 class, which adds functionality to
-
# deal with a request that can't be drained and must be interleaved with
-
# the response streams.
-
#
-
# The streams keeps send DATA frames while there's data; when they're ain't,
-
# the stream is kept open; it must be explicitly closed by the end user.
-
#
-
23
module HTTP2Methods
-
23
def initialize(*)
-
101
super
-
101
@lock = Thread::Mutex.new
-
end
-
-
23
%i[close empty? exhausted? send <<].each do |lock_meth|
-
115
class_eval(<<-METH, __FILE__, __LINE__ + 1)
-
5
# lock.aware version of +#{lock_meth}+
-
5
def #{lock_meth}(*) # def close(*)
-
return super unless @options.stream
-
-
return super if @lock.owned?
-
-
# small race condition between
-
# checking for ownership and
-
# acquiring lock.
-
# TODO: fix this at the parser.
-
@lock.synchronize { super }
-
end
-
METH
-
end
-
-
23
private
-
-
23
%i[join_headers join_trailers join_body].each do |lock_meth|
-
69
class_eval(<<-METH, __FILE__, __LINE__ + 1)
-
3
# lock.aware version of +#{lock_meth}+
-
3
private def #{lock_meth}(*) # private def join_headers(*)
-
return super unless @options.stream
-
-
return super if @lock.owned?
-
-
# small race condition between
-
# checking for ownership and
-
# acquiring lock.
-
# TODO: fix this at the parser.
-
@lock.synchronize { super }
-
end
-
METH
-
end
-
-
23
def handle_stream(stream, request)
-
101
return super unless @options.stream
-
-
92
request.flush_buffer_on_body do
-
341
next unless request.headers_sent
-
-
250
handle(request, stream)
-
-
250
emit(:flush_buffer)
-
end
-
92
super
-
end
-
-
# when there ain't more chunks, it makes the buffer as full.
-
23
def send_chunk(request, stream, chunk, next_chunk)
-
374
return super unless @options.stream
-
-
374
super
-
-
374
return if next_chunk
-
-
340
request.transition(:waiting_for_chunk)
-
340
throw(:buffer_full)
-
end
-
-
# sets end-stream flag when the request is closed.
-
23
def end_stream?(request, next_chunk)
-
374
return super unless @options.stream
-
-
374
request.closed? && next_chunk.nil?
-
end
-
end
-
-
# BidiBuffer is a thread-safe Buffer which can receive data from any thread.
-
#
-
# It uses a dual-buffer strategy with mutex protection:
-
# - +@buffer+ is the main buffer, protected by +@buffer_mutex+
-
# - +@oob_buffer+ receives data when +@buffer_mutex+ is contended
-
#
-
# This allows non-blocking writes from any thread while maintaining thread safety.
-
23
class BidiBuffer < Buffer
-
23
def initialize(*)
-
54
super
-
54
@buffer_mutex = Thread::Mutex.new
-
54
@oob_mutex = Thread::Mutex.new
-
54
@oob_buffer = "".b
-
end
-
-
# buffers the +chunk+ to be sent (thread-safe, non-blocking)
-
23
def <<(chunk)
-
if @buffer_mutex.try_lock
-
begin
-
super
-
ensure
-
@buffer_mutex.unlock
-
end
-
else
-
# another thread holds the lock, use OOB buffer to avoid blocking
-
@oob_mutex.synchronize { @oob_buffer << chunk }
-
end
-
end
-
-
# reconciles the main and secondary buffer (thread-safe, callable from any thread).
-
23
def rebuffer
-
3940
@buffer_mutex.synchronize do
-
3940
@oob_mutex.synchronize do
-
3940
return if @oob_buffer.empty?
-
-
@buffer << @oob_buffer
-
@oob_buffer.clear
-
end
-
end
-
end
-
-
23
Buffer.instance_methods - Object.instance_methods - %i[<<].each do |meth|
-
23
class_eval(<<-MOD, __FILE__, __LINE__ + 1)
-
1
def #{meth}(*) # def empty?
-
@buffer_mutex.synchronize { super }
-
end
-
MOD
-
end
-
end
-
-
# Proxy to wake up the session main loop when one
-
# of the connections has buffered data to write. It abides by the HTTPX::_Selectable API,
-
# which allows it to be registered in the selector alongside actual HTTP-based
-
# HTTPX::Connection objects.
-
23
class Signal
-
23
attr_reader :error
-
-
23
def initialize
-
108
@closed = false
-
108
@error = nil
-
108
@pipe_read, @pipe_write = IO.pipe
-
end
-
-
23
def state
-
834
@closed ? :closed : :open
-
end
-
-
# noop
-
23
def log(**, &_); end
-
-
23
def to_io
-
1557
@pipe_read.to_io
-
end
-
-
23
def wakeup
-
250
return if @closed
-
-
205
@pipe_write.write("\0")
-
end
-
-
23
def call
-
199
return if @closed
-
-
199
@pipe_read.readpartial(1)
-
end
-
-
23
def interests
-
834
return if @closed
-
-
825
:r
-
end
-
-
23
def timeout; end
-
-
23
def inflight?
-
!@closed
-
end
-
-
23
def force_close(*)
-
terminate
-
end
-
-
23
def terminate
-
81
return if @closed
-
-
63
@pipe_write.close
-
63
@pipe_read.close
-
63
@closed = true
-
end
-
-
23
def on_error(error)
-
@error = error
-
terminate
-
end
-
-
23
alias_method :on_io_error, :on_error
-
-
# noop (the owner connection will take of it)
-
23
def handle_socket_timeout(interval); end
-
end
-
-
23
class << self
-
23
def load_dependencies(klass)
-
81
klass.plugin(:stream)
-
end
-
-
23
def extra_options(options)
-
81
options.merge(fallback_protocol: "h2")
-
end
-
end
-
-
23
module InstanceMethods
-
23
def initialize(*)
-
108
@signal = Signal.new
-
108
super
-
end
-
-
23
def close(selector = Selector.new)
-
81
@signal.terminate
-
81
selector.deregister(@signal)
-
81
super
-
end
-
-
23
def select_connection(connection, selector)
-
155
return super unless connection.options.stream
-
-
146
super
-
146
selector.register(@signal)
-
146
connection.signal = @signal
-
end
-
-
23
def deselect_connection(connection, *)
-
92
return super unless connection.options.stream
-
-
83
super
-
-
83
connection.signal = nil
-
end
-
end
-
-
# Adds synchronization to request operations which may buffer payloads from different
-
# threads.
-
23
module RequestMethods
-
23
attr_accessor :headers_sent
-
-
23
def initialize(*)
-
81
super
-
72
@headers_sent = false
-
72
@closed = false
-
72
@flush_buffer_on_body_cb = nil
-
72
@mutex = Thread::Mutex.new
-
end
-
-
23
def flush_buffer_on_body(&cb)
-
92
@flush_buffer_on_body_cb = on(:body, &cb)
-
end
-
-
23
def closed?
-
374
return super unless @options.stream
-
-
374
@closed
-
end
-
-
23
def can_buffer?
-
821
return super unless @options.stream
-
-
805
super && @state != :waiting_for_chunk
-
end
-
-
# overrides state management transitions to introduce an intermediate
-
# +:waiting_for_chunk+ state, which the request transitions to once payload
-
# is buffered.
-
23
def transition(nextstate)
-
1383
return super unless @options.stream
-
-
1347
headers_sent = @headers_sent
-
-
1187
case nextstate
-
when :idle
-
29
headers_sent = false
-
-
29
if @flush_buffer_on_body_cb
-
29
callbacks(:body).delete(@flush_buffer_on_body_cb)
-
29
@flush_buffer_on_body_cb = nil
-
end
-
when :waiting_for_chunk
-
340
return unless @state == :body
-
when :body
-
546
case @state
-
when :headers
-
92
headers_sent = true
-
when :waiting_for_chunk
-
# HACK: to allow super to pass through
-
249
@state = :headers
-
end
-
end
-
-
1346
super.tap do
-
# delay setting this up until after the first transition to :body
-
1346
@headers_sent = headers_sent
-
end
-
end
-
-
23
def <<(chunk)
-
270
@mutex.synchronize do
-
270
if @drainer
-
250
@body.clear if @body.respond_to?(:clear)
-
250
@drainer = nil
-
end
-
270
@body << chunk
-
-
270
transition(:body)
-
end
-
end
-
-
23
def close
-
63
return super unless @options.stream
-
-
63
@mutex.synchronize do
-
63
return if @closed
-
-
63
@closed = true
-
end
-
-
# last chunk to send which ends the stream
-
63
self << ""
-
end
-
end
-
-
23
module RequestBodyMethods
-
23
def initialize(*, **)
-
81
super
-
-
81
return unless @options.stream
-
-
72
@headers.delete("content-length")
-
-
72
return unless @body
-
-
72
return if @body.is_a?(Transcoder::Body::Encoder)
-
-
9
raise Error, "bidirectional streams only allow the usage of the `:body` param to set request bodies." \
-
"You must encode it yourself if you wish to do so."
-
end
-
-
23
def empty?
-
495
return super unless @options.stream
-
-
468
false
-
end
-
end
-
-
# overrides the declaration of +@write_buffer+, which is now a thread-safe buffer
-
# responding to the same API.
-
23
module ConnectionMethods
-
23
attr_writer :signal
-
-
23
def initialize(*)
-
63
super
-
-
63
return unless @options.stream
-
-
54
@write_buffer = BidiBuffer.new(@options.buffer_size)
-
end
-
-
# rebuffers the +@write_buffer+ before calculating interests.
-
23
def interests
-
4106
return super unless @options.stream
-
-
3940
@write_buffer.rebuffer
-
-
3940
super
-
end
-
-
23
def call
-
756
return super unless @options.stream && (error = @signal.error)
-
-
on_error(error)
-
end
-
-
23
private
-
-
23
def set_parser_callbacks(parser)
-
101
return super unless @options.stream
-
-
92
super
-
92
parser.on(:flush_buffer) do
-
250
@signal.wakeup if @signal
-
end
-
end
-
end
-
end
-
23
register_plugin :stream_bidi, StreamBidi
-
end
-
end
-
# frozen_string_literal: true
-
-
29
module HTTPX::Plugins
-
#
-
# This plugin adds a simple interface to integrate request tracing SDKs.
-
#
-
# An example of such an integration is the datadog adapter.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Tracing
-
#
-
29
module Tracing
-
29
class Wrapper
-
29
attr_reader :tracers
-
29
protected :tracers
-
-
29
def initialize(*tracers)
-
54
@tracers = tracers.flat_map do |tracer|
-
96
case tracer
-
when Wrapper
-
18
tracer.tracers
-
else
-
90
tracer
-
end
-
end.uniq
-
end
-
-
29
def merge(tracer)
-
case tracer
-
when Wrapper
-
Wrapper.new(*@tracers, *tracer.tracers)
-
else
-
Wrapper.new(*@tracers, tracer)
-
end
-
end
-
-
29
def freeze
-
@tracers.each(&:freeze).freeze
-
super
-
end
-
-
29
%i[start finish reset enabled?].each do |callback|
-
116
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
4
# proxies ##{callback} calls to wrapper tracers.
-
4
def #{callback}(*args) # def start(*args)
-
4
@tracers.each { |t| t.#{callback}(*args) } # @tracers.each { |t| t.start(*args) }
-
end # end
-
OUT
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :tracer :: object which responds to #start, #finish and #reset.
-
29
module OptionsMethods
-
29
private
-
-
29
def option_tracer(tracer)
-
167
unless tracer.respond_to?(:start) &&
-
tracer.respond_to?(:finish) &&
-
tracer.respond_to?(:reset) &&
-
tracer.respond_to?(:enabled?)
-
raise TypeError, "#{tracer} must to respond to `#start(r)`, `#finish` and `#reset` and `#enabled?"
-
end
-
-
167
tracer = Wrapper.new(@tracer, tracer) if @tracer
-
167
tracer
-
end
-
end
-
-
29
module RequestMethods
-
29
attr_accessor :init_time
-
-
# intercepts request initialization to inject the tracing logic.
-
29
def initialize(*)
-
329
super
-
-
329
@init_time = nil
-
-
329
tracer = @options.tracer
-
-
329
return unless tracer && tracer.enabled?(self)
-
-
152
on(:idle) do
-
82
tracer.reset(self)
-
-
# request is reset when it's retried.
-
82
@init_time = nil
-
end
-
152
on(:headers) do
-
# the usual request init time (when not including the connection handshake)
-
# should be the time the request is buffered the first time.
-
220
@init_time ||= ::Time.now.utc
-
-
220
tracer.start(self)
-
end
-
372
on(:response) { |response| tracer.finish(self, response) }
-
end
-
-
29
def response=(*)
-
# init_time should be set when it's send to a connection.
-
# However, there are situations where connection initialization fails.
-
# Example is the :ssrf_filter plugin, which raises an error on
-
# initialize if the host is an IP which matches against the known set.
-
# in such cases, we'll just set here right here.
-
400
@init_time ||= ::Time.now.utc
-
-
400
super
-
end
-
end
-
-
# Connection mixin
-
29
module ConnectionMethods
-
29
def initialize(*)
-
309
super
-
-
309
@init_time = ::Time.now.utc
-
end
-
-
29
def send(request)
-
# request init time is only the same as the connection init time
-
# if the connection is going through the connection handshake.
-
286
request.init_time ||= @init_time unless open?
-
-
286
super
-
end
-
-
29
def idling
-
75
super
-
-
# time of initial request(s) is accounted from the moment
-
# the connection is back to :idle, and ready to connect again.
-
75
@init_time = ::Time.now.utc
-
end
-
end
-
end
-
29
register_plugin :tracing, Tracing
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
-
# Upgrade header.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Upgrade
-
#
-
23
module Upgrade
-
23
class << self
-
23
def configure(klass)
-
35
klass.plugin(:"upgrade/h2")
-
end
-
-
23
def extra_options(options)
-
35
options.merge(upgrade_handlers: {})
-
end
-
end
-
-
23
module OptionsMethods
-
23
private
-
-
23
def option_upgrade_handlers(value)
-
115
raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
-
-
115
value
-
end
-
end
-
-
23
module InstanceMethods
-
23
def fetch_response(request, selector, options)
-
183
response = super
-
-
183
return unless response
-
-
104
return response unless response.is_a?(Response)
-
-
104
return response unless response.headers.key?("upgrade")
-
-
42
upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
-
-
42
return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
-
-
42
protocol_handler = options.upgrade_handlers[upgrade_protocol]
-
-
42
return response unless protocol_handler
-
-
42
log { "upgrading to #{upgrade_protocol}..." }
-
42
connection = find_connection(request.uri, selector, options)
-
-
# do not upgrade already upgraded connections
-
42
return if connection.upgrade_protocol == upgrade_protocol
-
-
35
protocol_handler.call(connection, request, response)
-
-
# keep in the loop if the server is switching, unless
-
# the connection has been hijacked, in which case you want
-
# to terminante immediately
-
35
return fetch_response(request, selector, options) if response.status == 101 && !connection.hijacked
-
-
17
response
-
end
-
end
-
-
23
module ConnectionMethods
-
23
attr_reader :upgrade_protocol, :hijacked
-
-
23
def initialize(*)
-
44
super
-
-
44
@upgrade_protocol = nil
-
end
-
-
23
def hijack_io
-
9
@hijacked = true
-
-
# connection is taken away from selector and not given back to the pool.
-
9
@current_session.deselect_connection(self, @current_selector, true)
-
end
-
end
-
end
-
23
register_plugin(:upgrade, Upgrade)
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin adds support for upgrading an HTTP/1.1 connection to HTTP/2
-
# via an Upgrade: h2 response declaration
-
#
-
# https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2
-
#
-
23
module H2
-
23
class << self
-
23
def extra_options(options)
-
35
options.merge(upgrade_handlers: options.upgrade_handlers.merge("h2" => self))
-
end
-
-
23
def call(connection, _request, _response)
-
8
connection.upgrade_to_h2
-
end
-
end
-
-
23
module ConnectionMethods
-
23
using URIExtensions
-
-
23
def interests
-
914
return super unless connecting? && @parser
-
-
15
connect
-
-
15
return @io.interests if connecting?
-
-
super
-
end
-
-
23
def upgrade_to_h2
-
8
enqueue_pending_requests_from_parser(@parser)
-
-
8
@parser = @options.http2_class.new(@write_buffer, @options)
-
8
set_parser_callbacks(@parser)
-
8
@upgrade_protocol = "h2"
-
-
# what's happening here:
-
# a deviation from the state machine is done to perform the actions when a
-
# connection is closed, without transitioning, so the connection is kept in the pool.
-
# the state is reset to initial, so that the socket reconnect works out of the box,
-
# while the parser is already here.
-
8
purge_after_closed
-
8
transition(:idle)
-
end
-
end
-
end
-
23
register_plugin(:"upgrade/h2", H2)
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin implements convenience methods for performing WEBDAV requests.
-
#
-
# https://gitlab.com/os85/httpx/wikis/WebDav
-
#
-
23
module WebDav
-
23
def self.configure(klass)
-
108
klass.plugin(:xml)
-
end
-
-
23
module InstanceMethods
-
23
def copy(src, dest)
-
18
request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
-
end
-
-
23
def move(src, dest)
-
18
request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
-
end
-
-
23
def lock(path, timeout: nil, &blk)
-
54
headers = {}
-
48
headers["timeout"] = if timeout && timeout.positive?
-
18
"Second-#{timeout}"
-
else
-
36
"Infinite, Second-4100000000"
-
end
-
54
xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
-
"<D:lockinfo xmlns:D=\"DAV:\">" \
-
"<D:lockscope><D:exclusive/></D:lockscope>" \
-
"<D:locktype><D:write/></D:locktype>" \
-
"<D:owner>null</D:owner>" \
-
"</D:lockinfo>"
-
54
response = request("LOCK", path, headers: headers, xml: xml)
-
-
54
return response unless response.is_a?(Response)
-
-
54
return response unless blk && response.status == 200
-
-
18
lock_token = response.headers["lock-token"]
-
-
1
begin
-
18
blk.call(response)
-
ensure
-
18
unlock(path, lock_token)
-
end
-
-
18
response
-
end
-
-
23
def unlock(path, lock_token)
-
36
request("UNLOCK", path, headers: { "lock-token" => lock_token })
-
end
-
-
23
def mkcol(dir)
-
18
request("MKCOL", dir)
-
end
-
-
23
def propfind(path, xml = nil)
-
72
body = case xml
-
when :acl
-
18
'<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
-
"<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
-
when nil
-
36
'<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
-
else
-
18
xml
-
end
-
-
72
request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
-
end
-
-
23
def proppatch(path, xml)
-
8
body = "<?xml version=\"1.0\"?>" \
-
12
"<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
-
18
request("PROPPATCH", path, xml: body)
-
end
-
# %i[ orderpatch acl report search]
-
end
-
end
-
23
register_plugin(:webdav, WebDav)
-
end
-
end
-
# frozen_string_literal: true
-
-
23
module HTTPX
-
23
module Plugins
-
#
-
# This plugin supports request XML encoding/response decoding using the nokogiri gem.
-
#
-
# https://gitlab.com/os85/httpx/wikis/XML
-
#
-
23
module XML
-
23
MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
-
23
module Transcoder
-
23
module_function
-
-
23
class Encoder
-
23
def initialize(xml)
-
180
@raw = xml
-
end
-
-
23
def content_type
-
180
charset = @raw.respond_to?(:encoding) && @raw.encoding ? @raw.encoding.to_s.downcase : "utf-8"
-
180
"application/xml; charset=#{charset}"
-
end
-
-
23
def bytesize
-
576
@raw.to_s.bytesize
-
end
-
-
23
def to_s
-
180
@raw.to_s
-
end
-
end
-
-
23
def encode(xml)
-
180
Encoder.new(xml)
-
end
-
-
23
def decode(response)
-
27
content_type = response.content_type.mime_type
-
-
27
raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
-
-
27
Nokogiri::XML.method(:parse)
-
end
-
end
-
-
23
class << self
-
23
def load_dependencies(*)
-
162
require "nokogiri"
-
end
-
end
-
-
23
module ResponseMethods
-
# decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
-
# "application/xml" (requires the "nokogiri" gem).
-
23
def xml
-
18
decode(Transcoder)
-
end
-
end
-
-
23
module RequestBodyClassMethods
-
# ..., xml: Nokogiri::XML::Node #=> xml encoder
-
23
def initialize_body(params)
-
666
if (xml = params.delete(:xml))
-
# @type var xml: Nokogiri::XML::Node | String
-
160
return Transcoder.encode(xml)
-
end
-
-
486
super
-
end
-
end
-
end
-
-
23
register_plugin(:xml, XML)
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module ResponsePatternMatchExtensions
-
30
def deconstruct
-
46
[@status, @headers, @body]
-
end
-
-
30
def deconstruct_keys(_keys)
-
80
{ status: @status, headers: @headers, body: @body }
-
end
-
end
-
-
30
module ErrorResponsePatternMatchExtensions
-
30
def deconstruct
-
12
[@error]
-
end
-
-
30
def deconstruct_keys(_keys)
-
40
{ error: @error }
-
end
-
end
-
-
30
module HeadersPatternMatchExtensions
-
30
def deconstruct
-
8
to_a
-
end
-
end
-
-
30
Headers.include HeadersPatternMatchExtensions
-
30
Response.include ResponsePatternMatchExtensions
-
30
ErrorResponse.include ErrorResponsePatternMatchExtensions
-
end
-
# frozen_string_literal: true
-
-
30
require "httpx/selector"
-
30
require "httpx/connection"
-
30
require "httpx/connection/http2"
-
30
require "httpx/connection/http1"
-
30
require "httpx/resolver"
-
-
30
module HTTPX
-
30
class Pool
-
30
using URIExtensions
-
-
30
POOL_TIMEOUT = 5
-
-
# Sets up the connection pool with the given +options+, which can be the following:
-
#
-
# :max_connections:: the maximum number of connections held in the pool.
-
# :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
-
# :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
-
#
-
30
def initialize(options)
-
14589
@max_connections = options.fetch(:max_connections, Float::INFINITY)
-
14589
@max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
-
14589
@pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
-
22467
@resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
-
14589
@resolver_mtx = Thread::Mutex.new
-
14589
@connections = []
-
14589
@connection_mtx = Thread::Mutex.new
-
14589
@connections_counter = 0
-
14589
@max_connections_cond = ConditionVariable.new
-
14589
@origin_counters = Hash.new(0)
-
21541
@origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
-
end
-
-
# connections returned by this function are not expected to return to the connection pool.
-
30
def pop_connection
-
14972
@connection_mtx.synchronize do
-
14972
drop_connection
-
end
-
end
-
-
# opens a connection to the IP reachable through +uri+.
-
# Many hostnames are reachable through the same IP, so we try to
-
# maximize pipelining by opening as few connections as possible.
-
#
-
30
def checkout_connection(uri, options)
-
10467
return checkout_new_connection(uri, options) if options.io
-
-
10392
@connection_mtx.synchronize do
-
10392
acquire_connection(uri, options) || begin
-
9304
if @connections_counter == @max_connections
-
# this takes precedence over per-origin
-
-
18
expires_at = Utils.now + @pool_timeout
-
-
18
loop do
-
18
@max_connections_cond.wait(@connection_mtx, @pool_timeout)
-
-
18
if (conn = acquire_connection(uri, options))
-
4
return conn
-
end
-
-
# if one can afford to create a new connection, do it
-
14
break unless @connections_counter == @max_connections
-
-
# if no matching usable connection was found, the pool will make room and drop a closed connection.
-
15
if (conn = @connections.find { |c| c.state == :closed })
-
3
drop_connection(conn)
-
3
break
-
end
-
-
# happens when a condition was signalled, but another thread snatched the available connection before
-
# context was passed back here.
-
9
next if Utils.now < expires_at
-
-
9
raise PoolTimeoutError.new(@pool_timeout,
-
1
"Timed out after #{@pool_timeout} seconds while waiting for a connection")
-
end
-
-
end
-
-
9291
if @origin_counters[uri.origin] == @max_connections_per_origin
-
-
18
expires_at = Utils.now + @pool_timeout
-
-
18
loop do
-
18
@origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
-
-
18
if (conn = acquire_connection(uri, options))
-
9
return conn
-
end
-
-
# happens when a condition was signalled, but another thread snatched the available connection before
-
# context was passed back here.
-
9
next if Utils.now < expires_at
-
-
9
raise(PoolTimeoutError.new(@pool_timeout,
-
1
"Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
-
end
-
end
-
-
8388
@connections_counter += 1
-
9273
@origin_counters[uri.origin] += 1
-
-
9273
checkout_new_connection(uri, options)
-
end
-
end
-
end
-
-
30
def checkin_connection(connection)
-
10390
return if connection.options.io
-
-
10340
@connection_mtx.synchronize do
-
10340
if connection.coalesced? || connection.state == :idle
-
# when connections coalesce
-
57
drop_connection(connection)
-
-
57
return
-
end
-
-
10283
@connections << connection
-
-
10283
@max_connections_cond.signal
-
10283
@origin_conds[connection.origin.to_s].signal
-
end
-
end
-
-
30
def checkout_mergeable_connection(connection)
-
9272
return if connection.options.io
-
-
9272
@connection_mtx.synchronize do
-
9272
idx = @connections.find_index do |ch|
-
352
ch != connection && ch.mergeable?(connection)
-
end
-
9272
@connections.delete_at(idx) if idx
-
end
-
end
-
-
30
def reset_resolvers
-
18070
@resolver_mtx.synchronize { @resolvers.clear }
-
end
-
-
30
def checkout_resolver(options)
-
9032
resolver_type = options.resolver_class
-
-
9032
@resolver_mtx.synchronize do
-
9032
resolvers = @resolvers[resolver_type]
-
-
9032
idx = resolvers.find_index do |res|
-
284
res.options.resolver_options_match?(options)
-
end
-
9032
resolvers.delete_at(idx) if idx
-
end || checkout_new_resolver(resolver_type, options)
-
end
-
-
30
def checkin_resolver(resolver)
-
8834
if resolver.is_a?(Resolver::Multi)
-
8178
resolver_class = resolver.resolvers.first.class
-
else
-
656
resolver_class = resolver.class
-
-
656
resolver = resolver.multi
-
end
-
-
# a multi requires all sub-resolvers being closed in order to be
-
# correctly checked back in.
-
8834
return unless resolver.closed?
-
-
8810
@resolver_mtx.synchronize do
-
8810
resolvers = @resolvers[resolver_class]
-
-
8810
resolvers << resolver unless resolvers.include?(resolver)
-
end
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id} " \
-
skipped
"@max_connections=#{@max_connections} " \
-
skipped
"@max_connections_per_origin=#{@max_connections_per_origin} " \
-
skipped
"@pool_timeout=#{@pool_timeout} " \
-
skipped
"@connections=#{@connections.size}>"
-
skipped
end
-
skipped
# :nocov:
-
-
30
private
-
-
30
def acquire_connection(uri, options)
-
10428
idx = @connections.find_index do |connection|
-
1425
connection.match?(uri, options)
-
end
-
-
10428
return unless idx
-
-
1101
@connections.delete_at(idx)
-
end
-
-
30
def checkout_new_connection(uri, options)
-
9357
connection = options.connection_class.new(uri, options)
-
9425
connection.log(level: 2) { "created connection##{connection.object_id} in pool##{object_id}" }
-
9339
connection
-
end
-
-
30
def checkout_new_resolver(resolver_type, options)
-
8748
resolver = if resolver_type.multi?
-
8570
Resolver::Multi.new(resolver_type, options)
-
else
-
178
resolver_type.new(options)
-
end
-
8834
resolver.log(level: 2) { "created resolver##{resolver.object_id} in pool##{object_id}" }
-
8748
resolver
-
end
-
-
# drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),
-
# the first available connection from the pool will be dropped.
-
30
def drop_connection(connection = nil)
-
15032
if connection
-
60
@connections.delete(connection)
-
else
-
14972
connection = @connections.shift
-
-
14972
return unless connection
-
end
-
-
5418
@connections_counter -= 1
-
5997
@origin_conds.delete(connection.origin) if (@origin_counters[connection.origin.to_s] -= 1).zero?
-
-
5997
connection
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Punycode
-
30
module_function
-
-
begin
-
30
require "idnx"
-
-
29
def encode_hostname(hostname)
-
36
Idnx.to_punycode(hostname)
-
end
-
rescue LoadError
-
1
def encode_hostname(hostname)
-
1
warn "#{hostname} cannot be converted to punycode. Install the " \
-
"\"idnx\" gem: https://github.com/HoneyryderChuck/idnx"
-
-
1
hostname
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "delegate"
-
30
require "forwardable"
-
-
30
module HTTPX
-
# Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
-
# as well as maintaining the state machine which manages streaming the request onto the wire.
-
30
class Request
-
30
extend Forwardable
-
30
include Loggable
-
30
include Callbacks
-
-
30
using URIExtensions
-
-
30
ALLOWED_URI_SCHEMES = %w[https http].freeze
-
-
# the upcased string HTTP verb for this request.
-
30
attr_reader :verb
-
-
# the absolute URI object for this request.
-
30
attr_reader :uri
-
-
# an HTTPX::Headers object containing the request HTTP headers.
-
30
attr_reader :headers
-
-
# an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
-
30
attr_reader :body
-
-
# a symbol describing which frame is currently being flushed.
-
30
attr_reader :state
-
-
# an HTTPX::Options object containing request options.
-
30
attr_reader :options
-
-
# the corresponding HTTPX::Response object, when there is one.
-
30
attr_reader :response
-
-
# Exception raised during enumerable body writes.
-
30
attr_reader :drain_error
-
-
# The IP address from the peer server.
-
30
attr_accessor :peer_address
-
-
# the connection the request is currently being sent to (none if before or after transaction)
-
30
attr_writer :connection
-
-
# callback triggered when a response (which may not be the final response) was assigned to the request.
-
30
attr_writer :on_response_arrived
-
-
30
attr_writer :persistent
-
-
30
attr_reader :active_timeouts
-
-
# will be +true+ when request body has been completely flushed.
-
30
def_delegator :@body, :empty?
-
-
# closes the body
-
30
def_delegator :@body, :close
-
-
# initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
-
# an absolute or relative +uri+ (either as String or URI::HTTP object), the
-
# request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
-
#
-
# Besides any of the options documented in HTTPX::Options (which would override or merge with what
-
# +options+ sets), it accepts also the following:
-
#
-
# :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
-
# :body :: to be encoded in the request body payload. can be a String, an IO object (i.e. a File), or an Enumerable.
-
# :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
-
# :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
-
# :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
-
#
-
# :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
-
30
def initialize(verb, uri, options, params = EMPTY_HASH)
-
12852
@verb = verb.to_s.upcase
-
12852
@uri = Utils.to_uri(uri)
-
-
12851
@headers = options.headers.dup
-
12851
merge_headers(params.delete(:headers)) if params.key?(:headers)
-
-
12851
@query_params = params.delete(:params) if params.key?(:params)
-
-
12851
@body = options.request_body_class.new(@headers, options, **params)
-
-
12833
@options = @body.options
-
-
12833
if @uri.relative? || @uri.host.nil?
-
712
origin = @options.origin
-
712
raise(Error, "invalid URI: #{@uri}") unless origin
-
-
684
base_path = @options.base_path
-
-
684
@uri = origin.merge("#{base_path}#{@uri}")
-
end
-
-
12805
raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
-
-
12791
@state = :idle
-
2362
@connection = @response =
-
@drainer = @peer_address =
-
10429
@informational_status = @on_response_arrived = nil
-
12791
@ping = false
-
12791
@persistent = @options.persistent
-
12791
@active_timeouts = []
-
end
-
-
# dupped initialization
-
30
def initialize_dup(orig)
-
1510
super
-
1510
@uri = orig.instance_variable_get(:@uri).dup
-
1510
@headers = orig.instance_variable_get(:@headers).dup
-
1510
@body = orig.instance_variable_get(:@body).dup
-
end
-
-
30
def complete!(response = @response)
-
9939
emit(:complete, response)
-
end
-
-
# whether request has been buffered with a ping
-
30
def ping?
-
969
@ping
-
end
-
-
# marks the request as having been buffered with a ping
-
30
def ping!
-
64
@ping = true
-
end
-
-
# the read timeout defined for this request.
-
30
def read_timeout
-
21782
@options.timeout[:read_timeout]
-
end
-
-
# the write timeout defined for this request.
-
30
def write_timeout
-
21782
@options.timeout[:write_timeout]
-
end
-
-
# the request timeout defined for this request.
-
30
def request_timeout
-
21463
@options.timeout[:request_timeout]
-
end
-
-
30
def persistent?
-
6228
@persistent
-
end
-
-
# if the request contains trailer headers
-
30
def trailers?
-
3619
defined?(@trailers)
-
end
-
-
# returns an instance of HTTPX::Headers containing the trailer headers
-
30
def trailers
-
99
@trailers ||= @options.headers_class.new
-
end
-
-
# returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
-
30
def interests
-
58648
return :r if @state == :done || @state == :expect
-
-
5863
:w
-
end
-
-
30
def can_buffer?
-
33265
@state != :done
-
end
-
-
# merges +h+ into the instance of HTTPX::Headers of the request.
-
30
def merge_headers(h)
-
1397
@headers = @headers.merge(h)
-
1397
return unless @headers.key?("range")
-
-
18
@headers.delete("accept-encoding")
-
end
-
-
# the URI scheme of the request +uri+.
-
30
def scheme
-
4830
@uri.scheme
-
end
-
-
# sets the +response+ on this request.
-
30
def response=(response)
-
12309
return unless response
-
-
11166
case response
-
when Response
-
10516
if response.status < 200
-
# deal with informational responses
-
-
164
if response.status == 100 && @headers.key?("expect")
-
137
@informational_status = response.status
-
137
return
-
end
-
-
# 103 Early Hints advertises resources in document to browsers.
-
# not very relevant for an HTTP client, discard.
-
27
return if response.status >= 103
-
-
end
-
when ErrorResponse
-
1793
response.error.connection = nil if response.error.respond_to?(:connection=)
-
end
-
-
12172
@response = response
-
-
12172
emit(:response_started, response)
-
end
-
-
# returnns the URI path of the request +uri+.
-
30
def path
-
11764
path = uri.path.dup
-
11764
path = +"" if path.nil?
-
11764
path << "/" if path.empty?
-
11764
path << "?#{query}" unless query.empty?
-
11764
path
-
end
-
-
# returs the URI authority of the request.
-
#
-
# session.build_request("GET", "https://google.com/query").authority #=> "google.com"
-
# session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
-
30
def authority
-
11328
@uri.authority
-
end
-
-
# returs the URI origin of the request.
-
#
-
# session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
-
# session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
-
30
def origin
-
5159
@uri.origin
-
end
-
-
# returs the URI query string of the request (when available).
-
#
-
# session.build_request("GET", "https://search.com").query #=> ""
-
# session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
-
# session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
-
# session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
-
30
def query
-
12947
return @query if defined?(@query)
-
-
10557
query = []
-
10557
if (q = @query_params) && !q.empty?
-
190
query << Transcoder::Form.encode(q)
-
end
-
10557
query << @uri.query if @uri.query
-
10557
@query = query.join("&")
-
end
-
-
# consumes and returns the next available chunk of request body that can be sent
-
30
def drain_body
-
11386
return nil if @body.nil?
-
-
11386
@drainer ||= @body.each
-
11386
chunk = @drainer.next.dup
-
-
7436
emit(:body_chunk, chunk)
-
7436
chunk
-
rescue StopIteration
-
3922
nil
-
rescue StandardError => e
-
28
@drain_error = e
-
28
nil
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id} " \
-
skipped
"#{@verb} " \
-
skipped
"#{uri} " \
-
skipped
"@headers=#{@headers} " \
-
skipped
"@body=#{@body}>"
-
skipped
end
-
skipped
# :nocov:
-
-
# moves on to the +nextstate+ of the request state machine (when all preconditions are met)
-
30
def transition(nextstate)
-
59664
case nextstate
-
when :idle
-
12200
@body.rewind
-
12200
@ping = false
-
12200
@response = @drainer = nil
-
12200
@active_timeouts.clear
-
when :headers
-
14443
return unless @state == :idle
-
-
when :body
-
14695
return unless @state == :headers ||
-
@state == :expect
-
-
11933
if @headers.key?("expect")
-
525
if @informational_status && @informational_status == 100
-
# check for 100 Continue response, and deallocate the var
-
# if @informational_status == 100
-
# @response = nil
-
# end
-
else
-
397
return if @state == :expect # do not re-set it
-
-
146
nextstate = :expect
-
end
-
end
-
when :trailers
-
11590
return unless @state == :body
-
when :done
-
11599
return if @state == :expect
-
-
end
-
58162
log(level: 3) { "#{@state}] -> #{nextstate}" }
-
57906
@state = nextstate
-
57906
emit(@state, self)
-
32553
nil
-
end
-
-
# whether the request supports the 100-continue handshake and already processed the 100 response.
-
30
def expects?
-
10163
@headers["expect"] == "100-continue" && @informational_status == 100 && !@response
-
end
-
-
30
def set_timeout_callback(event, &callback)
-
78430
clb = once(event, &callback)
-
-
# reset timeout callbacks when requests get rerouted to a different connection
-
78430
once(:idle) do
-
28606
callbacks(event).delete(clb)
-
end
-
end
-
-
30
def handle_error(error)
-
785
if (connection = @connection)
-
767
connection.on_error(error, self)
-
else
-
18
response = ErrorResponse.new(self, error)
-
18
self.response = response
-
18
emit_response(response)
-
end
-
end
-
-
30
def emit_response(response)
-
11888
emit(:response, response)
-
-
11870
return unless @on_response_arrived
-
-
11303
@on_response_arrived.call
-
end
-
end
-
end
-
-
30
require_relative "request/body"
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
# Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
-
30
class Request::Body < SimpleDelegator
-
30
class << self
-
30
def new(_, options, body: nil, **params)
-
12860
if body.is_a?(self)
-
# request derives its options from body
-
18
body.options = options.merge(params)
-
16
return body
-
end
-
-
12842
super
-
end
-
end
-
-
30
attr_accessor :options
-
-
# inits the instance with the request +headers+, +options+ and +params+, which contain the payload definition.
-
# it wraps the given body with the appropriate encoder on initialization.
-
#
-
# ..., json: { foo: "bar" }) #=> json encoder
-
# ..., form: { foo: "bar" }) #=> form urlencoded encoder
-
# ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
-
# ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
-
# ..., form: { body: "bla") }) #=> raw data encoder
-
30
def initialize(h, options, **params)
-
12842
@headers = h
-
12842
@body = self.class.initialize_body(params)
-
12842
@options = options.merge(params)
-
-
12842
if @body
-
3847
if @options.compress_request_body && @headers.key?("content-encoding")
-
-
109
@headers.get("content-encoding").each do |encoding|
-
109
@body = self.class.initialize_deflater_body(@body, encoding)
-
end
-
end
-
-
3847
@headers["content-type"] ||= @body.content_type
-
3847
@headers["content-length"] = @body.bytesize unless unbounded_body?
-
end
-
-
12833
super(@body)
-
end
-
-
# consumes and yields the request payload in chunks.
-
30
def each(&block)
-
8167
return enum_for(__method__) unless block
-
4088
return if @body.nil?
-
-
4007
body = stream(@body)
-
4007
if body.respond_to?(:read)
-
6539
while (chunk = body.read(16_384))
-
4005
block.call(chunk)
-
end
-
# TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
-
# IO.copy_stream(body, ProcIO.new(block))
-
2365
elsif body.respond_to?(:each)
-
838
body.each(&block)
-
else
-
1528
block[body.to_s]
-
end
-
end
-
-
30
def close
-
531
@body.close if @body.respond_to?(:close)
-
end
-
-
# if the +@body+ is rewindable, it rewinnds it.
-
30
def rewind
-
12272
return if empty?
-
-
244
@body.rewind if @body.respond_to?(:rewind)
-
end
-
-
# return +true+ if the +body+ has been fully drained (or does nnot exist).
-
30
def empty?
-
35941
return true if @body.nil?
-
10201
return false if chunked?
-
-
10093
@body.bytesize.zero?
-
end
-
-
# returns the +@body+ payload size in bytes.
-
30
def bytesize
-
4546
return 0 if @body.nil?
-
-
144
@body.bytesize
-
end
-
-
# sets the body to yield using chunked trannsfer encoding format.
-
30
def stream(body)
-
4007
return body unless chunked?
-
-
108
Transcoder::Chunker.encode(body.enum_for(:each))
-
end
-
-
# returns whether the body yields infinitely.
-
30
def unbounded_body?
-
4482
return @unbounded_body if defined?(@unbounded_body)
-
-
3928
@unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
-
end
-
-
# returns whether the chunked transfer encoding header is set.
-
30
def chunked?
-
24545
@headers["transfer-encoding"] == "chunked"
-
end
-
-
# sets the chunked transfer encoding header.
-
30
def chunk!
-
36
@headers.add("transfer-encoding", "chunked")
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id} " \
-
skipped
"#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
-
skipped
end
-
skipped
# :nocov:
-
-
30
class << self
-
30
def initialize_body(params)
-
12662
if (body = params.delete(:body))
-
# @type var body: bodyIO
-
1761
Transcoder::Body.encode(body)
-
10901
elsif (form = params.delete(:form))
-
1804
if Transcoder::Multipart.multipart?(form)
-
# @type var form: Transcoder::Multipart::multipart_input
-
1142
Transcoder::Multipart.encode(form)
-
else
-
# @type var form: Transcoder::urlencoded_input
-
662
Transcoder::Form.encode(form)
-
end
-
9097
elsif (json = params.delete(:json))
-
# @type var body: _ToJson
-
102
Transcoder::JSON.encode(json)
-
end
-
end
-
-
# returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
-
30
def initialize_deflater_body(body, encoding)
-
100
case encoding
-
when "gzip"
-
55
Transcoder::GZIP.encode(body)
-
when "deflate"
-
27
Transcoder::Deflate.encode(body)
-
when "identity"
-
18
body
-
else
-
9
body
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "socket"
-
30
require "resolv"
-
-
30
module HTTPX
-
30
module Resolver
-
30
extend self
-
-
30
RESOLVE_TIMEOUT = [2, 3].freeze
-
30
require "httpx/resolver/entry"
-
30
require "httpx/resolver/cache"
-
30
require "httpx/resolver/resolver"
-
30
require "httpx/resolver/system"
-
30
require "httpx/resolver/native"
-
30
require "httpx/resolver/https"
-
30
require "httpx/resolver/multi"
-
-
30
@identifier_mutex = Thread::Mutex.new
-
30
@identifier = 1
-
-
30
def supported_ip_families
-
9098
if Utils.in_ractor?
-
Ractor.store_if_absent(:httpx_supported_ip_families) { find_supported_ip_families }
-
else
-
9098
@supported_ip_families ||= find_supported_ip_families
-
end
-
end
-
-
30
def generate_id
-
1056
if Utils.in_ractor?
-
identifier = Ractor.store_if_absent(:httpx_resolver_identifier) { -1 }
-
Ractor.current[:httpx_resolver_identifier] = (identifier + 1) & 0xFFFF
-
else
-
2112
id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
-
end
-
end
-
-
30
def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
-
1002
Resolv::DNS::Message.new(message_id).tap do |query|
-
1056
query.rd = 1
-
1056
query.add_question(hostname, type)
-
107
end.encode
-
end
-
-
30
def decode_dns_answer(payload)
-
53
begin
-
818
message = Resolv::DNS::Message.decode(payload)
-
rescue Resolv::DNS::DecodeError => e
-
7
return :decode_error, e
-
end
-
-
# no domain was found
-
811
return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
-
-
367
return :message_truncated if message.tc == 1
-
-
353
if message.rcode != Resolv::DNS::RCode::NoError
-
14
case message.rcode
-
when Resolv::DNS::RCode::ServFail
-
7
return :retriable_error, message.rcode
-
else
-
7
return :dns_error, message.rcode
-
end
-
end
-
-
339
addresses = []
-
-
339
now = Utils.now
-
339
message.each_answer do |question, _, value|
-
1236
case value
-
when Resolv::DNS::Resource::IN::CNAME
-
21
addresses << {
-
"name" => question.to_s,
-
21
"TTL" => (now + value.ttl),
-
"alias" => value.name.to_s,
-
}
-
when Resolv::DNS::Resource::IN::A,
-
Resolv::DNS::Resource::IN::AAAA
-
1243
addresses << {
-
"name" => question.to_s,
-
1243
"TTL" => (now + value.ttl),
-
"data" => value.address.to_s,
-
}
-
end
-
end
-
-
339
[:ok, addresses]
-
end
-
-
30
private
-
-
30
def id_synchronize(&block)
-
1056
@identifier_mutex.synchronize(&block)
-
end
-
-
30
def find_supported_ip_families
-
36
list = Socket.ip_address_list
-
-
5
begin
-
144
if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
-
[Socket::AF_INET6, Socket::AF_INET]
-
else
-
36
[Socket::AF_INET]
-
end
-
rescue NotImplementedError
-
[Socket::AF_INET]
-
15
end.freeze
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "httpx/resolver/cache/base"
-
30
require "httpx/resolver/cache/memory"
-
-
30
module HTTPX::Resolver
-
# The internal resolvers cache adapters are defined under this namespace.
-
#
-
# Adapters must comply with the Resolver Cache Adapter API and implement the following methods:
-
#
-
# * #resolve: (String hostname) -> Array[HTTPX::Entry]? => resolves hostname to a list of cached IPs (if found in cache or system)
-
# * #get: (String hostname) -> Array[HTTPX::Entry]? => resolves hostname to a list of cached IPs (if found in cache)
-
# * #set: (String hostname, Integer ip_family, Array[dns_result]) -> void => stores the set of results in the cache indexes for
-
# the hostname and the IP family
-
# * #evict: (String hostname, _ToS ip) -> void => evicts the ip for the hostname from the cache (usually done when no longer reachable)
-
30
module Cache
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "resolv"
-
-
30
module HTTPX
-
30
module Resolver::Cache
-
# Base class of the Resolver Cache adapter implementations.
-
#
-
# While resolver caches are not required to inherit from this class, it nevertheless provides
-
# common useful functions for desired functionality, such as singleton object ractor-safe access,
-
# or a default #resolve implementation which deals with IPs and the system hosts file.
-
#
-
30
class Base
-
30
MAX_CACHE_SIZE = 512
-
30
CACHE_MUTEX = Thread::Mutex.new
-
30
HOSTS = Resolv::Hosts.new
-
30
@cache = nil
-
-
30
class << self
-
30
attr_reader :hosts_resolver
-
-
# returns the singleton instance to be used within the current ractor.
-
30
def cache(label)
-
15316
return Ractor.store_if_absent(:"httpx_resolver_cache_#{label}") { new } if Utils.in_ractor?
-
-
15316
@cache ||= CACHE_MUTEX.synchronize do
-
25
@cache || new
-
end
-
end
-
end
-
-
# resolves +hostname+ into an instance of HTTPX::Resolver::Entry if +hostname+ is an IP,
-
# or can be found in the cache, or can be found in the system hosts file.
-
30
def resolve(hostname)
-
8665
ip_resolve(hostname) || get(hostname) || hosts_resolve(hostname)
-
end
-
-
30
private
-
-
# tries to convert +hostname+ into an IPAddr, returns <tt>nil</tt> otherwise.
-
30
def ip_resolve(hostname)
-
8665
[Resolver::Entry.new(hostname)]
-
rescue ArgumentError
-
end
-
-
# matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
-
# found, or there is no hosts file.
-
30
def hosts_resolve(hostname)
-
789
ips = if Utils.in_ractor?
-
Ractor.store_if_absent(:httpx_hosts_resolver) { Resolv::Hosts.new }
-
else
-
789
HOSTS
-
end.getaddresses(hostname)
-
-
789
return if ips.empty?
-
-
1140
ips.map { |ip| Resolver::Entry.new(ip) }
-
rescue IOError
-
end
-
-
# not to be used directly!
-
30
def _get(hostname, lookups, hostnames, ttl)
-
7939
return unless lookups.key?(hostname)
-
-
7144
entries = lookups[hostname]
-
-
7144
return unless entries
-
-
7144
entries.delete_if do |address|
-
24486
address["TTL"] < ttl
-
end
-
-
7144
if entries.empty?
-
21
lookups.delete(hostname)
-
21
hostnames.delete(hostname)
-
end
-
-
7144
ips = entries.flat_map do |address|
-
24465
if (als = address["alias"])
-
9
_get(als, lookups, hostnames, ttl)
-
else
-
24456
Resolver::Entry.new(address["data"], address["TTL"])
-
end
-
end.compact
-
-
7144
ips unless ips.empty?
-
end
-
-
30
def _set(hostname, family, entries, lookups, hostnames)
-
# lru cleanup
-
6743
while lookups.size >= MAX_CACHE_SIZE
-
1692
hs = hostnames.shift
-
1692
lookups.delete(hs)
-
end
-
6555
hostnames << hostname
-
-
6555
lookups[hostname] ||= [] # when there's no default proc
-
-
5829
case family
-
when Socket::AF_INET6
-
45
lookups[hostname].concat(entries)
-
when Socket::AF_INET
-
6510
lookups[hostname].unshift(*entries)
-
end
-
6555
entries.each do |entry|
-
6640
name = entry["name"]
-
6640
next unless name != hostname
-
-
222
lookups[name] ||= []
-
-
201
case family
-
when Socket::AF_INET6
-
9
lookups[name] << entry
-
when Socket::AF_INET
-
213
lookups[name].unshift(entry)
-
end
-
end
-
end
-
-
30
def _evict(hostname, ip, lookups, hostnames)
-
16
return unless lookups.key?(hostname)
-
-
9
entries = lookups[hostname]
-
-
9
return unless entries
-
-
18
entries.delete_if { |entry| entry["data"] == ip }
-
-
9
return unless entries.empty?
-
-
9
lookups.delete(hostname)
-
9
hostnames.delete(hostname)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Resolver::Cache
-
# Implementation of a thread-safe in-memory LRU resolver cache.
-
30
class Memory < Base
-
30
def initialize
-
44
super
-
44
@hostnames = []
-
5975
@lookups = Hash.new { |h, k| h[k] = [] }
-
44
@lookup_mutex = Thread::Mutex.new
-
end
-
-
30
def get(hostname)
-
7930
now = Utils.now
-
7930
synchronize do |lookups, hostnames|
-
7930
_get(hostname, lookups, hostnames, now)
-
end
-
end
-
-
30
def set(hostname, family, entries)
-
6555
synchronize do |lookups, hostnames|
-
6555
_set(hostname, family, entries, lookups, hostnames)
-
end
-
end
-
-
30
def evict(hostname, ip)
-
16
ip = ip.to_s
-
-
16
synchronize do |lookups, hostnames|
-
16
_evict(hostname, ip, lookups, hostnames)
-
end
-
end
-
-
30
private
-
-
30
def synchronize
-
29002
@lookup_mutex.synchronize { yield(@lookups, @hostnames) }
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "ipaddr"
-
-
30
module HTTPX
-
30
module Resolver
-
30
class Entry < SimpleDelegator
-
30
attr_reader :address
-
-
30
def self.convert(address)
-
56
new(address, rescue_on_convert: true)
-
end
-
-
30
def initialize(address, expires_in = Float::INFINITY, rescue_on_convert: false)
-
35512
@expires_in = expires_in
-
35512
@address = address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
-
27657
super(@address)
-
rescue IPAddr::InvalidAddressError
-
7855
raise unless rescue_on_convert
-
-
24
@address = address.to_s
-
24
super(@address)
-
end
-
-
30
def expired?
-
3859
@expires_in < Utils.now
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "resolv"
-
30
require "uri"
-
30
require "forwardable"
-
30
require "httpx/base64"
-
-
30
module HTTPX
-
# Implementation of a DoH name resolver (https://www.youtube.com/watch?v=unMXvnY2FNM).
-
# It wraps an HTTPX::Connection object which integrates with the main session in the
-
# same manner as other performed HTTP requests.
-
#
-
30
class Resolver::HTTPS < Resolver::Resolver
-
30
extend Forwardable
-
-
30
using URIExtensions
-
-
30
module DNSExtensions
-
30
refine Resolv::DNS do
-
30
def generate_candidates(name)
-
133
@config.generate_candidates(name)
-
end
-
end
-
end
-
30
using DNSExtensions
-
-
30
NAMESERVER = "https://1.1.1.1/dns-query"
-
-
2
DEFAULTS = {
-
28
uri: NAMESERVER,
-
use_get: false,
-
}.freeze
-
-
30
def_delegators :@resolver_connection, :connecting?, :to_io, :call, :close,
-
:closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout
-
-
30
def initialize(_, options)
-
149
super
-
149
@resolver_options = DEFAULTS.merge(@options.resolver_options)
-
149
@queries = {}
-
149
@requests = {}
-
149
@_timeouts = Array(@resolver_options[:timeouts])
-
296
@timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-
149
@uri = URI(@resolver_options[:uri])
-
149
@name = @uri_addresses = nil
-
149
@resolver = Resolv::DNS.new
-
149
@resolver.timeouts = @_timeouts.empty? ? Resolver::RESOLVE_TIMEOUT : @_timeouts
-
149
@resolver.lazy_initialize
-
end
-
-
30
def state
-
28
@resolver_connection ? @resolver_connection.state : :idle
-
end
-
-
30
def <<(connection)
-
140
return if @uri.origin == connection.peer.to_s
-
-
140
@uri_addresses ||= @options.resolver_cache.resolve(@uri.host) || @resolver.getaddresses(@uri.host)
-
-
140
if @uri_addresses.empty?
-
7
ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
-
7
ex.set_backtrace(caller)
-
7
connection.force_close
-
7
throw(:resolve_error, ex)
-
end
-
-
133
resolve(connection)
-
end
-
-
30
def resolver_connection
-
# TODO: leaks connection object into the pool
-
154
@resolver_connection ||=
-
@current_session.find_connection(
-
@uri,
-
@current_selector,
-
@options.merge(resolver_class: :system, ssl: { alpn_protocols: %w[h2] })
-
).tap do |conn|
-
126
emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
-
126
conn.on(:force_closed, &method(:force_close))
-
end
-
end
-
-
30
private
-
-
30
def resolve(connection = nil, hostname = nil)
-
161
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
161
connection ||= @connections.first
-
-
161
return unless connection
-
-
161
hostname ||= @queries.key(connection)
-
-
161
if hostname.nil?
-
133
hostname = connection.peer.host
-
log do
-
"resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
-
133
end if connection.peer.non_ascii_hostname
-
-
133
hostname = @resolver.generate_candidates(hostname).each do |name|
-
399
@queries[name.to_s] = connection
-
end.first.to_s
-
else
-
28
@queries[hostname] = connection
-
end
-
-
161
@name = hostname
-
-
161
log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
-
-
161
send_request(hostname, connection)
-
end
-
-
30
def send_request(hostname, connection)
-
161
request = build_request(hostname)
-
154
request.on(:response, &method(:on_response).curry(2)[request])
-
154
request.on(:promise, &method(:on_promise))
-
154
@requests[request] = hostname
-
154
resolver_connection.send(request)
-
154
@connections << connection
-
rescue ResolveError, Resolv::DNS::EncodeError => e
-
7
reset_hostname(hostname)
-
7
throw(:resolve_error, e) if connection.pending.empty?
-
emit_resolve_error(connection, connection.peer.host, e)
-
close_or_resolve
-
end
-
-
30
def on_response(request, response)
-
112
response.raise_for_status
-
rescue StandardError => e
-
21
hostname = @requests.delete(request)
-
21
connection = reset_hostname(hostname)
-
21
emit_resolve_error(connection, connection.peer.host, e)
-
21
close_or_resolve
-
else
-
# @type var response: HTTPX::Response
-
91
if response.status.between?(300, 399) && response.headers.key?("location")
-
hostname = @requests[request]
-
connection = @queries[hostname]
-
location_uri = URI(response.headers["location"])
-
location_uri = response.uri.merge(location_uri) if location_uri.relative?
-
-
# we assume that the DNS server URI changed permanently and move on
-
@uri = location_uri
-
send_request(hostname, connection)
-
return
-
end
-
-
91
parse(request, response)
-
ensure
-
112
@requests.delete(request)
-
end
-
-
30
def on_promise(_, stream)
-
log(level: 2) { "#{stream.id}: refusing stream!" }
-
stream.refuse
-
end
-
-
30
def parse(request, response)
-
91
hostname = @name
-
-
91
@name = nil
-
-
91
code, result = decode_response_body(response)
-
-
91
case code
-
when :ok
-
35
parse_addresses(result, request)
-
when :no_domain_found
-
# Indicates no such domain was found.
-
-
42
host = @requests.delete(request)
-
42
connection = reset_hostname(host, reset_candidates: false)
-
-
42
unless @queries.value?(connection)
-
14
emit_resolve_error(connection)
-
14
close_or_resolve
-
14
return
-
end
-
-
28
resolve
-
when :retriable_error
-
timeouts = @timeouts[hostname]
-
-
unless timeouts.empty?
-
log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
-
-
connection = @queries[hostname]
-
-
resolve(connection, hostname)
-
return
-
end
-
-
host = @requests.delete(request)
-
connection = reset_hostname(host)
-
-
emit_resolve_error(connection)
-
close_or_resolve
-
when :dns_error
-
7
host = @requests.delete(request)
-
7
connection = reset_hostname(host)
-
-
7
emit_resolve_error(connection)
-
7
close_or_resolve
-
when :decode_error
-
7
host = @requests.delete(request)
-
7
connection = reset_hostname(host)
-
7
emit_resolve_error(connection, connection.peer.host, result)
-
7
close_or_resolve
-
end
-
end
-
-
30
def parse_addresses(answers, request)
-
35
if answers.empty?
-
# no address found, eliminate candidates
-
7
host = @requests.delete(request)
-
7
connection = reset_hostname(host)
-
7
emit_resolve_error(connection)
-
7
close_or_resolve
-
7
return
-
-
else
-
63
answers = answers.group_by { |answer| answer["name"] }
-
28
answers.each do |hostname, addresses|
-
35
addresses = addresses.flat_map do |address|
-
35
if address.key?("alias")
-
7
alias_address = answers[address["alias"]]
-
7
if alias_address.nil?
-
reset_hostname(address["name"])
-
if early_resolve(connection, hostname: address["alias"])
-
@connections.delete(connection)
-
else
-
resolve(connection, address["alias"])
-
return # rubocop:disable Lint/NonLocalExitFromIterator
-
end
-
else
-
7
alias_address
-
end
-
else
-
28
address
-
end
-
end.compact
-
35
next if addresses.empty?
-
-
35
hostname.delete_suffix!(".") if hostname.end_with?(".")
-
35
connection = reset_hostname(hostname, reset_candidates: false)
-
35
next unless connection # probably a retried query for which there's an answer
-
-
28
@connections.delete(connection)
-
-
# eliminate other candidates
-
84
@queries.delete_if { |_, conn| connection == conn }
-
-
28
@options.resolver_cache.set(hostname, @family, addresses) if @resolver_options[:cache]
-
84
catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) }) }
-
end
-
end
-
28
close_or_resolve(true)
-
end
-
-
30
def build_request(hostname)
-
147
uri = @uri.dup
-
147
rklass = @options.request_class
-
147
payload = Resolver.encode_dns_query(hostname, type: @record_type)
-
147
timeouts = @timeouts[hostname]
-
147
request_timeout = timeouts.shift
-
147
options = @options.merge(timeout: { request_timeout: request_timeout })
-
-
147
if @resolver_options[:use_get]
-
7
params = URI.decode_www_form(uri.query.to_s)
-
7
params << ["type", FAMILY_TYPES[@record_type]]
-
7
params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
-
7
uri.query = URI.encode_www_form(params)
-
7
request = rklass.new("GET", uri, options)
-
else
-
140
request = rklass.new("POST", uri, options, body: [payload])
-
140
request.headers["content-type"] = "application/dns-message"
-
end
-
147
request.headers["accept"] = "application/dns-message"
-
147
request
-
end
-
-
30
def decode_response_body(response)
-
77
case response.headers["content-type"]
-
when "application/dns-udpwireformat",
-
"application/dns-message"
-
77
Resolver.decode_dns_answer(response.to_s)
-
else
-
raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
-
end
-
end
-
-
30
def reset_hostname(hostname, reset_candidates: true)
-
126
@timeouts.delete(hostname)
-
126
connection = @queries.delete(hostname)
-
-
126
return connection unless connection && reset_candidates
-
-
# eliminate other candidates
-
147
candidates = @queries.select { |_, conn| connection == conn }.keys
-
147
@queries.delete_if { |h, _| candidates.include?(h) }
-
# reset timeouts
-
49
@timeouts.delete_if { |h, _| candidates.include?(h) }
-
-
49
connection
-
end
-
-
30
def close_or_resolve(should_deactivate = false)
-
# drop already closed connections
-
84
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
84
if (@connections - @queries.values).empty?
-
# the same resolver connection may be serving different https resolvers (AAAA and A).
-
84
return if inflight?
-
-
70
if should_deactivate
-
25
deactivate
-
else
-
45
disconnect
-
end
-
else
-
resolve
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "forwardable"
-
30
require "resolv"
-
-
30
module HTTPX
-
30
class Resolver::Multi
-
30
attr_reader :resolvers, :options
-
-
30
def initialize(resolver_type, options)
-
8570
@current_selector = @current_session = nil
-
8570
@options = options
-
8570
@resolver_options = @options.resolver_options
-
-
8570
ip_families = options.ip_families || Resolver.supported_ip_families
-
-
8570
@resolvers = ip_families.map do |ip_family|
-
8612
resolver = resolver_type.new(ip_family, options)
-
8612
resolver.multi = self
-
8612
resolver
-
end
-
end
-
-
30
def state
-
86
@resolvers.map(&:state).uniq.join(",")
-
end
-
-
30
def current_selector=(s)
-
8836
@current_selector = s
-
17714
@resolvers.each { |r| r.current_selector = s }
-
end
-
-
30
def current_session=(s)
-
8836
@current_session = s
-
17714
@resolvers.each { |r| r.current_session = s }
-
end
-
-
30
def log(*args, **kwargs, &blk)
-
34910
@resolvers.each { |r| r.log(*args, **kwargs, &blk) }
-
end
-
-
30
def closed?
-
8723
@resolvers.all?(&:closed?)
-
end
-
-
30
def early_resolve(connection)
-
8843
hostname = connection.peer.host
-
8843
addresses = @resolver_options[:cache] && (connection.addresses || nolookup_resolve(hostname, connection.options))
-
8843
return false unless addresses
-
-
8215
ip_families = connection.options.ip_families
-
-
8215
resolved = false
-
8595
addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
-
8576
next unless ip_families.nil? || ip_families.include?(family)
-
-
# try to match the resolver by family. However, there are cases where that's not possible, as when
-
# the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
-
17152
resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
-
-
8576
next unless resolver # this should ever happen
-
-
# it does not matter which resolver it is, as early-resolve code is shared.
-
8576
resolver.emit_addresses(connection, family, addrs, true)
-
-
8540
resolved = true
-
end
-
-
8179
resolved
-
end
-
-
30
def lazy_resolve(connection)
-
629
@resolvers.each do |resolver|
-
671
resolver.lazy_resolve(connection)
-
end
-
end
-
-
30
private
-
-
30
def nolookup_resolve(hostname, options)
-
8525
options.resolver_cache.resolve(hostname)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "forwardable"
-
30
require "resolv"
-
-
30
module HTTPX
-
# Implements a pure ruby name resolver, which abides by the Selectable API.
-
# It delegates DNS payload encoding/decoding to the +resolv+ stlid gem.
-
#
-
30
class Resolver::Native < Resolver::Resolver
-
30
extend Forwardable
-
-
30
using URIExtensions
-
-
20
DEFAULTS = {
-
10
nameserver: nil,
-
**Resolv::DNS::Config.default_config_hash,
-
packet_size: 512,
-
timeouts: Resolver::RESOLVE_TIMEOUT,
-
}.freeze
-
-
30
DNS_PORT = 53
-
-
30
def_delegator :@connections, :empty?
-
-
30
attr_reader :state
-
-
30
def initialize(family, options)
-
8463
super
-
8463
@ns_index = 0
-
8463
@resolver_options = DEFAULTS.merge(@options.resolver_options)
-
8463
@socket_type = @resolver_options.fetch(:socket_type, :udp)
-
8463
@nameserver = if (nameserver = @resolver_options[:nameserver])
-
8456
nameserver = nameserver[family] if nameserver.is_a?(Hash)
-
8456
Array(nameserver)
-
end
-
8463
@ndots = @resolver_options.fetch(:ndots, 1)
-
25382
@search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
-
8463
@_timeouts = Array(@resolver_options[:timeouts])
-
9243
@timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-
8463
@name = nil
-
8463
@queries = {}
-
8463
@read_buffer = "".b
-
8463
@write_buffer = Buffer.new(@resolver_options[:packet_size])
-
8463
@state = :idle
-
8463
@timer = nil
-
end
-
-
30
def close
-
562
transition(:closed)
-
end
-
-
30
def force_close(*)
-
42
@timer.cancel if @timer
-
42
@timer = @name = nil
-
42
@queries.clear
-
42
@timeouts.clear
-
42
close
-
42
super
-
ensure
-
42
terminate
-
end
-
-
30
def terminate
-
62
disconnect
-
end
-
-
30
def closed?
-
9277
@state == :idle || @state == :closed
-
end
-
-
30
def to_io
-
1444
@io.to_io
-
end
-
-
30
def call
-
1162
case @state
-
when :open
-
1208
consume
-
end
-
end
-
-
30
def interests
-
1375
case @state
-
when :idle
-
545
transition(:open)
-
when :closed
-
17
transition(:idle)
-
17
transition(:open)
-
end
-
-
1444
calculate_interests
-
end
-
-
30
def <<(connection)
-
531
if @nameserver.nil?
-
7
ex = ResolveError.new("No available nameserver")
-
7
ex.set_backtrace(caller)
-
7
connection.force_close
-
7
throw(:resolve_error, ex)
-
else
-
524
@connections << connection
-
524
resolve
-
end
-
end
-
-
30
def timeout
-
1444
return unless @name
-
-
1416
@start_timeout = Utils.now
-
-
1416
timeouts = @timeouts[@name]
-
-
1416
return if timeouts.empty?
-
-
1305
log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
-
-
1305
timeouts.first
-
end
-
-
30
def handle_socket_timeout(interval); end
-
-
30
def handle_error(error)
-
42
if error.respond_to?(:connection) &&
-
error.respond_to?(:host)
-
21
reset_hostname(error.host, connection: error.connection)
-
else
-
21
@queries.each do |host, connection|
-
21
reset_hostname(host, connection: connection)
-
end
-
end
-
-
42
super
-
end
-
-
30
private
-
-
30
def calculate_interests
-
6327
if @queries.empty?
-
397
return @io.interests if (@socket_type == :tcp) && (@state == :idle)
-
-
355
return
-
end
-
-
5930
return :r if @write_buffer.empty?
-
-
2110
:w
-
end
-
-
30
def consume
-
1222
loop do
-
2068
dread if calculate_interests == :r
-
-
2029
break unless calculate_interests == :w
-
-
867
dwrite
-
-
846
break unless calculate_interests == :r
-
end
-
rescue Errno::EHOSTUNREACH => e
-
21
@ns_index += 1
-
21
nameserver = @nameserver
-
21
if nameserver && @ns_index < nameserver.size
-
14
log { "resolver #{FAMILY_TYPES[@record_type]}: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
-
14
transition(:idle)
-
14
@timeouts.clear
-
14
retry
-
else
-
7
handle_error(e)
-
7
disconnect
-
end
-
rescue NativeResolveError => e
-
21
handle_error(e)
-
21
close_or_resolve
-
21
retry unless closed?
-
end
-
-
30
def schedule_retry
-
846
h = @name
-
-
846
return unless h
-
-
846
connection = @queries[h]
-
-
846
timeouts = @timeouts[h]
-
846
timeout = timeouts.shift
-
-
846
@timer = @current_selector.after(timeout) do
-
110
next unless @connections.include?(connection)
-
-
110
@timer = @name = nil
-
-
110
do_retry(h, connection, timeout)
-
end
-
end
-
-
30
def do_retry(h, connection, interval)
-
110
timeouts = @timeouts[h]
-
-
110
if !timeouts.empty?
-
75
log { "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..." }
-
# must downgrade to tcp AND retry on same host as last
-
75
downgrade_socket
-
75
resolve(connection, h)
-
35
elsif @ns_index + 1 < @nameserver.size
-
# try on the next nameserver
-
7
@ns_index += 1
-
7
log do
-
"resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
-
end
-
7
transition(:idle)
-
7
@timeouts.clear
-
7
resolve(connection, h)
-
else
-
28
reset_hostname(h, reset_candidates: false)
-
-
28
unless @queries.empty?
-
21
resolve(connection)
-
21
return
-
end
-
-
7
@connections.delete(connection)
-
-
7
host = connection.peer.host
-
-
# This loop_time passed to the exception is bogus. Ideally we would pass the total
-
# resolve timeout, including from the previous retries.
-
7
ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
-
7
ex.set_backtrace(ex ? ex.backtrace : caller)
-
7
emit_resolve_error(connection, host, ex)
-
-
7
close_or_resolve
-
end
-
end
-
-
30
def dread(wsize = @resolver_options[:packet_size])
-
1471
loop do
-
1492
wsize = @large_packet.capacity if @large_packet
-
-
1492
siz = @io.read(wsize, @read_buffer)
-
-
1492
unless siz
-
ex = EOFError.new("descriptor closed")
-
ex.set_backtrace(caller)
-
raise ex
-
end
-
-
1492
return unless siz.positive?
-
-
755
if @socket_type == :tcp
-
# packet may be incomplete, need to keep draining from the socket
-
35
if @large_packet
-
# large packet buffer already exists, continue pumping
-
14
@large_packet << @read_buffer
-
-
14
next unless @large_packet.full?
-
-
14
parse(@large_packet.to_s)
-
14
@large_packet = nil
-
# downgrade to udp again
-
14
downgrade_socket
-
14
return
-
else
-
21
size = @read_buffer[0, 2].unpack1("n")
-
21
buffer = @read_buffer.byteslice(2..-1)
-
-
21
if size > @read_buffer.bytesize
-
# only do buffer logic if it's worth it, and the whole packet isn't here already
-
14
@large_packet = Buffer.new(size)
-
14
@large_packet << buffer
-
-
14
next
-
else
-
7
parse(buffer)
-
end
-
end
-
else # udp
-
720
parse(@read_buffer)
-
end
-
-
688
return if @state == :closed || !@write_buffer.empty?
-
end
-
end
-
-
30
def dwrite
-
846
loop do
-
1692
return if @write_buffer.empty?
-
-
846
siz = @io.write(@write_buffer)
-
-
846
unless siz
-
ex = EOFError.new("descriptor closed")
-
ex.set_backtrace(caller)
-
raise ex
-
end
-
-
846
return unless siz.positive?
-
-
846
schedule_retry if @write_buffer.empty?
-
-
846
return if @state == :closed
-
end
-
end
-
-
30
def parse(buffer)
-
741
code, result = Resolver.decode_dns_answer(buffer)
-
-
687
case code
-
when :ok
-
304
reset_query
-
304
parse_addresses(result)
-
when :no_domain_found
-
402
reset_query
-
# Indicates no such domain was found.
-
402
hostname, connection = @queries.first
-
402
reset_hostname(hostname, reset_candidates: false)
-
-
670
other_candidate, _ = @queries.find { |_, conn| conn == connection }
-
-
402
if other_candidate
-
268
resolve(connection, other_candidate)
-
else
-
134
@connections.delete(connection)
-
134
ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
-
134
ex.set_backtrace(ex ? ex.backtrace : caller)
-
134
emit_resolve_error(connection, connection.peer.host, ex)
-
116
close_or_resolve
-
end
-
when :message_truncated
-
14
reset_query
-
# TODO: what to do if it's already tcp??
-
14
return if @socket_type == :tcp
-
-
14
@socket_type = :tcp
-
-
14
hostname, _ = @queries.first
-
14
reset_hostname(hostname)
-
14
transition(:closed)
-
when :retriable_error
-
7
if @name && @timer
-
7
log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
-
7
return
-
end
-
# retry now!
-
# connection = @queries[@name].shift
-
# @timer.fire
-
reset_query
-
hostname, connection = @queries.first
-
reset_hostname(hostname)
-
@connections.delete(connection)
-
ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
-
raise ex
-
when :dns_error
-
7
reset_query
-
7
hostname, connection = @queries.first
-
7
reset_hostname(hostname)
-
7
@connections.delete(connection)
-
7
ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
-
7
raise ex
-
when :decode_error
-
7
reset_query
-
7
hostname, connection = @queries.first
-
7
reset_hostname(hostname)
-
7
@connections.delete(connection)
-
7
ex = NativeResolveError.new(connection, connection.peer.host, result.message)
-
7
ex.set_backtrace(result.backtrace)
-
7
raise ex
-
end
-
end
-
-
30
def parse_addresses(addresses)
-
304
if addresses.empty?
-
# no address found, eliminate candidates
-
7
hostname, connection = @queries.first
-
7
reset_hostname(hostname)
-
7
@connections.delete(connection)
-
7
raise NativeResolveError.new(connection, connection.peer.host)
-
else
-
297
address = addresses.first
-
297
name = address["name"]
-
-
297
connection = @queries.delete(name)
-
-
297
unless connection
-
283
orig_name = name
-
# absolute name
-
283
name_labels = Resolv::DNS::Name.create(name).to_a
-
283
name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
-
-
# probably a retried query for which there's an answer
-
283
unless name
-
@timeouts.delete(orig_name)
-
return
-
end
-
-
262
address["name"] = name
-
283
connection = @queries.delete(name)
-
end
-
-
1526
alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
-
-
297
if addresses.empty? && !alias_addresses.empty? # CNAME
-
hostname_alias = alias_addresses.first["alias"]
-
# clean up intermediate queries
-
@timeouts.delete(name) unless connection.peer.host == name
-
-
if early_resolve(connection, hostname: hostname_alias)
-
@connections.delete(connection)
-
else
-
if @socket_type == :tcp
-
# must downgrade to udp if tcp
-
@socket_type = @resolver_options.fetch(:socket_type, :udp)
-
transition(:idle)
-
transition(:open)
-
end
-
log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
-
resolve(connection, hostname_alias)
-
return
-
end
-
else
-
297
reset_hostname(name, connection: connection)
-
297
@timeouts.delete(connection.peer.host)
-
297
@connections.delete(connection)
-
297
@options.resolver_cache.set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
-
297
catch(:coalesced) do
-
1505
emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) })
-
end
-
end
-
end
-
297
close_or_resolve
-
end
-
-
30
def resolve(connection = nil, hostname = nil)
-
913
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
1459
connection ||= @connections.find { |c| !@queries.value?(c) }
-
-
913
raise Error, "no URI to resolve" unless connection
-
-
# do not buffer query if previous is still in the buffer or awaiting reply/retry
-
913
return unless @write_buffer.empty? && @timer.nil?
-
-
909
hostname ||= @queries.key(connection)
-
-
909
if hostname.nil?
-
538
hostname = connection.peer.host
-
538
if connection.peer.non_ascii_hostname
-
log { "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" }
-
end
-
-
538
hostname = generate_candidates(hostname).each do |name|
-
1511
@queries[name] = connection
-
end.first
-
else
-
347
@queries[hostname] = connection
-
end
-
-
909
@name = hostname
-
-
909
log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
-
53
begin
-
909
@write_buffer << encode_dns_query(hostname)
-
rescue Resolv::DNS::EncodeError => e
-
reset_hostname(hostname, connection: connection)
-
@connections.delete(connection)
-
emit_resolve_error(connection, hostname, e)
-
close_or_resolve
-
end
-
end
-
-
30
def encode_dns_query(hostname)
-
909
message_id = Resolver.generate_id
-
909
msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
-
909
msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
-
909
msg
-
end
-
-
30
def generate_candidates(name)
-
538
return [name] if name.end_with?(".")
-
-
538
name_parts = name.scan(/[^.]+/)
-
1607
candidates = @search.map { |domain| [*name_parts, *domain].join(".") }
-
538
fname = "#{name}."
-
538
if @ndots <= name_parts.size - 1
-
538
candidates.unshift(fname)
-
else
-
candidates << fname
-
end
-
538
candidates
-
end
-
-
30
def build_socket
-
555
ip, port = @nameserver[@ns_index]
-
555
port ||= DNS_PORT
-
-
523
case @socket_type
-
when :udp
-
534
log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
-
534
UDP.new(ip, port, @options)
-
when :tcp
-
21
log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
-
21
origin = URI("tcp://#{ip}:#{port}")
-
21
TCP.new(origin, [Resolver::Entry.new(ip)], @options)
-
end
-
end
-
-
30
def downgrade_socket
-
89
return unless @socket_type == :tcp
-
-
14
@socket_type = @resolver_options.fetch(:socket_type, :udp)
-
14
transition(:idle)
-
14
transition(:open)
-
end
-
-
30
def transition(nextstate)
-
1140
case nextstate
-
when :idle
-
52
if @io
-
45
@io.close
-
45
@io = nil
-
end
-
when :open
-
576
return unless @state == :idle
-
-
576
@io ||= build_socket
-
-
576
@io.connect
-
576
return unless @io.connected?
-
-
555
resolve if @queries.empty? && !@connections.empty?
-
when :closed
-
576
return if @state == :closed
-
-
534
@io.close if @io
-
534
@start_timeout = nil
-
534
@write_buffer.clear
-
534
@read_buffer.clear
-
end
-
1141
log(level: 3) { "#{@state} -> #{nextstate}" }
-
1141
@state = nextstate
-
rescue Errno::ECONNREFUSED,
-
Errno::EADDRNOTAVAIL,
-
Errno::EHOSTUNREACH,
-
SocketError,
-
IOError,
-
ConnectTimeoutError => e
-
# these errors may happen during TCP handshake
-
# treat them as resolve errors.
-
on_error(e)
-
end
-
-
30
def reset_query
-
734
@timer.cancel
-
-
734
@timer = @name = nil
-
end
-
-
30
def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
-
804
@timeouts.delete(hostname)
-
-
804
return unless connection && reset_candidates
-
-
# eliminate other candidates
-
1087
candidates = @queries.select { |_, conn| connection == conn }.keys
-
1087
@queries.delete_if { |h, _| candidates.include?(h) }
-
# reset timeouts
-
374
@timeouts.delete_if { |h, _| candidates.include?(h) }
-
end
-
-
30
def close_or_resolve
-
# drop already closed connections
-
441
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
441
if (@connections - @queries.values).empty?
-
437
disconnect
-
else
-
4
resolve
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "resolv"
-
-
30
module HTTPX
-
# Base class for all internal internet name resolvers. It handles basic blocks
-
# from the Selectable API.
-
#
-
30
class Resolver::Resolver
-
30
include Loggable
-
-
30
using ArrayExtensions::Intersect
-
-
2
RECORD_TYPES = {
-
28
Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
-
Socket::AF_INET => Resolv::DNS::Resource::IN::A,
-
}.freeze
-
-
2
FAMILY_TYPES = {
-
28
Resolv::DNS::Resource::IN::AAAA => "AAAA",
-
Resolv::DNS::Resource::IN::A => "A",
-
}.freeze
-
-
30
class << self
-
30
def multi?
-
8570
true
-
end
-
end
-
-
30
attr_reader :family, :options
-
-
30
attr_writer :current_selector, :current_session
-
-
30
attr_accessor :multi
-
-
30
def initialize(family, options)
-
8790
@family = family
-
8790
@record_type = RECORD_TYPES[family]
-
8790
@options = options
-
8790
@connections = []
-
end
-
-
30
def each_connection(&block)
-
512
enum_for(__method__) unless block
-
-
512
return unless @connections
-
-
512
@connections.each(&block)
-
end
-
-
30
def close; end
-
-
30
alias_method :terminate, :close
-
-
30
def force_close(*args)
-
504
while (connection = @connections.shift)
-
140
connection.force_close(*args)
-
end
-
end
-
-
30
def closed?
-
true
-
end
-
-
30
def empty?
-
126
true
-
end
-
-
30
def inflight?
-
78
false
-
end
-
-
30
def emit_addresses(connection, family, addresses, early_resolve = false)
-
36386
addresses.map! { |address| address.is_a?(Resolver::Entry) ? address : Resolver::Entry.new(address) }
-
-
# double emission check, but allow early resolution to work
-
9089
conn_addrs = connection.addresses
-
9089
return if !early_resolve && conn_addrs && !conn_addrs.empty? && !addresses.intersect?(conn_addrs)
-
-
9089
log do
-
100
"resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
-
8
"answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
-
end
-
-
# do not apply resolution delay for non-dns name resolution
-
9089
if !early_resolve &&
-
# just in case...
-
@current_selector &&
-
# resolution delay only applies to IPv4
-
family == Socket::AF_INET &&
-
# connection already has addresses and initiated/ended handshake
-
!connection.io &&
-
# no need to delay if not supporting dual stack / multi-homed IP
-
410
(connection.options.ip_families || Resolver.supported_ip_families).size > 1 &&
-
# connection URL host is already the IP (early resolve included perhaps?)
-
addresses.first.to_s != connection.peer.host.to_s
-
14
log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
-
-
14
@current_selector.after(0.05) do
-
# double emission check
-
11
unless connection.addresses && addresses.intersect?(connection.addresses)
-
11
emit_resolved_connection(connection, addresses, early_resolve)
-
end
-
end
-
else
-
9075
emit_resolved_connection(connection, addresses, early_resolve)
-
end
-
end
-
-
30
def handle_error(error)
-
42
if error.respond_to?(:connection) &&
-
error.respond_to?(:host)
-
21
@connections.delete(error.connection)
-
21
emit_resolve_error(error.connection, error.host, error)
-
else
-
63
while (connection = @connections.shift)
-
21
emit_resolve_error(connection, connection.peer.host, error)
-
end
-
end
-
end
-
-
30
def on_io_error(e)
-
on_error(e)
-
force_close(true)
-
end
-
-
30
def on_error(error)
-
14
handle_error(error)
-
14
disconnect
-
end
-
-
30
def early_resolve(connection, hostname: connection.peer.host) # rubocop:disable Naming/PredicateMethod
-
addresses = @resolver_options[:cache] && (connection.addresses || @options.resolver_cache.resolve(hostname))
-
-
return false unless addresses
-
-
addresses = addresses.select { |addr| addr.family == @family }
-
-
return false if addresses.empty?
-
-
emit_addresses(connection, @family, addresses, true)
-
-
true
-
end
-
-
30
def lazy_resolve(connection)
-
671
return unless @current_session && @current_selector
-
-
671
conn_to_resolve = @current_session.try_clone_connection(connection, @current_selector, @family)
-
671
self << conn_to_resolve
-
-
650
return if empty?
-
-
# both the resolver and the connection it's resolving must be pinned to the session
-
524
@current_session.pin(conn_to_resolve, @current_selector)
-
524
@current_session.select_resolver(self, @current_selector)
-
end
-
-
30
private
-
-
30
def emit_resolved_connection(connection, addresses, early_resolve)
-
begin
-
9086
connection.addresses = addresses
-
-
9034
return if connection.state == :closed
-
-
9029
resolve_connection(connection)
-
24
rescue StandardError => e
-
52
if early_resolve
-
36
connection.force_close
-
36
throw(:resolve_error, e)
-
else
-
16
emit_connection_error(connection, e)
-
end
-
end
-
end
-
-
30
def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
-
268
emit_connection_error(connection, resolve_error(hostname, ex))
-
end
-
-
30
def resolve_error(hostname, ex = nil)
-
268
return ex if ex.is_a?(ResolveError) || ex.is_a?(ResolveTimeoutError)
-
-
84
message = ex ? ex.message : "Can't resolve #{hostname}"
-
84
error = ResolveError.new(message)
-
84
error.set_backtrace(ex ? ex.backtrace : caller)
-
84
error
-
end
-
-
30
def resolve_connection(connection)
-
9029
@current_session.__send__(:on_resolver_connection, connection, @current_selector)
-
end
-
-
30
def emit_connection_error(connection, error)
-
284
return connection.handle_connect_error(error) if connection.connecting?
-
-
7
connection.on_error(error)
-
end
-
-
30
def disconnect
-
678
close
-
-
678
return unless closed?
-
-
678
@current_session.deselect_resolver(self, @current_selector)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "resolv"
-
-
30
module HTTPX
-
# Implementation of a synchronous name resolver which relies on the system resolver,
-
# which is lib'c getaddrinfo function (abstracted in ruby via Addrinfo.getaddrinfo).
-
#
-
# Its main advantage is relying on the reference implementation for name resolution
-
# across most/all OSs which deploy ruby (it's what TCPSocket also uses), its main
-
# disadvantage is the inability to set timeouts / check socket for readiness events,
-
# hence why it relies on using the Timeout module, which poses a lot of problems for
-
# the selector loop, specially when network is unstable.
-
#
-
30
class Resolver::System < Resolver::Resolver
-
30
using URIExtensions
-
-
30
RESOLV_ERRORS = [Resolv::ResolvError,
-
Resolv::DNS::Requester::RequestError,
-
Resolv::DNS::EncodeError,
-
Resolv::DNS::DecodeError].freeze
-
-
30
DONE = 1
-
30
ERROR = 2
-
-
30
class << self
-
30
def multi?
-
178
false
-
end
-
end
-
-
30
attr_reader :state
-
-
30
def initialize(options)
-
178
super(0, options)
-
178
@resolver_options = @options.resolver_options
-
178
resolv_options = @resolver_options.dup
-
178
timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
-
178
@_timeouts = Array(timeouts)
-
347
@timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
-
178
resolv_options.delete(:cache)
-
178
@queries = []
-
178
@ips = []
-
178
@pipe_mutex = Thread::Mutex.new
-
178
@state = :idle
-
end
-
-
30
def resolvers
-
56
return enum_for(__method__) unless block_given?
-
-
28
yield self
-
end
-
-
30
def multi
-
111
self
-
end
-
-
30
def empty?
-
169
@connections.empty?
-
end
-
-
30
def close
-
169
transition(:closed)
-
end
-
-
30
def force_close(*)
-
56
close
-
56
@queries.clear
-
56
@timeouts.clear
-
56
@ips.clear
-
56
super
-
end
-
-
30
def closed?
-
226
@state == :closed
-
end
-
-
30
def to_io
-
416
@pipe_read.to_io
-
end
-
-
30
def call
-
98
case @state
-
when :open
-
98
consume
-
end
-
29
nil
-
end
-
-
30
def interests
-
287
return if @queries.empty?
-
-
287
:r
-
end
-
-
30
def timeout
-
287
_, connection = @queries.first
-
-
287
return unless connection
-
-
287
timeouts = @timeouts[connection.peer.host]
-
-
287
return if timeouts.empty?
-
-
287
log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
-
-
287
timeouts.first
-
end
-
-
30
def lazy_resolve(connection)
-
169
@connections << connection
-
169
resolve
-
-
169
return if empty?
-
-
168
@current_session.select_resolver(self, @current_selector)
-
end
-
-
30
def early_resolve(_, **) # rubocop:disable Naming/PredicateMethod
-
169
false
-
end
-
-
30
def handle_socket_timeout(interval)
-
14
error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
-
14
error.set_backtrace(caller)
-
14
@queries.each do |_, connection| # rubocop:disable Style/HashEachMethods
-
14
emit_resolve_error(connection, connection.peer.host, error) if @connections.delete(connection)
-
end
-
-
28
while (connection = @connections.shift)
-
emit_resolve_error(connection, connection.peer.host, error)
-
end
-
-
14
close_or_resolve
-
end
-
-
30
private
-
-
30
def transition(nextstate)
-
338
case nextstate
-
when :idle
-
@timeouts.clear
-
when :open
-
169
return unless @state == :idle
-
-
169
@pipe_read, @pipe_write = IO.pipe
-
when :closed
-
169
return unless @state == :open
-
-
169
@pipe_write.close
-
169
@pipe_read.close
-
end
-
338
@state = nextstate
-
end
-
-
30
def consume
-
267
return if @connections.empty?
-
-
267
event = @pipe_read.read_nonblock(1, exception: false)
-
-
267
return if event == :wait_readable
-
-
99
raise ResolveError, "socket pipe closed unexpectedly" if event.nil?
-
-
99
case event.unpack1("C")
-
when DONE
-
168
*pair, addrs = @pipe_mutex.synchronize { @ips.pop }
-
84
if pair
-
84
@queries.delete(pair)
-
84
family, connection = pair
-
84
@connections.delete(connection)
-
-
168
catch(:coalesced) { emit_addresses(connection, family, addrs) }
-
end
-
when ERROR
-
30
*pair, error = @pipe_mutex.synchronize { @ips.pop }
-
15
if pair && error
-
15
@queries.delete(pair)
-
15
_, connection = pair
-
15
@connections.delete(connection)
-
-
15
emit_resolve_error(connection, connection.peer.host, error)
-
end
-
end
-
-
99
return disconnect if @connections.empty?
-
-
resolve
-
rescue StandardError => e
-
on_error(e)
-
end
-
-
30
def resolve(connection = nil, hostname = nil)
-
169
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
169
connection ||= @connections.first
-
-
169
raise Error, "no URI to resolve" unless connection
-
-
169
return unless @queries.empty?
-
-
169
hostname ||= connection.peer.host
-
169
scheme = connection.origin.scheme
-
log do
-
"resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
-
169
end if connection.peer.non_ascii_hostname
-
-
169
transition(:open)
-
-
169
ip_families = connection.options.ip_families || Resolver.supported_ip_families
-
-
169
ip_families.each do |family|
-
204
@queries << [family, connection]
-
end
-
169
async_resolve(connection, hostname, scheme)
-
169
consume
-
end
-
-
30
def async_resolve(connection, hostname, scheme)
-
169
families = connection.options.ip_families || Resolver.supported_ip_families
-
169
log { "resolver: query for #{hostname}" }
-
169
timeouts = @timeouts[connection.peer.host]
-
169
resolve_timeout = timeouts.first
-
-
169
Thread.start do
-
169
Thread.current.report_on_exception = false
-
begin
-
169
addrs = if resolve_timeout
-
-
169
Timeout.timeout(resolve_timeout) do
-
169
__addrinfo_resolve(hostname, scheme)
-
end
-
else
-
__addrinfo_resolve(hostname, scheme)
-
end
-
154
addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
-
154
families.each do |family|
-
189
addresses = addrs[family]
-
189
next unless addresses
-
-
154
addresses.map!(&:ip_address)
-
154
addresses.uniq!
-
154
@pipe_mutex.synchronize do
-
154
@ips.unshift([family, connection, addresses])
-
154
@pipe_write.putc(DONE) unless @pipe_write.closed?
-
end
-
end
-
rescue StandardError => e
-
15
if e.is_a?(Timeout::Error)
-
1
timeouts.shift
-
1
retry unless timeouts.empty?
-
1
e = ResolveTimeoutError.new(resolve_timeout, e.message)
-
1
e.set_backtrace(e.backtrace)
-
end
-
15
@pipe_mutex.synchronize do
-
15
families.each do |family|
-
15
@ips.unshift([family, connection, e])
-
15
@pipe_write.putc(ERROR) unless @pipe_write.closed?
-
end
-
end
-
end
-
end
-
169
Thread.pass
-
end
-
-
30
def close_or_resolve
-
# drop already closed connections
-
14
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
14
if (@connections - @queries.map(&:last)).empty?
-
14
disconnect
-
else
-
resolve
-
end
-
end
-
-
30
def __addrinfo_resolve(host, scheme)
-
169
Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "objspace"
-
30
require "stringio"
-
30
require "tempfile"
-
30
require "fileutils"
-
30
require "forwardable"
-
-
30
module HTTPX
-
# Defines a HTTP response is handled internally, with a few properties exposed as attributes.
-
#
-
# It delegates the following methods to the corresponding HTTPX::Request:
-
#
-
# * HTTPX::Request#uri
-
# * HTTPX::Request#peer_address
-
#
-
# It implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads.
-
#
-
# It implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
-
# (of payload chunks).
-
#
-
30
class Response
-
30
extend Forwardable
-
30
include Callbacks
-
-
# the HTTP response status code
-
30
attr_reader :status
-
-
# an HTTPX::Headers object containing the response HTTP headers.
-
30
attr_reader :headers
-
-
# a HTTPX::Response::Body object wrapping the response body. The following methods are delegated to it:
-
#
-
# * HTTPX::Response::Body#to_s
-
# * HTTPX::Response::Body#to_str
-
# * HTTPX::Response::Body#read
-
# * HTTPX::Response::Body#copy_to
-
# * HTTPX::Response::Body#close
-
30
attr_reader :body
-
-
# The HTTP protocol version used to fetch the response.
-
30
attr_reader :version
-
-
# returns the response body buffered in a string.
-
30
def_delegator :@body, :to_s
-
-
30
def_delegator :@body, :to_str
-
-
# implements the IO reader +#read+ interface.
-
30
def_delegator :@body, :read
-
-
# copies the response body to a different location.
-
30
def_delegator :@body, :copy_to
-
-
# the corresponding request uri.
-
30
def_delegator :@request, :uri
-
-
# the IP address of the peer server.
-
30
def_delegator :@request, :peer_address
-
-
# inits the instance with the corresponding +request+ to this response, an the
-
# response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
-
30
def initialize(request, status, version, headers)
-
11509
@request = request
-
11509
@options = request.options
-
11509
@version = version
-
11509
@status = Integer(status)
-
11509
@headers = @options.headers_class.new(headers)
-
11509
@body = @options.response_body_class.new(self, @options)
-
11509
@finished = complete?
-
11509
@content_type = nil
-
end
-
-
# dupped initialization
-
30
def initialize_dup(orig)
-
72
super
-
# if a response gets dupped, the body handle must also get dupped to prevent
-
# two responses from using the same file handle to read.
-
72
@body = orig.body.dup
-
end
-
-
# closes the respective +@request+ and +@body+.
-
30
def close
-
522
@request.close
-
522
@body.close
-
end
-
-
# merges headers defined in +h+ into the response headers.
-
30
def merge_headers(h)
-
223
@headers = @headers.merge(h)
-
end
-
-
# writes +data+ chunk into the response body.
-
30
def <<(data)
-
15553
@body.write(data)
-
end
-
-
# returns the HTTPX::ContentType for the response, as per what's declared in the content-type header.
-
#
-
# response.content_type #=> #<HTTPX::ContentType:xxx @header_value="text/plain">
-
# response.content_type.mime_type #=> "text/plain"
-
30
def content_type
-
12004
@content_type ||= ContentType.new(@headers["content-type"])
-
end
-
-
# returns whether the response has been fully fetched.
-
30
def finished?
-
15753
@finished
-
end
-
-
# marks the response as finished, freezes the headers.
-
30
def finish!
-
10239
@finished = true
-
10239
@headers.freeze
-
10239
@request.connection = nil
-
end
-
-
# returns whether the response contains body payload.
-
30
def bodyless?
-
11509
@request.verb == "HEAD" ||
-
@status < 200 || # informational response
-
@status == 204 ||
-
@status == 205 ||
-
@status == 304 || begin
-
10991
content_length = @headers["content-length"]
-
10991
return false if content_length.nil?
-
-
9280
content_length == "0"
-
end
-
end
-
-
30
def complete?
-
11509
bodyless? || (@request.verb == "CONNECT" && @status == 200)
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id} " \
-
skipped
"HTTP/#{version} " \
-
skipped
"@status=#{@status} " \
-
skipped
"@headers=#{@headers} " \
-
skipped
"@body=#{@body.bytesize}>"
-
skipped
end
-
skipped
# :nocov:
-
-
# returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
-
# status code, or nothing.
-
#
-
# ok_response.error #=> nil
-
# not_found_response.error #=> HTTPX::HTTPError instance, status 404
-
30
def error
-
839
return if @status < 400
-
-
70
HTTPError.new(self)
-
end
-
-
# it raises the exception returned by +error+, or itself otherwise.
-
#
-
# ok_response.raise_for_status #=> ok_response
-
# not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
-
30
def raise_for_status
-
754
return self unless (err = error)
-
-
50
raise err
-
end
-
-
# decodes the response payload into a ruby object **if** the payload is valid json.
-
#
-
# response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
-
# response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
-
30
def json(*args)
-
216
decode(Transcoder::JSON, *args)
-
end
-
-
# decodes the response payload into a ruby object **if** the payload is valid
-
# "application/x-www-urlencoded" or "multipart/form-data".
-
30
def form
-
72
decode(Transcoder::Form)
-
end
-
-
30
def xml
-
# TODO: remove at next major version.
-
9
warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
-
1
"Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
-
9
require "httpx/plugins/xml"
-
9
decode(Plugins::XML::Transcoder)
-
end
-
-
30
private
-
-
# decodes the response payload using the given +transcoder+, which implements the decoding logic.
-
#
-
# +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
-
# which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
-
30
def decode(transcoder, *args)
-
# TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
-
-
315
decoder = transcoder.decode(self)
-
-
288
raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
-
-
288
@body.rewind
-
-
288
decoder.call(self, *args)
-
end
-
end
-
-
# Helper class which decodes the HTTP "content-type" header.
-
30
class ContentType
-
30
MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
-
30
CHARSET_RE = /;\s*charset=([^;]+)/i.freeze
-
-
30
def initialize(header_value)
-
11962
@header_value = header_value
-
11962
@mime_type = @charset = nil
-
11962
@initialized = false
-
end
-
-
# returns the mime type declared in the header.
-
#
-
# ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
-
30
def mime_type
-
315
return @mime_type if @initialized
-
-
273
load
-
-
273
@mime_type
-
end
-
-
# returns the charset declared in the header.
-
#
-
# ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
-
# ContentType.new("text/plain").charset #=> nil
-
30
def charset
-
11689
return @charset if @initialized
-
-
11689
load
-
-
11689
@charset
-
end
-
-
30
private
-
-
30
def load
-
11962
m = @header_value.to_s[MIME_TYPE_RE, 1]
-
11962
m && @mime_type = m.strip.downcase
-
-
11962
c = @header_value.to_s[CHARSET_RE, 1]
-
11962
c && @charset = c.strip.delete('"')
-
-
11962
@initialized = true
-
end
-
end
-
-
# Wraps an error which has happened while processing an HTTP Request. It has partial
-
# public API parity with HTTPX::Response, so users should rely on it to infer whether
-
# the returned response is one or the other.
-
#
-
# response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
-
# response.raise_for_status #=> raises if it wraps an error
-
30
class ErrorResponse
-
30
include Loggable
-
30
extend Forwardable
-
-
# the corresponding HTTPX::Request instance.
-
30
attr_reader :request
-
-
# the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
-
30
attr_reader :response
-
-
# the wrapped exception.
-
30
attr_reader :error
-
-
# the request uri
-
30
def_delegator :@request, :uri
-
-
# the IP address of the peer server.
-
30
def_delegator :@request, :peer_address
-
-
30
def initialize(request, error)
-
1908
@request = request
-
1908
@response = request.response if request.response.is_a?(Response)
-
1908
@error = error
-
1908
@options = request.options
-
1908
log_exception(@error)
-
1908
finish!
-
end
-
-
# returns the exception full message.
-
30
def to_s
-
10
@error.full_message(highlight: false)
-
end
-
-
# closes the error resources.
-
30
def close
-
45
@response.close if @response
-
end
-
-
# always true for error responses.
-
30
def finished?
-
1805
true
-
end
-
-
30
def finish!
-
1999
@request.connection = nil
-
end
-
-
# raises the wrapped exception.
-
30
def raise_for_status
-
105
raise @error
-
end
-
-
# buffers lost chunks to error response
-
30
def <<(data)
-
9
return unless @response
-
-
9
@response << data
-
end
-
end
-
end
-
-
30
require_relative "response/body"
-
30
require_relative "response/buffer"
-
30
require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
# Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
-
# (for buffering the response payload), the IO reader protocol (for consuming the response payload),
-
# and can be iterated over (via #each, which yields the payload in chunks).
-
30
class Response::Body
-
# the payload encoding (i.e. "utf-8", "ASCII-8BIT")
-
30
attr_reader :encoding
-
-
# Array of encodings contained in the response "content-encoding" header.
-
30
attr_reader :encodings
-
-
30
attr_reader :buffer
-
30
protected :buffer
-
-
# initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
-
30
def initialize(response, options)
-
11689
@response = response
-
11689
@headers = response.headers
-
11689
@options = options
-
11689
@window_size = options.window_size
-
11689
@encodings = []
-
11689
@length = 0
-
11689
@buffer = nil
-
11689
@reader = nil
-
11689
@state = :idle
-
-
# initialize response encoding
-
11689
@encoding = if (enc = response.content_type.charset)
-
181
begin
-
1841
Encoding.find(enc)
-
rescue ArgumentError
-
36
Encoding::BINARY
-
end
-
else
-
9848
Encoding::BINARY
-
end
-
-
11689
initialize_inflaters
-
end
-
-
30
def initialize_dup(other)
-
126
super
-
-
126
@buffer = other.instance_variable_get(:@buffer).dup
-
end
-
-
30
def closed?
-
85
@state == :closed
-
end
-
-
# write the response payload +chunk+ into the buffer. Inflates the chunk when required
-
# and supported.
-
30
def write(chunk)
-
15414
return if @state == :closed
-
-
15414
return 0 if chunk.empty?
-
-
14802
chunk = decode_chunk(chunk)
-
-
14802
transition(:open)
-
14802
@buffer.write(chunk)
-
-
14802
@response.emit(:chunk_received, chunk)
-
14784
chunk.bytesize
-
end
-
-
# reads a chunk from the payload (implementation of the IO reader protocol).
-
30
def read(*args)
-
467
return unless @buffer
-
-
467
unless @reader
-
219
rewind
-
219
@reader = @buffer
-
end
-
-
467
@reader.read(*args)
-
end
-
-
# size of the decoded response payload. May differ from "content-length" header if
-
# response was encoded over-the-wire.
-
30
def bytesize
-
253
@length
-
end
-
-
# yields the payload in chunks.
-
30
def each
-
54
return enum_for(__method__) unless block_given?
-
-
3
begin
-
36
if @buffer
-
36
rewind
-
96
while (chunk = @buffer.read(@window_size))
-
36
yield(chunk.force_encoding(@encoding))
-
end
-
end
-
ensure
-
36
close
-
end
-
end
-
-
# returns the declared filename in the "contennt-disposition" header, when present.
-
30
def filename
-
54
return unless @headers.key?("content-disposition")
-
-
45
Utils.get_filename(@headers["content-disposition"])
-
end
-
-
# returns the full response payload as a string.
-
30
def to_s
-
5634
return "".b unless @buffer
-
-
5239
@buffer.to_s
-
end
-
-
30
alias_method :to_str, :to_s
-
-
# whether the payload is empty.
-
30
def empty?
-
43
@length.zero?
-
end
-
-
# copies the payload to +dest+.
-
#
-
# body.copy_to("path/to/file")
-
# body.copy_to(Pathname.new("path/to/file"))
-
# body.copy_to(File.new("path/to/file"))
-
30
def copy_to(dest)
-
54
return unless @buffer
-
-
54
rewind
-
-
54
if dest.respond_to?(:path) && @buffer.respond_to?(:path)
-
9
FileUtils.mv(@buffer.path, dest.path)
-
else
-
45
IO.copy_stream(@buffer, dest)
-
end
-
ensure
-
54
close
-
end
-
-
# closes/cleans the buffer, resets everything
-
30
def close
-
1042
if @buffer
-
704
@buffer.close
-
704
@buffer = nil
-
end
-
1042
@length = 0
-
1042
transition(:closed)
-
end
-
-
30
def ==(other)
-
349
super || case other
-
when Response::Body
-
171
@buffer == other.buffer
-
else
-
115
@buffer = other
-
end
-
end
-
-
skipped
# :nocov:
-
skipped
def inspect
-
skipped
"#<#{self.class}:#{object_id} " \
-
skipped
"@state=#{@state} " \
-
skipped
"@length=#{@length}>"
-
skipped
end
-
skipped
# :nocov:
-
-
# rewinds the response payload buffer.
-
30
def rewind
-
1101
return unless @buffer
-
-
# in case there's some reading going on
-
1101
@reader = nil
-
-
1101
@buffer.rewind
-
end
-
-
30
private
-
-
# prepares inflaters for the advertised encodings in "content-encoding" header.
-
30
def initialize_inflaters
-
11689
@inflaters = nil
-
-
11689
return unless @headers.key?("content-encoding")
-
-
225
return unless @options.decompress_response_body
-
-
207
@inflaters = @headers.get("content-encoding").filter_map do |encoding|
-
207
next if encoding == "identity"
-
-
207
inflater = self.class.initialize_inflater_by_encoding(encoding, @response)
-
-
# do not uncompress if there is no decoder available. In fact, we can't reliably
-
# continue decompressing beyond that, so ignore.
-
207
break unless inflater
-
-
207
@encodings << encoding
-
207
inflater
-
end
-
end
-
-
# passes the +chunk+ through all inflaters to decode it.
-
30
def decode_chunk(chunk)
-
53
@inflaters.reverse_each do |inflater|
-
768
chunk = inflater.call(chunk)
-
15158
end if @inflaters
-
-
13716
@length += chunk.bytesize
-
-
15159
chunk
-
end
-
-
# tries transitioning the body STM to the +nextstate+.
-
30
def transition(nextstate)
-
14302
case nextstate
-
when :open
-
14802
return unless @state == :idle
-
-
9116
@buffer = Response::Buffer.new(
-
threshold_size: @options.body_threshold_size,
-
bytesize: @length,
-
encoding: @encoding
-
)
-
when :closed
-
1014
return if @state == :closed
-
end
-
-
10103
@state = nextstate
-
end
-
-
30
class << self
-
30
def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
-
188
case encoding
-
when "gzip"
-
188
Transcoder::GZIP.decode(response, **kwargs)
-
when "deflate"
-
18
Transcoder::Deflate.decode(response, **kwargs)
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "delegate"
-
30
require "stringio"
-
30
require "tempfile"
-
-
30
module HTTPX
-
# wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
-
30
class Response::Buffer < SimpleDelegator
-
30
attr_reader :buffer
-
30
protected :buffer
-
-
# initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
-
# the initial +bytesize+, and the +encoding+.
-
30
def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
-
9338
@threshold_size = threshold_size
-
9338
@bytesize = bytesize
-
9338
@encoding = encoding
-
9338
@buffer = StringIO.new("".b)
-
9338
super(@buffer)
-
end
-
-
30
def initialize_dup(other)
-
126
super
-
-
# create new descriptor in READ-ONLY mode
-
14
@buffer =
-
111
case other.buffer
-
when StringIO
-
117
StringIO.new(other.buffer.string, mode: File::RDONLY)
-
else
-
9
other.buffer.class.new(other.buffer.path, encoding: Encoding::BINARY, mode: File::RDONLY).tap do |temp|
-
9
FileUtils.copy_file(other.buffer.path, temp.path)
-
end
-
end
-
end
-
-
# size in bytes of the buffered content.
-
30
def size
-
370
@bytesize
-
end
-
-
# writes the +chunk+ into the buffer.
-
30
def write(chunk)
-
13802
@bytesize += chunk.bytesize
-
15259
try_upgrade_buffer
-
15259
@buffer.write(chunk)
-
end
-
-
# returns the buffered content as a string.
-
30
def to_s
-
4791
case @buffer
-
when StringIO
-
528
begin
-
5257
@buffer.string.force_encoding(@encoding)
-
rescue ArgumentError
-
@buffer.string
-
end
-
when Tempfile
-
81
rewind
-
81
content = @buffer.read
-
8
begin
-
81
content.force_encoding(@encoding)
-
rescue ArgumentError # ex: unknown encoding name - utf
-
content
-
end
-
end
-
end
-
-
# closes the buffer.
-
30
def close
-
830
@buffer.close
-
830
@buffer.unlink if @buffer.respond_to?(:unlink)
-
end
-
-
30
def ==(other)
-
171
super || begin
-
171
return false unless other.is_a?(Response::Buffer)
-
-
171
buffer_pos = @buffer.pos
-
171
other_pos = other.buffer.pos
-
171
@buffer.rewind
-
171
other.buffer.rewind
-
18
begin
-
171
FileUtils.compare_stream(@buffer, other.buffer)
-
ensure
-
171
@buffer.pos = buffer_pos
-
171
other.buffer.pos = other_pos
-
end
-
end
-
end
-
-
30
private
-
-
# initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
-
# has been reached.
-
30
def try_upgrade_buffer
-
15259
return unless @bytesize > @threshold_size
-
-
704
return if @buffer.is_a?(Tempfile)
-
-
180
aux = @buffer
-
-
180
@buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
-
-
180
if aux
-
180
aux.rewind
-
180
IO.copy_stream(aux, @buffer)
-
180
aux.close
-
end
-
-
180
__setobj__(@buffer)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "io/wait"
-
-
30
module HTTPX
-
#
-
# Implements the selector loop, where it registers and monitors "Selectable" objects.
-
#
-
# A Selectable object is an object which can calculate the **interests** (<tt>:r</tt>, <tt>:w</tt> or <tt>:rw</tt>,
-
# respectively "read", "write" or "read-write") it wants to monitor for, and returns (via <tt>to_io</tt> method) an
-
# IO object which can be passed to functions such as IO.select . More exhaustively, a Selectable **must** implement
-
# the following methods:
-
#
-
# state :: returns the state as a Symbol, must return <tt>:closed</tt> when disposed of resources.
-
# to_io :: returns the IO object.
-
# call :: gets called when the IO is ready.
-
# interests :: returns the current interests to monitor for, as described above.
-
# timeout :: returns nil or an integer, representing how long to wait for interests.
-
# handle_socket_timeout(Numeric) :: called when waiting for interest times out.
-
#
-
30
class Selector
-
30
extend Forwardable
-
-
30
READABLE = %i[rw r].freeze
-
30
WRITABLE = %i[rw w].freeze
-
-
30
private_constant :READABLE
-
30
private_constant :WRITABLE
-
-
30
def_delegator :@timers, :after
-
-
30
def_delegator :@selectables, :each
-
-
30
def initialize
-
9637
@timers = Timers.new
-
9637
@selectables = []
-
9637
@is_timer_interval = false
-
end
-
-
30
def empty?
-
40631
@selectables.empty? && @timers.empty?
-
end
-
-
30
def next_tick
-
40633
catch(:jump_tick) do
-
40633
timeout = next_timeout
-
40633
if timeout && timeout.negative?
-
@timers.fire
-
throw(:jump_tick)
-
end
-
-
8831
begin
-
40633
select(timeout) do |c|
-
39559
c.log(level: 2) { "[#{c.state}] selected from selector##{object_id} #{" after #{timeout} secs" unless timeout.nil?}..." }
-
-
39380
c.call
-
end
-
-
40387
@timers.fire
-
rescue TimeoutError => e
-
@timers.fire(e)
-
end
-
end
-
end
-
-
30
def terminate
-
# array may change during iteration
-
9369
selectables = @selectables.reject(&:inflight?)
-
-
9369
selectables.delete_if do |sel|
-
3694
sel.terminate
-
3685
sel.state == :closed
-
end
-
-
9360
until selectables.empty?
-
2
next_tick
-
-
2
selectables &= @selectables
-
end
-
end
-
-
30
def find_resolver(options)
-
9012
res = @selectables.find do |c|
-
87
c.is_a?(Resolver::Resolver) &&
-
options.resolver_options_match?(c.options)
-
end
-
-
9012
res.multi if res
-
end
-
-
30
def each_connection(&block)
-
44036
return enum_for(__method__) unless block
-
-
22018
@selectables.each do |c|
-
3166
case c
-
when Resolver::Resolver
-
512
c.each_connection(&block)
-
when Connection
-
2757
yield c
-
end
-
end
-
end
-
-
30
def find_connection(request_uri, options)
-
12051
each_connection.find do |connection|
-
1752
connection.match?(request_uri, options)
-
end
-
end
-
-
30
def find_mergeable_connection(connection)
-
9308
each_connection.find do |ch|
-
679
ch != connection && ch.mergeable?(connection)
-
end
-
end
-
-
# deregisters +io+ from selectables.
-
30
def deregister(io)
-
11418
@selectables.delete(io)
-
end
-
-
# register +io+.
-
30
def register(io)
-
11809
return if @selectables.include?(io)
-
-
11032
@selectables << io
-
end
-
-
30
private
-
-
30
def select(interval, &block)
-
# do not cause an infinite loop here.
-
#
-
# this may happen if timeout calculation actually triggered an error which causes
-
# the connections to be reaped (such as the total timeout error) before #select
-
# gets called.
-
40633
if @selectables.empty?
-
160
sleep(interval) if interval
-
143
return
-
end
-
-
# @type var r: (selectable | Array[selectable])?
-
# @type var w: (selectable | Array[selectable])?
-
40473
r, w = nil
-
-
40473
@selectables.delete_if do |io|
-
48057
interests = io.interests
-
-
48056
is_closed = io.state == :closed
-
-
48056
if is_closed
-
# the process by which io was closed may have already triggered the on_close callback,
-
# which already deregistered the io. this check prevents it from deleting the wrong io,
-
# because of https://bugs.ruby-lang.org/issues/22021 .
-
104
next(@selectables.include?(io))
-
end
-
-
47952
if interests
-
47677
io.log(level: 2) do
-
179
"[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
-
end
-
-
47677
if READABLE.include?(interests)
-
35441
r = r.nil? ? io : (Array(r) << io)
-
end
-
-
47677
if WRITABLE.include?(interests)
-
12830
w = w.nil? ? io : (Array(w) << io)
-
end
-
end
-
-
47952
is_closed
-
end
-
-
37856
case r
-
when Array
-
6946
w = Array(w) unless w.nil?
-
-
6946
select_many(r, w, interval, &block)
-
when nil
-
10894
case w
-
when Array
-
65
select_many(r, w, interval, &block)
-
when nil
-
146
return unless interval && @selectables.any?
-
-
# no selectables
-
# TODO: replace with sleep?
-
57
select_many(r, w, interval, &block)
-
else
-
11775
select_one(w, :w, interval, &block)
-
end
-
-
else
-
20094
case w
-
when Array
-
4
select_many(Array(r), w, interval, &block)
-
when nil
-
20708
select_one(r, :r, interval, &block)
-
else
-
828
if r == w
-
525
select_one(r, :rw, interval, &block)
-
else
-
303
select_many(Array(r), Array(w), interval, &block)
-
end
-
end
-
end
-
end
-
-
30
def select_many(r, w, interval, &block)
-
6150
begin
-
7375
readers, writers = ::IO.select(r, w, nil, interval)
-
rescue IOError => e
-
(Array(r) + Array(w)).each do |sel|
-
# TODO: is there a way to cheaply find the IO associated with the error?
-
sel.on_io_error(e)
-
end
-
rescue StandardError => e
-
(Array(r) + Array(w)).each do |sel|
-
sel.on_error(e)
-
end
-
-
return
-
rescue Exception => e # rubocop:disable Lint/RescueException
-
42
(Array(r) + Array(w)).each do |sel|
-
84
sel.force_close(true)
-
end
-
-
42
raise e
-
end
-
-
7333
if readers.nil? && writers.nil? && interval
-
96
[*r, *w].each { |io| io.handle_socket_timeout(interval) }
-
59
return
-
end
-
-
7263
if writers
-
99
readers.each do |io|
-
6961
yield io
-
-
# so that we don't yield 2 times
-
6952
writers.delete(io)
-
7262
end if readers
-
-
7254
writers.each(&block)
-
else
-
readers.each(&block) if readers
-
end
-
end
-
-
30
def select_one(io, interests, interval)
-
2657
begin
-
5164
result =
-
27844
case interests
-
20708
when :r then io.to_io.wait_readable(interval)
-
11775
when :w then io.to_io.wait_writable(interval)
-
525
when :rw then rw_wait(io, interval)
-
end
-
rescue IOError => e
-
io.on_io_error(e)
-
-
return
-
rescue StandardError => e
-
14
io.on_error(e)
-
-
14
return
-
rescue Exception => e # rubocop:disable Lint/RescueException
-
42
io.force_close(true)
-
-
42
raise e
-
end
-
-
32952
unless result || interval.nil?
-
966
io.handle_socket_timeout(interval) unless @is_timer_interval
-
860
return
-
end
-
-
31986
yield io
-
end
-
-
30
def next_timeout
-
40633
@is_timer_interval = false
-
-
40633
timer_interval = @timers.wait_interval
-
-
40633
connection_interval = @selectables.filter_map(&:timeout).min
-
-
40633
return connection_interval unless timer_interval
-
-
20422
if connection_interval.nil? || timer_interval <= connection_interval
-
14284
@is_timer_interval = true
-
-
13336
return timer_interval
-
end
-
-
6138
connection_interval
-
end
-
-
30
if RUBY_ENGINE == "jruby"
-
1
def rw_wait(io, interval)
-
68
io.to_io.wait(interval, :read_write)
-
end
-
29
elsif IO.const_defined?(:READABLE)
-
27
def rw_wait(io, interval)
-
404
io.to_io.wait(IO::READABLE | IO::WRITABLE, interval)
-
end
-
else
-
2
def rw_wait(io, interval)
-
53
if interval
-
51
io.to_io.wait(interval, :read_write)
-
else
-
2
io.to_io.wait(:read_write)
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
# Class implementing the APIs being used publicly.
-
#
-
# HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
-
# HTTPX.plugin(..).get(..) #=> creating an intermediate HTTPX::Session with plugin, then sending the GET request
-
30
class Session
-
30
include Loggable
-
30
include Chainable
-
-
# initializes the session with a set of +options+, which will be shared by all
-
# requests sent from it.
-
#
-
# When pass a block, it'll yield itself to it, then closes after the block is evaluated.
-
30
def initialize(options = EMPTY_HASH, &blk)
-
14553
@options = self.class.default_options.merge(options)
-
14553
@persistent = @options.persistent
-
14553
@pool = @options.pool_class.new(@options.pool_options)
-
14553
@wrapped = false
-
14553
@closing = false
-
14553
INSTANCES[self] = self if @persistent && @options.close_on_fork && INSTANCES
-
14553
wrap(&blk) if blk
-
end
-
-
# Yields itself the block, then closes it after the block is evaluated.
-
#
-
# session.wrap do |http|
-
# http.get("https://wikipedia.com")
-
# end # wikipedia connection closes here
-
30
def wrap
-
848
prev_wrapped = @wrapped
-
848
@wrapped = true
-
848
was_initialized = false
-
848
current_selector = get_current_selector do
-
848
selector = Selector.new
-
-
848
set_current_selector(selector)
-
-
848
was_initialized = true
-
-
848
selector
-
end
-
61
begin
-
848
yield self
-
ensure
-
848
unless prev_wrapped
-
848
if @persistent
-
11
deactivate(current_selector)
-
else
-
837
close(current_selector)
-
end
-
end
-
848
@wrapped = prev_wrapped
-
848
set_current_selector(nil) if was_initialized
-
end
-
end
-
-
# closes all the active connections from the session.
-
#
-
# when called directly without specifying +selector+, all available connections
-
# will be picked up from the connection pool and closed. Connections in use
-
# by other sessions, or same session in a different thread, will not be reaped.
-
30
def close(selector = Selector.new)
-
# throw resolvers away from the pool
-
9035
@pool.reset_resolvers
-
-
# preparing to throw away connections
-
21659
while (connection = @pool.pop_connection)
-
5937
next if connection.state == :closed
-
-
248
select_connection(connection, selector)
-
end
-
-
9035
selector_close(selector)
-
end
-
-
# performs one, or multple requests; it accepts:
-
#
-
# 1. one or multiple HTTPX::Request objects;
-
# 2. an HTTP verb, then a sequence of URIs or URI/options tuples;
-
# 3. one or multiple HTTP verb / uri / (optional) options tuples;
-
#
-
# when present, the set of +options+ kwargs is applied to all of the
-
# sent requests.
-
#
-
# respectively returns a single HTTPX::Response response, or all of them in an Array, in the same order.
-
#
-
# resp1 = session.request(req1)
-
# resp1, resp2 = session.request(req1, req2)
-
# resp1 = session.request("GET", "https://server.org/a")
-
# resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"])
-
# resp1, resp2 = session.request(["GET", "https://server.org/a"], ["GET", "https://server.org/b"])
-
# resp1 = session.request("POST", "https://server.org/a", form: { "foo" => "bar" })
-
# resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
-
# resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
-
#
-
30
def request(*args, **params)
-
9460
raise ArgumentError, "must perform at least one request" if args.empty?
-
-
9460
requests = args.first.is_a?(Request) ? args : build_requests(*args, params)
-
9417
responses = send_requests(*requests)
-
9148
return responses.first if responses.size == 1
-
-
353
responses
-
end
-
-
# returns a HTTP::Request instance built from the HTTP +verb+, the request +uri+, and
-
# the optional set of request-specific +options+. This request **must** be sent through
-
# the same session it was built from.
-
#
-
# req = session.build_request("GET", "https://server.com")
-
# resp = session.request(req)
-
30
def build_request(verb, uri, params = EMPTY_HASH, options = @options)
-
11507
rklass = options.request_class
-
11507
request = rklass.new(verb, uri, options, params)
-
11455
request.persistent = @persistent
-
11455
set_request_callbacks(request)
-
11455
request
-
end
-
-
30
def select_connection(connection, selector)
-
11663
pin(connection, selector)
-
11663
connection.log(level: 2) do
-
100
"registering into selector##{selector.object_id}"
-
end
-
11663
selector.register(connection)
-
end
-
-
30
def pin(conn_or_resolver, selector)
-
30236
conn_or_resolver.current_session = self
-
30236
conn_or_resolver.current_selector = selector
-
end
-
-
30
alias_method :select_resolver, :select_connection
-
-
30
def deselect_connection(connection, selector, cloned = false)
-
10659
connection.log(level: 2) do
-
86
"deregistering connection##{connection.object_id}(#{connection.state}) from selector##{selector.object_id}"
-
end
-
10659
selector.deregister(connection)
-
-
# do not check-in connections only created for Happy Eyeballs
-
10659
return if cloned
-
-
10636
return if @closing && connection.state == :closed && !connection.used?
-
-
10470
connection.log(level: 2) { "check-in connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
-
10384
@pool.checkin_connection(connection)
-
end
-
-
30
def deselect_resolver(resolver, selector)
-
678
resolver.log(level: 2) do
-
"deregistering resolver##{resolver.object_id}(#{resolver.state}) from selector##{selector.object_id}"
-
end
-
678
selector.deregister(resolver)
-
-
678
return if @closing && resolver.closed?
-
-
656
resolver.log(level: 2) { "check-in resolver##{resolver.object_id}(#{resolver.state}) in pool##{@pool.object_id}" }
-
656
@pool.checkin_resolver(resolver)
-
end
-
-
30
def try_clone_connection(connection, selector, family)
-
671
connection.family ||= family
-
-
671
return connection if connection.family == family
-
-
42
new_connection = connection.class.new(connection.origin, connection.options)
-
-
42
new_connection.family = family
-
-
42
connection.sibling = new_connection
-
-
42
do_init_connection(new_connection, selector)
-
42
new_connection
-
end
-
-
# returns the HTTPX::Connection through which the +request+ should be sent through.
-
30
def find_connection(request_uri, selector, options)
-
12051
if (connection = selector.find_connection(request_uri, options))
-
1584
connection.idling if connection.state == :closed
-
1584
log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
-
1490
return connection
-
end
-
-
10467
connection = @pool.checkout_connection(request_uri, options)
-
-
10431
log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
-
-
9426
case connection.state
-
when :idle
-
9255
do_init_connection(connection, selector)
-
when :open
-
75
if options.io
-
75
select_connection(connection, selector)
-
else
-
pin(connection, selector)
-
end
-
when :closing, :closed
-
1069
connection.idling
-
1069
if connection.addresses?
-
1061
select_connection(connection, selector)
-
else
-
# if addresses expired, resolve again
-
8
resolve_connection(connection, selector)
-
end
-
else
-
32
pin(connection, selector)
-
end
-
-
10374
connection
-
end
-
-
30
private
-
-
30
def selector_close(selector)
-
begin
-
9369
@closing = true
-
9369
selector.terminate
-
ensure
-
9369
@closing = false
-
end
-
end
-
-
# tries deactivating connections in the +selector+, deregistering the ones that have been deactivated.
-
30
def deactivate(selector)
-
659
selector.each_connection.to_a.each(&:deactivate)
-
end
-
-
# callback executed when an HTTP/2 promise frame has been received.
-
30
def on_promise(_, stream)
-
9
log(level: 2) { "#{stream.id}: refusing stream!" }
-
9
stream.refuse
-
end
-
-
# returns the corresponding HTTP::Response to the given +request+ if it has been received.
-
30
def fetch_response(request, _selector, _options)
-
23003
response = request.response
-
-
23003
return unless response && response.finished?
-
-
11675
log(level: 2) { "response##{response.object_id} fetched" }
-
-
11675
response
-
end
-
-
# sends the +request+ to the corresponding HTTPX::Connection
-
30
def send_request(request, selector, options = request.options)
-
2177
error = begin
-
11874
catch(:resolve_error) do
-
11874
log(level: 2) { "finding connection for request##{request.object_id}..." }
-
11874
connection = find_connection(request.uri, selector, options)
-
11754
connection.send(request)
-
end
-
rescue StandardError => e
-
45
e
-
end
-
11867
return unless error && error.is_a?(Exception)
-
-
120
raise error unless error.is_a?(Error)
-
-
113
response = ErrorResponse.new(request, error)
-
113
request.response = response
-
113
request.emit_response(response)
-
end
-
-
# returns a set of HTTPX::Request objects built from the given +args+ and +options+.
-
30
def build_requests(*args, params)
-
8625
requests = if args.size == 1
-
88
reqs = args.first
-
88
reqs.map do |verb, uri, ps = EMPTY_HASH|
-
176
request_params = params
-
176
request_params = request_params.merge(ps) unless ps.empty?
-
176
build_request(verb, uri, request_params)
-
end
-
else
-
8537
verb, uris = args
-
8537
if uris.respond_to?(:each)
-
8267
uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
-
9237
request_params = params
-
9237
request_params = request_params.merge(ps) unless ps.empty?
-
9237
build_request(verb, uri, request_params)
-
end
-
else
-
270
[build_request(verb, uris, params)]
-
end
-
end
-
8582
raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
-
-
8582
requests
-
end
-
-
30
def set_request_callbacks(request)
-
11311
request.on(:promise, &method(:on_promise))
-
end
-
-
30
def do_init_connection(connection, selector)
-
9297
resolve_connection(connection, selector) unless connection.family
-
end
-
-
# sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
-
30
def send_requests(*requests)
-
17954
selector = get_current_selector { Selector.new }
-
906
begin
-
9527
receive_requests(requests, selector)
-
ensure
-
9500
unless @wrapped
-
8484
if @persistent
-
648
deactivate(selector)
-
else
-
7836
close(selector)
-
end
-
end
-
end
-
end
-
-
# returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
-
30
def receive_requests(requests, selector)
-
9527
pending_idxs = [] #: Array[Integer]
-
9527
pending = 0
-
-
9527
waiting = false
-
-
9527
responses = requests.each_with_index.map do |request, idx|
-
10568
send_request(request, selector)
-
-
10554
fetch_response(request, selector, request.options).tap do |response|
-
10554
if response.nil?
-
9235
pending += 1
-
10185
request.on_response_arrived = lambda do
-
11303
pending_idxs << idx if waiting
-
end
-
end
-
end
-
end
-
-
12128
until pending.zero? || selector.empty?
-
# loop on selector until at least one response has been received.
-
40631
waiting = true
-
81262
catch(:coalesced) { selector.next_tick }
-
40385
waiting = false
-
-
79556
while (idx = pending_idxs.shift)
-
11269
request = requests[idx]
-
-
11269
response = fetch_response(request, selector, request.options)
-
-
# stop on first pending response. this avoids traversing pending idxs all the way
-
# (which is more expensive in the beginning, when the array is larger and N) while
-
# making the next loop cheaper (because we're dropping).
-
11269
next unless response
-
-
9939
request.complete!(response)
-
9007
responses[idx] = response
-
9939
request.on_response_arrived = nil
-
9007
pending -= 1
-
end
-
end
-
-
9267
raise Error, "something went wrong, responses not found and requests not resent" unless pending.zero?
-
-
9267
responses
-
end
-
-
30
def resolve_connection(connection, selector)
-
9291
if connection.addresses? || connection.open?
-
#
-
# there are two cases in which we want to activate initialization of
-
# connection immediately:
-
#
-
# 1. when the connection already has addresses, i.e. it doesn't need to
-
# resolve a name (not the same as name being an IP, yet)
-
# 2. when the connection is initialized with an external already open IO.
-
#
-
279
on_resolver_connection(connection, selector)
-
277
return
-
end
-
-
9012
resolver = find_resolver_for(connection, selector)
-
-
9012
pin(connection, selector)
-
9012
if early_resolve(resolver, connection)
-
8178
@pool.checkin_resolver(resolver)
-
else
-
798
resolver.lazy_resolve(connection)
-
end
-
end
-
-
30
def early_resolve(resolver, connection)
-
9012
resolver.early_resolve(connection)
-
end
-
-
30
def on_resolver_connection(connection, selector)
-
9308
from_pool = false
-
9308
found_connection = selector.find_mergeable_connection(connection) || begin
-
9272
from_pool = true
-
9272
connection.log(level: 2) do
-
100
"try finding a mergeable connection in pool##{@pool.object_id}"
-
end
-
9272
@pool.checkout_mergeable_connection(connection)
-
end
-
-
9308
return select_connection(connection, selector) unless found_connection
-
-
58
connection.log(level: 2) do
-
"try coalescing from #{from_pool ? "pool##{@pool.object_id}" : "selector##{selector.object_id}"} " \
-
"(connection##{found_connection.object_id}[#{found_connection.origin}])"
-
end
-
-
58
coalesce_connections(found_connection, connection, selector, from_pool)
-
end
-
-
30
def find_resolver_for(connection, selector)
-
9012
if (resolver = selector.find_resolver(connection.options))
-
7
resolver.log(level: 2) { "found resolver##{resolver.object_id}(#{resolver.state}) in selector##{selector.object_id}" }
-
7
return resolver
-
end
-
-
9005
resolver = @pool.checkout_resolver(connection.options)
-
9091
resolver.log(level: 2) { "found resolver##{resolver.object_id}(#{resolver.state}) in pool##{@pool.object_id}" }
-
9005
pin(resolver, selector)
-
-
9005
resolver
-
end
-
-
# coalesces +conn2+ into +conn1+. if +conn1+ was loaded from the connection pool
-
# (it is known via +from_pool+), then it adds its to the +selector+.
-
30
def coalesce_connections(conn1, conn2, selector, from_pool)
-
58
unless conn1.coalescable?(conn2)
-
29
conn2.log(level: 2) { "not coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
-
29
select_connection(conn2, selector)
-
29
if from_pool
-
8
conn1.log(level: 2) { "check-in connection##{conn1.object_id}(#{conn1.state}) in pool##{@pool.object_id}" }
-
8
@pool.checkin_connection(conn1)
-
end
-
29
return
-
end
-
-
29
conn2.log(level: 2) { "coalescing with connection##{conn1.object_id}[#{conn1.origin}])" }
-
29
select_connection(conn1, selector) if from_pool
-
29
conn2.coalesce!(conn1)
-
29
conn2.disconnect
-
end
-
-
30
def get_current_selector
-
10403
selector_store[self] || (yield if block_given?)
-
end
-
-
30
def set_current_selector(selector)
-
2294
if selector
-
1339
selector_store[self] = selector
-
else
-
848
selector_store.delete(self)
-
end
-
end
-
-
30
def selector_store
-
12697
th_current = Thread.current
-
-
12697
thread_selector_store(th_current) || begin
-
248
{}.compare_by_identity.tap do |store|
-
248
th_current.thread_variable_set(:httpx_persistent_selector_store, store)
-
end
-
end
-
end
-
-
30
def thread_selector_store(th)
-
17089
th.thread_variable_get(:httpx_persistent_selector_store)
-
end
-
-
30
Options.freeze
-
30
@default_options = Options.new
-
30
@default_options.freeze
-
30
@plugins = []
-
-
30
class << self
-
30
attr_reader :default_options
-
-
30
def inherited(klass)
-
7603
super
-
7603
klass.instance_variable_set(:@default_options, @default_options)
-
7603
klass.instance_variable_set(:@plugins, @plugins.dup)
-
7603
klass.instance_variable_set(:@callbacks, @callbacks.dup)
-
end
-
-
# returns a new HTTPX::Session instance, with the plugin pointed by +pl+ loaded.
-
#
-
# session_with_retries = session.plugin(:retries)
-
# session_with_custom = session.plugin(CustomPlugin)
-
#
-
30
def plugin(pl, options = nil, &block)
-
11402
label = pl
-
11402
pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
-
11402
raise ArgumentError, "Invalid plugin type: #{pl.class.inspect}" unless pl.is_a?(Module)
-
-
11394
if !@plugins.include?(pl)
-
11083
@plugins << pl
-
11083
pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
-
-
11083
@default_options = @default_options.dup
-
-
11083
include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
-
11083
extend(pl::ClassMethods) if defined?(pl::ClassMethods)
-
-
11083
opts = @default_options
-
11083
opts.extend_with_plugin_classes(pl)
-
-
11083
if defined?(pl::OptionsMethods)
-
# when a class gets dup'ed, the #initialize_dup callbacks isn't triggered.
-
# moreover, and because #method_added does not get triggered on mixin include,
-
# the callback is also forcefully manually called here.
-
4980
opts.options_class.instance_variable_set(:@options_names, opts.options_class.options_names.dup)
-
4980
(pl::OptionsMethods.instance_methods + pl::OptionsMethods.private_instance_methods - Object.instance_methods).each do |meth|
-
15257
opts.options_class.method_added(meth)
-
end
-
4980
@default_options = opts.options_class.new(opts)
-
end
-
-
11083
@default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
-
11083
@default_options = @default_options.merge(options) if options
-
-
11083
if pl.respond_to?(:subplugins)
-
2032
pl.subplugins.transform_keys(&Plugins.method(:load_plugin)).each do |main_pl, sub_pl|
-
# in case the main plugin has already been loaded, then apply subplugin functionality
-
# immediately
-
2649
next unless @plugins.include?(main_pl)
-
-
98
plugin(sub_pl, options, &block)
-
end
-
end
-
-
11083
pl.configure(self, &block) if pl.respond_to?(:configure)
-
-
11083
if label.is_a?(Symbol)
-
# in case an already-loaded plugin complements functionality of
-
# the plugin currently being loaded, loaded it now
-
8257
@plugins.each do |registered_pl|
-
22377
next if registered_pl == pl
-
-
14120
next unless registered_pl.respond_to?(:subplugins)
-
-
3898
sub_pl = registered_pl.subplugins[label]
-
-
3898
next unless sub_pl
-
-
203
plugin(sub_pl, options, &block)
-
end
-
end
-
-
11083
@default_options.freeze
-
11083
set_temporary_name("#{superclass}/#{pl}") if respond_to?(:set_temporary_name) # ruby 3.4 only
-
310
elsif options
-
# this can happen when two plugins are loaded, an one of them calls the other under the hood,
-
# albeit changing some default.
-
26
@default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
-
26
@default_options = @default_options.merge(options) if options
-
-
18
@default_options.freeze
-
end
-
-
11386
self
-
end
-
end
-
-
# setup of the support for close_on_fork sessions.
-
# adapted from https://github.com/mperham/connection_pool/blob/main/lib/connection_pool.rb#L48
-
30
if Process.respond_to?(:fork)
-
28
INSTANCES = ObjectSpace::WeakMap.new
-
28
private_constant :INSTANCES
-
-
28
def self.after_fork
-
1
INSTANCES.each_value(&:close)
-
1
nil
-
end
-
-
28
if ::Process.respond_to?(:_fork)
-
24
module ForkTracker
-
24
def _fork
-
1
pid = super
-
1
Session.after_fork if pid.zero?
-
1
pid
-
end
-
end
-
24
Process.singleton_class.prepend(ForkTracker)
-
end
-
else
-
2
INSTANCES = nil
-
2
private_constant :INSTANCES
-
-
2
def self.after_fork
-
# noop
-
end
-
end
-
end
-
-
# session may be overridden by certain adapters.
-
30
S = Session
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
unless ENV.keys.grep(/\Ahttps?_proxy\z/i).empty?
-
1
proxy_session = plugin(:proxy)
-
1
remove_const(:Session)
-
1
const_set(:Session, proxy_session.class)
-
-
# redefine the default options static var, which needs to
-
# refresh options_class
-
1
options = proxy_session.class.default_options.to_hash
-
1
original_verbosity = $VERBOSE
-
1
$VERBOSE = nil
-
1
new_options_class = proxy_session.class.default_options.options_class.dup
-
1
const_set(:Options, new_options_class)
-
1
options[:options_class] = Class.new(new_options_class).freeze
-
1
options.freeze
-
1
Options.send(:const_set, :DEFAULT_OPTIONS, options)
-
1
Session.instance_variable_set(:@default_options, Options.new(options))
-
1
$VERBOSE = original_verbosity
-
end
-
-
skipped
# :nocov:
-
skipped
if Session.default_options.debug_level > 2
-
skipped
proxy_session = plugin(:internal_telemetry)
-
skipped
remove_const(:Session)
-
skipped
const_set(:Session, proxy_session.class)
-
skipped
end
-
skipped
# :nocov:
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
class Timers
-
30
def initialize
-
9637
@intervals = []
-
end
-
-
30
def empty?
-
160
@intervals.empty?
-
end
-
-
30
def after(interval_in_secs, cb = nil, &blk)
-
23846
callback = cb || blk
-
-
23846
raise Error, "timer must have a callback" unless callback
-
-
# I'm assuming here that most requests will have the same
-
# request timeout, as in most cases they share common set of
-
# options. A user setting different request timeouts for 100s of
-
# requests will already have a hard time dealing with that.
-
39096
unless (interval = @intervals.bsearch { |t| t.interval == interval_in_secs })
-
12202
interval = Interval.new(interval_in_secs)
-
12202
@intervals << interval
-
12202
@intervals.sort!
-
end
-
-
23846
interval << callback
-
-
23846
@next_interval_at = nil
-
-
23846
Timer.new(interval, callback)
-
end
-
-
30
def wait_interval
-
40633
return if @intervals.empty?
-
-
20422
first_interval = @intervals.first
-
-
20422
drop_elapsed!(0) if first_interval.elapsed?(0)
-
-
20422
@next_interval_at = Utils.now
-
-
20422
first_interval.interval
-
end
-
-
30
def fire(error = nil)
-
40387
raise error if error && error.timeout != @intervals.first
-
40387
return if @intervals.empty? || !@next_interval_at
-
-
19150
elapsed_time = Utils.elapsed_time(@next_interval_at)
-
-
19150
drop_elapsed!(elapsed_time)
-
-
19150
@next_interval_at = nil if @intervals.empty?
-
end
-
-
30
private
-
-
30
def drop_elapsed!(elapsed_time)
-
40372
@intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
-
end
-
-
30
class Timer
-
30
def initialize(interval, callback)
-
23846
@interval = interval
-
23846
@callback = callback
-
end
-
-
30
def cancel
-
33334
@interval.delete(@callback)
-
end
-
end
-
-
30
class Interval
-
30
include Comparable
-
-
30
attr_reader :interval
-
-
30
def initialize(interval)
-
12202
@interval = interval
-
12202
@callbacks = []
-
end
-
-
30
def <=>(other)
-
1746
@interval <=> other.interval
-
end
-
-
30
def ==(other)
-
return @interval == other if other.is_a?(Numeric)
-
-
@interval == other.to_f # rubocop:disable Lint/FloatComparison
-
end
-
-
30
def to_f
-
Float(@interval)
-
end
-
-
30
def <<(callback)
-
23846
@callbacks << callback
-
end
-
-
30
def delete(callback)
-
33334
@callbacks.delete(callback)
-
end
-
-
30
def no_callbacks?
-
@callbacks.empty?
-
end
-
-
30
def elapsed?(elapsed = 0)
-
20422
(@interval - elapsed) <= 0 || @callbacks.empty?
-
end
-
-
30
def elapse(elapsed)
-
# same as elapsing
-
20687
return 0 if @callbacks.empty?
-
-
10805
@interval -= elapsed
-
-
11222
if @interval <= 0
-
1127
cb = @callbacks.dup
-
1127
cb.each(&:call)
-
end
-
-
11222
@interval
-
end
-
end
-
30
private_constant :Interval
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Transcoder
-
30
module_function
-
-
30
def normalize_keys(key, value, transcoder = self, &block)
-
2846
if value.respond_to?(:to_ary)
-
511
if value.empty?
-
144
block.call("#{key}[]")
-
else
-
367
value.to_ary.each do |element|
-
590
transcoder.normalize_keys("#{key}[]", element, transcoder, &block)
-
end
-
end
-
2334
elsif value.respond_to?(:to_hash)
-
648
value.to_hash.each do |child_key, child_value|
-
648
transcoder.normalize_keys("#{key}[#{child_key}]", child_value, transcoder, &block)
-
end
-
else
-
1687
block.call(key.to_s, value)
-
end
-
end
-
-
# based on https://github.com/rack/rack/blob/d15dd728440710cfc35ed155d66a98dc2c07ae42/lib/rack/query_parser.rb#L82
-
30
def normalize_query(params, name, v, depth)
-
207
raise Error, "params depth surpasses what's supported" if depth <= 0
-
-
207
name =~ /\A[\[\]]*([^\[\]]+)\]*/
-
207
k = Regexp.last_match(1) || ""
-
207
after = Regexp.last_match ? Regexp.last_match.post_match : ""
-
-
207
if k.empty?
-
18
return Array(v) if !v.empty? && name == "[]"
-
-
8
return
-
end
-
-
168
case after
-
when ""
-
56
params[k] = v
-
when "["
-
8
params[name] = v
-
when "[]"
-
18
params[k] ||= []
-
18
raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
-
-
18
params[k] << v
-
when /^\[\]\[([^\[\]]+)\]$/, /^\[\](.+)$/
-
36
child_key = Regexp.last_match(1)
-
36
params[k] ||= []
-
36
raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
-
-
36
if params[k].last.is_a?(Hash) && !params_hash_has_key?(params[k].last, child_key)
-
9
normalize_query(params[k].last, child_key, v, depth - 1)
-
else
-
27
params[k] << normalize_query({}, child_key, v, depth - 1)
-
end
-
else
-
63
params[k] ||= {}
-
63
raise Error, "expected Hash (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Hash)
-
-
56
params[k] = normalize_query(params[k], after, v, depth - 1)
-
end
-
-
189
params
-
end
-
-
30
def params_hash_has_key?(hash, key)
-
18
return false if key.include?("[]")
-
-
18
key.split(/[\[\]]+/).inject(hash) do |h, part|
-
18
next h if part == ""
-
18
return false unless h.is_a?(Hash) && h.key?(part)
-
-
9
h[part]
-
end
-
-
9
true
-
end
-
end
-
end
-
-
30
require "httpx/transcoder/body"
-
30
require "httpx/transcoder/form"
-
30
require "httpx/transcoder/json"
-
30
require "httpx/transcoder/chunker"
-
30
require "httpx/transcoder/deflate"
-
30
require "httpx/transcoder/gzip"
-
# frozen_string_literal: true
-
-
30
require "delegate"
-
-
30
module HTTPX::Transcoder
-
30
module Body
-
30
class Error < HTTPX::Error; end
-
-
30
module_function
-
-
30
class Encoder < SimpleDelegator
-
30
def initialize(body)
-
1761
body = body.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && body.is_a?(Pathname)
-
1761
@body = body
-
1761
super
-
end
-
-
30
def bytesize
-
6761
if @body.respond_to?(:bytesize)
-
2859
@body.bytesize
-
3901
elsif @body.respond_to?(:to_ary)
-
1475
@body.sum(&:bytesize)
-
2426
elsif @body.respond_to?(:size)
-
1689
@body.size || Float::INFINITY
-
737
elsif @body.respond_to?(:length)
-
405
@body.length || Float::INFINITY
-
332
elsif @body.respond_to?(:each)
-
324
Float::INFINITY
-
else
-
9
raise Error, "cannot determine size of body: #{@body.inspect}"
-
end
-
end
-
-
30
def content_type
-
1628
"application/octet-stream"
-
end
-
end
-
-
30
def encode(body)
-
1761
Encoder.new(body)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "forwardable"
-
-
30
module HTTPX::Transcoder
-
30
module Chunker
-
30
class Error < HTTPX::Error; end
-
-
30
CRLF = "\r\n".b
-
-
30
class Encoder
-
30
extend Forwardable
-
-
30
def initialize(body)
-
108
@raw = body
-
end
-
-
30
def each
-
108
return enum_for(__method__) unless block_given?
-
-
108
@raw.each do |chunk|
-
504
yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
-
end
-
108
yield "0#{CRLF}"
-
end
-
-
30
def respond_to_missing?(meth, *args)
-
120
@raw.respond_to?(meth, *args) || super
-
end
-
end
-
-
30
class Decoder
-
30
extend Forwardable
-
-
30
def_delegator :@buffer, :empty?
-
-
30
def_delegator :@buffer, :<<
-
-
30
def_delegator :@buffer, :clear
-
-
30
def initialize(buffer, trailers = false)
-
165
@buffer = buffer
-
165
@chunk_buffer = "".b
-
165
@finished = false
-
165
@state = :length
-
165
@trailers = trailers
-
end
-
-
30
def to_s
-
165
@buffer
-
end
-
-
30
def each
-
221
loop do
-
1290
case @state
-
when :length
-
429
index = @buffer.index(CRLF)
-
429
return unless index && index.positive?
-
-
# Read hex-length
-
429
hexlen = @buffer.byteslice(0, index)
-
429
@buffer = @buffer.byteslice(index..-1) || "".b
-
429
hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
-
429
@chunk_length = hexlen.hex
-
# check if is last chunk
-
429
@finished = @chunk_length.zero?
-
429
nextstate(:crlf)
-
when :crlf
-
702
crlf_size = @finished && !@trailers ? 4 : 2
-
# consume CRLF
-
702
return if @buffer.bytesize < crlf_size
-
702
raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
-
-
702
@buffer = @buffer.byteslice(crlf_size..-1)
-
702
if @chunk_length.nil?
-
273
nextstate(:length)
-
else
-
429
return if @finished
-
-
282
nextstate(:data)
-
end
-
when :data
-
318
chunk = @buffer.byteslice(0, @chunk_length)
-
318
@buffer = @buffer.byteslice(@chunk_length..-1) || "".b
-
318
@chunk_buffer << chunk
-
283
@chunk_length -= chunk.bytesize
-
318
if @chunk_length.zero?
-
282
yield @chunk_buffer unless @chunk_buffer.empty?
-
273
@chunk_buffer.clear
-
273
@chunk_length = nil
-
273
nextstate(:crlf)
-
end
-
end
-
1293
break if @buffer.empty?
-
end
-
end
-
-
30
def finished?
-
212
@finished
-
end
-
-
30
private
-
-
30
def nextstate(state)
-
1257
@state = state
-
end
-
end
-
-
30
module_function
-
-
30
def encode(chunks)
-
108
Encoder.new(chunks)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "zlib"
-
30
require_relative "utils/deflater"
-
-
30
module HTTPX
-
30
module Transcoder
-
30
module Deflate
-
30
class Deflater < Transcoder::Deflater
-
30
def deflate(chunk)
-
78
@deflater ||= Zlib::Deflate.new
-
-
78
unless chunk.nil?
-
27
chunk = @deflater.deflate(chunk)
-
-
# deflate call may return nil, while still
-
# retaining the last chunk in the deflater.
-
27
return chunk unless chunk.empty?
-
end
-
-
54
return if @deflater.closed?
-
-
27
last = @deflater.finish
-
27
@deflater.close
-
-
27
last unless last.empty?
-
end
-
end
-
-
30
module_function
-
-
30
def encode(body)
-
27
Deflater.new(body)
-
end
-
-
30
def decode(response, bytesize: nil)
-
18
bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
-
18
GZIP::Inflater.new(bytesize)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "forwardable"
-
30
require "uri"
-
30
require_relative "multipart"
-
-
30
module HTTPX
-
30
module Transcoder
-
30
module Form
-
30
module_function
-
-
30
PARAM_DEPTH_LIMIT = 32
-
-
30
class Encoder
-
30
extend Forwardable
-
-
30
def_delegator :@raw, :to_s
-
-
30
def_delegator :@raw, :to_str
-
-
30
def_delegator :@raw, :bytesize
-
-
30
def_delegator :@raw, :==
-
-
30
def initialize(form)
-
852
@raw = form.each_with_object("".b) do |(key, val), buf|
-
1464
HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
-
1687
buf << "&" unless buf.empty?
-
1687
buf << URI.encode_www_form_component(k)
-
1687
buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
-
end
-
end
-
end
-
-
30
def content_type
-
662
"application/x-www-form-urlencoded"
-
end
-
end
-
-
30
module Decoder
-
30
module_function
-
-
30
def call(response, *)
-
45
URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
-
108
HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
-
end
-
end
-
end
-
-
30
def encode(form)
-
852
Encoder.new(form)
-
end
-
-
30
def decode(response)
-
72
content_type = response.content_type.mime_type
-
-
64
case content_type
-
when "application/x-www-form-urlencoded"
-
45
Decoder
-
when "multipart/form-data"
-
18
Multipart::Decoder.new(response)
-
else
-
9
raise Error, "invalid form mime type (#{content_type})"
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "zlib"
-
-
30
module HTTPX
-
30
module Transcoder
-
30
module GZIP
-
30
class Deflater < Transcoder::Deflater
-
30
def initialize(body)
-
55
@compressed_chunk = "".b
-
55
@deflater = nil
-
55
super
-
end
-
-
30
def deflate(chunk)
-
110
@deflater ||= Zlib::GzipWriter.new(self)
-
-
110
if chunk.nil?
-
55
unless @deflater.closed?
-
55
@deflater.flush
-
55
@deflater.close
-
55
compressed_chunk
-
end
-
else
-
55
@deflater.write(chunk)
-
55
compressed_chunk
-
end
-
end
-
-
30
private
-
-
30
def write(*chunks)
-
165
chunks.sum do |chunk|
-
165
chunk = chunk.to_s
-
165
@compressed_chunk << chunk
-
165
chunk.bytesize
-
end
-
end
-
-
30
def compressed_chunk
-
110
@compressed_chunk.dup
-
ensure
-
110
@compressed_chunk.clear
-
end
-
end
-
-
30
class Inflater
-
30
def initialize(bytesize)
-
206
@inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
-
206
@bytesize = bytesize
-
end
-
-
30
def call(chunk)
-
755
buffer = @inflater.inflate(chunk)
-
702
@bytesize -= chunk.bytesize
-
755
if @bytesize <= 0
-
144
buffer << @inflater.finish
-
144
@inflater.close
-
end
-
755
buffer
-
end
-
end
-
-
30
module_function
-
-
30
def encode(body)
-
55
Deflater.new(body)
-
end
-
-
30
def decode(response, bytesize: nil)
-
188
bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
-
188
Inflater.new(bytesize)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "forwardable"
-
-
30
module HTTPX::Transcoder
-
30
module JSON
-
30
module_function
-
-
30
JSON_REGEX = %r{
-
\b
-
application/
-
# optional vendor specific type
-
(?:
-
# token as per https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6
-
[!#$%&'*+\-.^_`|~0-9a-z]+
-
# literal plus sign
-
\+
-
)?
-
json
-
\b
-
}ix.freeze
-
-
30
class Encoder
-
30
extend Forwardable
-
-
30
def_delegator :@raw, :to_s
-
-
30
def_delegator :@raw, :bytesize
-
-
30
def_delegator :@raw, :==
-
-
30
def initialize(json)
-
102
@raw = JSON.json_dump(json)
-
102
@charset = @raw.encoding.name.downcase
-
end
-
-
30
def content_type
-
102
"application/json; charset=#{@charset}"
-
end
-
end
-
-
30
def encode(json)
-
102
Encoder.new(json)
-
end
-
-
30
def decode(response)
-
216
content_type = response.content_type.mime_type
-
-
216
raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
-
-
198
method(:json_load)
-
end
-
-
# rubocop:disable Style/SingleLineMethods
-
30
if defined?(MultiJson)
-
5
def json_load(*args); MultiJson.load(*args); end
-
3
def json_dump(*args); MultiJson.dump(*args); end
-
27
elsif defined?(Oj)
-
5
def json_load(response, *args); Oj.load(response.to_s, *args); end
-
3
def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
-
25
elsif defined?(Yajl)
-
4
def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
-
2
def json_dump(*args); Yajl::Encoder.encode(*args); end
-
else
-
25
require "json"
-
193
def json_load(*args); ::JSON.parse(*args); end
-
113
def json_dump(*args); ::JSON.generate(*args); end
-
end
-
# rubocop:enable Style/SingleLineMethods
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require_relative "multipart/encoder"
-
30
require_relative "multipart/decoder"
-
30
require_relative "multipart/part"
-
30
require_relative "multipart/mime_type_detector"
-
-
30
module HTTPX::Transcoder
-
30
module Multipart
-
30
module_function
-
-
30
def multipart?(form_data)
-
1804
form_data.any? do |_, v|
-
2344
multipart_value?(v) ||
-
2210
(v.respond_to?(:to_ary) && v.to_ary.any? { |av| multipart_value?(av) }) ||
-
2210
(v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| multipart_value?(e) })
-
end
-
end
-
-
30
def multipart_value?(value)
-
5343
value.respond_to?(:read) ||
-
3798
(value.is_a?(Hash) &&
-
value.key?(:body) &&
-
868
(value.key?(:filename) || value.key?(:content_type)))
-
end
-
-
30
def normalize_keys(key, value, transcoder = self, &block)
-
1991
if multipart_value?(value)
-
1271
block.call(key.to_s, value)
-
else
-
720
HTTPX::Transcoder.normalize_keys(key, value, transcoder, &block)
-
end
-
end
-
-
30
def encode(form_data)
-
1142
Encoder.new(form_data)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "tempfile"
-
30
require "delegate"
-
-
30
module HTTPX
-
30
module Transcoder
-
30
module Multipart
-
30
class FilePart < SimpleDelegator
-
30
attr_reader :original_filename, :content_type
-
-
30
def initialize(filename, content_type)
-
36
@original_filename = filename
-
36
@content_type = content_type
-
36
@file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
-
36
super(@file)
-
end
-
end
-
-
30
class Decoder
-
30
include HTTPX::Utils
-
-
30
CRLF = "\r\n"
-
30
BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
-
30
MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
-
30
MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
-
30
MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
-
30
WINDOW_SIZE = 2 << 14
-
-
30
def initialize(response)
-
2
@boundary = begin
-
18
m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
-
18
raise Error, "no boundary declared in content-type header" unless m
-
-
18
m.strip
-
end
-
18
@buffer = "".b
-
18
@parts = {}
-
18
@intermediate_boundary = "--#{@boundary}"
-
18
@state = :idle
-
18
@current = nil
-
end
-
-
30
def call(response, *)
-
18
response.body.each do |chunk|
-
18
@buffer << chunk
-
-
18
parse
-
end
-
-
18
raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
-
-
18
@parts
-
end
-
-
30
private
-
-
30
def parse
-
16
case @state
-
when :idle
-
18
raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
-
-
18
@buffer = @buffer.byteslice((@intermediate_boundary.bytesize + 2)..-1)
-
-
18
@state = :part_header
-
when :part_header
-
54
idx = @buffer.index("#{CRLF}#{CRLF}")
-
-
# raise Error, "couldn't parse part headers" unless idx
-
54
return unless idx
-
-
# @type var head: String
-
54
head = @buffer.byteslice(0..(idx + 4 - 1))
-
-
54
@buffer = @buffer.byteslice(head.bytesize..-1)
-
-
54
content_type = head[MULTIPART_CONTENT_TYPE, 1] || "text/plain"
-
96
if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
-
54
name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
-
54
name.gsub!(/\\(.)/, "\\1")
-
12
name
-
else
-
name = head[MULTIPART_CONTENT_ID, 1]
-
end
-
-
54
filename = HTTPX::Utils.get_filename(head)
-
-
54
name = filename || +"#{content_type}[]" if name.nil? || name.empty?
-
-
54
@current = name
-
-
48
@parts[name] = if filename
-
36
FilePart.new(filename, content_type)
-
else
-
18
"".b
-
end
-
-
54
@state = :part_body
-
when :part_body
-
54
part = @parts[@current]
-
-
54
body_separator = if part.is_a?(FilePart)
-
32
"#{CRLF}#{CRLF}"
-
else
-
18
CRLF
-
end
-
54
idx = @buffer.index(body_separator)
-
-
54
if idx
-
54
payload = @buffer.byteslice(0..(idx - 1))
-
54
@buffer = @buffer.byteslice((idx + body_separator.bytesize)..-1)
-
54
part << payload
-
54
part.rewind if part.respond_to?(:rewind)
-
54
@state = :parse_boundary
-
else
-
part << @buffer
-
@buffer.clear
-
end
-
when :parse_boundary
-
54
raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
-
-
54
@buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
-
-
54
if @buffer == "--"
-
18
@buffer.clear
-
18
@state = :done
-
18
return
-
35
elsif @buffer.start_with?(CRLF)
-
36
@buffer = @buffer.byteslice(2..-1)
-
36
@state = :part_header
-
else
-
return
-
end
-
when :done
-
raise Error, "parsing should have been over by now"
-
20
end until @buffer.empty?
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Transcoder::Multipart
-
30
class Encoder
-
30
attr_reader :bytesize
-
-
30
def initialize(form)
-
1142
@boundary = ("-" * 21) << SecureRandom.hex(21)
-
1142
@part_index = 0
-
1142
@buffer = "".b
-
-
1142
@form = form
-
1142
@bytesize = 0
-
1142
@parts = to_parts(form)
-
end
-
-
30
def content_type
-
1142
"multipart/form-data; boundary=#{@boundary}"
-
end
-
-
30
def to_s
-
21
read || ""
-
ensure
-
21
rewind
-
end
-
-
30
def read(length = nil, outbuf = nil)
-
4245
data = String(outbuf).clear.force_encoding(Encoding::BINARY) if outbuf
-
4245
data ||= "".b
-
-
4245
read_chunks(data, length)
-
-
4245
data unless length && data.empty?
-
end
-
-
30
def rewind
-
57
form = @form.each_with_object([]) do |(key, val), aux|
-
57
if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
-
# @type var val: File
-
57
val = val.reopen(val.path, File::RDONLY)
-
end
-
57
val.rewind if val.respond_to?(:rewind)
-
57
aux << [key, val]
-
end
-
57
@form = form
-
57
@bytesize = 0
-
57
@parts = to_parts(form)
-
57
@part_index = 0
-
end
-
-
30
private
-
-
30
def to_parts(form)
-
1199
params = form.each_with_object([]) do |(key, val), aux|
-
1415
Transcoder::Multipart.normalize_keys(key, val) do |k, v|
-
1415
next if v.nil?
-
-
1415
value, content_type, filename = Part.call(v)
-
-
1415
header = header_part(k, content_type, filename)
-
1263
@bytesize += header.size
-
1415
aux << header
-
-
1263
@bytesize += value.size
-
1415
aux << value
-
-
1415
delimiter = StringIO.new("\r\n")
-
1263
@bytesize += delimiter.size
-
1415
aux << delimiter
-
end
-
end
-
1199
final_delimiter = StringIO.new("--#{@boundary}--\r\n")
-
1071
@bytesize += final_delimiter.size
-
1199
params << final_delimiter
-
-
1199
params
-
end
-
-
30
def header_part(key, content_type, filename)
-
1415
header = "--#{@boundary}\r\n".b
-
1415
header << "Content-Disposition: form-data; name=#{key.inspect}".b
-
1415
header << "; filename=#{filename.inspect}" if filename
-
1415
header << "\r\nContent-Type: #{content_type}\r\n\r\n"
-
1415
StringIO.new(header)
-
end
-
-
30
def read_chunks(buffer, length = nil)
-
5409
while @part_index < @parts.size
-
12629
chunk = read_from_part(length)
-
-
12629
next unless chunk
-
-
7241
buffer << chunk.force_encoding(Encoding::BINARY)
-
-
7241
next unless length
-
-
6376
length -= chunk.bytesize
-
-
7164
break if length.zero?
-
end
-
end
-
-
# if there's a current part to read from, tries to read a chunk.
-
30
def read_from_part(max_length = nil)
-
12629
part = @parts[@part_index]
-
-
12629
chunk = part.read(max_length, @buffer)
-
-
12629
return chunk if chunk && !chunk.empty?
-
-
5388
part.close if part.respond_to?(:close)
-
-
4804
@part_index += 1
-
-
2388
nil
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Transcoder::Multipart
-
30
module MimeTypeDetector
-
30
module_function
-
-
30
DEFAULT_MIMETYPE = "application/octet-stream"
-
-
# inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
-
30
if defined?(FileMagic)
-
1
MAGIC_NUMBER = 256 * 1024
-
-
1
def call(file, _)
-
1
return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
-
-
1
mime = FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
-
1
filemagic.buffer(file.read(MAGIC_NUMBER))
-
end
-
-
1
file.rewind
-
-
1
mime
-
end
-
28
elsif defined?(Marcel)
-
1
def call(file, filename)
-
1
return nil if file.eof? # marcel returns "application/octet-stream" for empty files
-
-
1
Marcel::MimeType.for(file, name: filename)
-
end
-
-
27
elsif defined?(MimeMagic)
-
-
1
def call(file, _)
-
1
mime = MimeMagic.by_magic(file)
-
1
mime.type if mime
-
end
-
-
26
elsif system("which file", out: File::NULL)
-
27
require "open3"
-
-
27
def call(file, _)
-
835
return if file.eof? # file command returns "application/x-empty" for empty files
-
-
785
Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
-
83
begin
-
785
IO.copy_stream(file, stdin.binmode)
-
rescue Errno::EPIPE
-
end
-
785
file.rewind
-
785
stdin.close
-
-
785
status = thread.value
-
-
# call to file command failed
-
785
if status.nil? || !status.success?
-
$stderr.print(stderr.read)
-
else
-
-
785
output = stdout.read.strip
-
-
785
if output.include?("cannot open")
-
$stderr.print(output)
-
else
-
785
output
-
end
-
end
-
end
-
end
-
-
else
-
-
def call(_, _); end
-
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Transcoder::Multipart
-
30
module Part
-
30
module_function
-
-
30
def call(value)
-
# take out specialized objects of the way
-
1415
if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
-
128
return value, value.content_type, value.filename
-
end
-
-
1271
content_type = filename = nil
-
-
1271
if value.is_a?(Hash)
-
434
content_type = value[:content_type]
-
434
filename = value[:filename]
-
434
value = value[:body]
-
end
-
-
1271
value = value.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
-
-
1271
if value.respond_to?(:path) && value.respond_to?(:read)
-
# either a File, a Tempfile, or something else which has to quack like a file
-
839
filename ||= File.basename(value.path)
-
839
content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
-
839
[value, content_type, filename]
-
else
-
432
[StringIO.new(value.to_s), content_type || "text/plain", filename]
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require "stringio"
-
-
30
module HTTPX
-
30
module Transcoder
-
30
class BodyReader
-
30
def initialize(body)
-
243
@body = if body.respond_to?(:read)
-
23
body.rewind if body.respond_to?(:rewind)
-
23
body
-
219
elsif body.respond_to?(:each)
-
42
body.enum_for(:each)
-
else
-
178
StringIO.new(body.to_s)
-
end
-
end
-
-
30
def bytesize
-
525
return @body.bytesize if @body.respond_to?(:bytesize)
-
-
483
Float::INFINITY
-
end
-
-
30
def read(length = nil, outbuf = nil)
-
552
return @body.read(length, outbuf) if @body.respond_to?(:read)
-
-
begin
-
112
chunk = @body.next
-
56
if outbuf
-
outbuf.replace(chunk)
-
else
-
56
outbuf = chunk
-
end
-
56
outbuf unless length && outbuf.empty?
-
32
rescue StopIteration
-
end
-
end
-
-
30
def close
-
55
@body.close if @body.respond_to?(:close)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
require_relative "body_reader"
-
-
30
module HTTPX
-
30
module Transcoder
-
30
class Deflater
-
30
attr_reader :content_type
-
-
30
def initialize(body)
-
96
@content_type = body.content_type
-
96
@body = BodyReader.new(body)
-
96
@closed = false
-
end
-
-
30
def bytesize
-
370
buffer_deflate!
-
-
370
@buffer.size
-
end
-
-
30
def read(length = nil, outbuf = nil)
-
486
return @buffer.read(length, outbuf) if @buffer
-
-
285
return if @closed
-
-
230
chunk = @body.read(length)
-
-
230
compressed_chunk = deflate(chunk)
-
-
230
return unless compressed_chunk
-
-
203
if outbuf
-
194
outbuf.replace(compressed_chunk)
-
else
-
9
compressed_chunk
-
end
-
end
-
-
30
def close
-
55
return if @closed
-
-
55
@buffer.close if @buffer
-
-
55
@body.close
-
-
55
@closed = true
-
end
-
-
30
def rewind
-
32
return unless @buffer
-
-
18
@buffer.rewind
-
end
-
-
30
private
-
-
# rubocop:disable Naming/MemoizedInstanceVariableName
-
30
def buffer_deflate!
-
370
return @buffer if defined?(@buffer)
-
-
96
buffer = Response::Buffer.new(
-
threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
-
)
-
96
IO.copy_stream(self, buffer)
-
-
96
buffer.rewind if buffer.respond_to?(:rewind)
-
-
96
@buffer = buffer
-
end
-
# rubocop:enable Naming/MemoizedInstanceVariableName
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
30
module HTTPX
-
30
module Utils
-
30
using URIExtensions
-
-
30
TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
-
30
VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
-
30
FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
-
30
FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
-
-
30
module_function
-
-
30
def now
-
53969
Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
end
-
-
30
def elapsed_time(monotonic_timestamp)
-
19763
Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
-
end
-
-
# The value of this field can be either an HTTP-date or a number of
-
# seconds to delay after the response is received.
-
30
def parse_retry_after(retry_after)
-
# first: bet on it being an integer
-
72
Integer(retry_after)
-
rescue ArgumentError
-
# Then it's a datetime
-
18
time = Time.httpdate(retry_after)
-
18
time - Time.now
-
end
-
-
30
def get_filename(header, _prefix_regex = nil)
-
99
filename = nil
-
88
case header
-
when FILENAME_REGEX
-
63
filename = Regexp.last_match(1)
-
63
filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
-
when FILENAME_EXTENSION_REGEX
-
18
filename = Regexp.last_match(1)
-
18
encoding, _, filename = filename.split("'", 3)
-
end
-
-
99
return unless filename
-
-
153
filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
-
-
81
filename.scrub!
-
-
81
filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
-
-
81
filename.force_encoding ::Encoding.find(encoding) if encoding
-
-
81
filename
-
end
-
-
30
URIParser = URI::RFC2396_Parser.new
-
-
30
def to_uri(uri)
-
22604
return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
-
-
37
uri = URI(URIParser.escape(uri))
-
-
37
non_ascii_hostname = URIParser.unescape(uri.host)
-
-
37
non_ascii_hostname.force_encoding(Encoding::UTF_8)
-
-
37
idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
-
-
37
uri.host = idna_hostname
-
36
uri.non_ascii_hostname = non_ascii_hostname
-
36
uri
-
end
-
-
30
if defined?(Ractor) &&
-
# no ractor support for 3.0
-
RUBY_VERSION >= "3.1.0"
-
-
24
def in_ractor?
-
15400
Ractor.main != Ractor.current
-
end
-
else
-
6
def in_ractor?
-
11041
false
-
end
-
end
-
end
-
end
