All Files ( 42.07% covered at 8.05 hits/line )
872 files in total.
49913 relevant lines,
20996 lines covered and
28917 lines missed.
(
42.07%
)
-
# frozen_string_literal: true
-
-
1
module DatadogHelpers
-
1
DATADOG_VERSION = defined?(DDTrace) ? DDTrace::VERSION : Datadog::VERSION
-
1
ERROR_TAG = if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.8.0")
-
1
"error.message"
-
else
-
"error.msg"
-
end
-
-
1
private
-
-
1
def verify_instrumented_request(status, verb:, uri:, span: fetch_spans.first, service: datadog_service_name.to_s, error: nil)
-
25
if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
-
assert span.type == "http"
-
else
-
25
assert span.span_type == "http"
-
end
-
25
assert span.name == "#{datadog_service_name}.request"
-
25
assert span.service == service
-
-
25
assert span.get_tag("out.host") == uri.host
-
25
assert span.get_tag("out.port") == 80
-
25
assert span.get_tag("http.method") == verb
-
25
assert span.get_tag("http.url") == uri.path
-
-
25
if status && status >= 400
-
7
verify_http_error_span(span, status, error)
-
18
elsif error
-
2
verify_error_span(span)
-
else
-
16
assert span.status.zero?
-
16
assert span.get_tag("http.status_code") == status.to_s
-
# peer service
-
# assert span.get_tag("peer.service") == span.service
-
end
-
end
-
-
1
def verify_http_error_span(span, status, error)
-
5
assert span.get_tag("http.status_code") == status.to_s
-
5
assert span.get_tag("error.type") == error
-
5
assert !span.get_tag(ERROR_TAG).nil?
-
5
assert span.status == 1
-
end
-
-
1
def verify_error_span(span)
-
2
assert span.get_tag("error.type") == "HTTPX::NativeResolveError"
-
2
assert !span.get_tag(ERROR_TAG).nil?
-
2
assert span.status == 1
-
end
-
-
1
def verify_no_distributed_headers(request_headers)
-
1
assert !request_headers.key?("x-datadog-parent-id")
-
1
assert !request_headers.key?("x-datadog-trace-id")
-
1
assert !request_headers.key?("x-datadog-sampling-priority")
-
end
-
-
1
def verify_distributed_headers(request_headers, span: fetch_spans.first, sampling_priority: 1)
-
13
if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
-
assert request_headers["x-datadog-parent-id"] == span.id.to_s
-
else
-
13
assert request_headers["x-datadog-parent-id"] == span.span_id.to_s
-
end
-
13
assert request_headers["x-datadog-trace-id"] == trace_id(span)
-
13
assert request_headers["x-datadog-sampling-priority"] == sampling_priority.to_s
-
end
-
-
1
if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.17.0")
-
1
def trace_id(span)
-
13
Datadog::Tracing::Utils::TraceId.to_low_order(span.trace_id).to_s
-
end
-
else
-
def trace_id(span)
-
span.trace_id.to_s
-
end
-
end
-
-
1
def verify_analytics_headers(span, sample_rate: nil)
-
8
assert span.get_metric("_dd1.sr.eausr") == sample_rate
-
end
-
-
1
def set_datadog(options = {}, &blk)
-
22
Datadog.configure do |c|
-
22
c.tracing.instrument(datadog_service_name, options, &blk)
-
end
-
-
22
tracer # initialize tracer patches
-
end
-
-
1
def tracer
-
80
@tracer ||= begin
-
22
tr = Datadog::Tracing.send(:tracer)
-
22
def tr.write(trace)
-
25
@traces ||= []
-
25
@traces << trace
-
end
-
22
tr
-
end
-
end
-
-
1
def trace_with_sampling_priority(priority)
-
2
tracer.trace("foo.bar") do
-
2
tracer.active_trace.sampling_priority = priority
-
2
yield
-
end
-
end
-
-
# Returns spans and caches it (similar to +let(:spans)+).
-
1
def spans
-
@spans ||= fetch_spans
-
end
-
-
# Retrieves and sorts all spans in the current tracer instance.
-
# This method does not cache its results.
-
1
def fetch_spans
-
54
spans = (tracer.instance_variable_get(:@traces) || []).map(&:spans)
-
54
spans.flatten.sort! do |a, b|
-
10
if a.name == b.name
-
6
if a.resource == b.resource
-
4
if a.start_time == b.start_time
-
a.end_time <=> b.end_time
-
else
-
4
a.start_time <=> b.start_time
-
end
-
else
-
2
a.resource <=> b.resource
-
end
-
else
-
4
a.name <=> b.name
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
begin
-
# upcoming 2.0
-
1
require "datadog"
-
rescue LoadError
-
1
require "ddtrace"
-
end
-
-
1
require "test_helper"
-
1
require "support/http_helpers"
-
1
require "httpx/adapters/faraday"
-
1
require_relative "datadog_helpers"
-
-
1
DATADOG_VERSION = defined?(DDTrace) ? DDTrace::VERSION : Datadog::VERSION
-
-
1
class FaradayDatadogTest < Minitest::Test
-
1
include HTTPHelpers
-
1
include DatadogHelpers
-
1
include FaradayHelpers
-
-
1
def test_faraday_datadog_successful_get_request
-
1
set_datadog
-
1
uri = URI(build_uri("/status/200"))
-
-
1
response = faraday_connection.get(uri)
-
1
verify_status(response, 200)
-
-
1
assert !fetch_spans.empty?, "expected to have spans"
-
1
verify_instrumented_request(response.status, verb: "GET", uri: uri)
-
1
verify_distributed_headers(request_headers(response))
-
end
-
-
1
def test_faraday_datadog_successful_post_request
-
1
set_datadog
-
1
uri = URI(build_uri("/status/200"))
-
-
1
response = faraday_connection.post(uri, "bla")
-
1
verify_status(response, 200)
-
-
1
assert !fetch_spans.empty?, "expected to have spans"
-
1
verify_instrumented_request(response.status, verb: "POST", uri: uri)
-
1
verify_distributed_headers(request_headers(response))
-
end
-
-
1
def test_faraday_datadog_server_error_request
-
1
set_datadog
-
1
uri = URI(build_uri("/status/500"))
-
-
1
ex = assert_raises(Faraday::ServerError) do
-
1
faraday_connection.tap do |conn|
-
1
adapter_handler = conn.builder.handlers.last
-
1
conn.builder.insert_before adapter_handler, Faraday::Response::RaiseError
-
end.get(uri)
-
end
-
-
1
assert !fetch_spans.empty?, "expected to have spans"
-
1
verify_instrumented_request(ex.response[:status], verb: "GET", uri: uri, error: "Error 500")
-
-
1
verify_distributed_headers(request_headers(ex.response))
-
end
-
-
1
def test_faraday_datadog_client_error_request
-
1
set_datadog
-
1
uri = URI(build_uri("/status/404"))
-
-
1
ex = assert_raises(Faraday::ResourceNotFound) do
-
1
faraday_connection.tap do |conn|
-
1
adapter_handler = conn.builder.handlers.last
-
1
conn.builder.insert_before adapter_handler, Faraday::Response::RaiseError
-
end.get(uri)
-
end
-
-
1
assert !fetch_spans.empty?, "expected to have spans"
-
1
verify_instrumented_request(ex.response[:status], verb: "GET", uri: uri, error: "Error 404")
-
1
verify_distributed_headers(request_headers(ex.response))
-
end
-
-
1
def test_faraday_datadog_some_other_error
-
1
set_datadog
-
1
uri = URI("http://unexisting/")
-
-
2
assert_raises(HTTPX::NativeResolveError) { faraday_connection.get(uri) }
-
-
1
assert !fetch_spans.empty?, "expected to have spans"
-
1
verify_instrumented_request(nil, verb: "GET", uri: uri, error: "HTTPX::NativeResolveError")
-
end
-
-
1
def test_faraday_datadog_host_config
-
1
uri = URI(build_uri("/status/200"))
-
1
set_datadog(describe: /#{uri.host}/) do |http|
-
1
http.service_name = "httpbin"
-
1
http.split_by_domain = false
-
end
-
-
1
response = faraday_connection.get(uri)
-
1
verify_status(response, 200)
-
-
1
assert !fetch_spans.empty?, "expected to have spans"
-
1
verify_instrumented_request(response.status, service: "httpbin", verb: "GET", uri: uri)
-
1
verify_distributed_headers(request_headers(response))
-
end
-
-
1
def test_faraday_datadog_split_by_domain
-
1
uri = URI(build_uri("/status/200"))
-
1
set_datadog do |http|
-
1
http.split_by_domain = true
-
end
-
-
1
response = faraday_connection.get(uri)
-
1
verify_status(response, 200)
-
-
1
assert !fetch_spans.empty?, "expected to have spans"
-
1
verify_instrumented_request(response.status, service: uri.host, verb: "GET", uri: uri)
-
1
verify_distributed_headers(request_headers(response))
-
end
-
-
def test_faraday_datadog_distributed_headers_disabled
-
set_datadog(distributed_tracing: false)
-
uri = URI(build_uri("/status/200"))
-
-
sampling_priority = 10
-
response = trace_with_sampling_priority(sampling_priority) do
-
faraday_connection.get(uri)
-
end
-
verify_status(response, 200)
-
-
assert !fetch_spans.empty?, "expected to have spans"
-
span = fetch_spans.last
-
verify_instrumented_request(response.status, span: span, verb: "GET", uri: uri)
-
verify_no_distributed_headers(request_headers(response))
-
verify_analytics_headers(span)
-
1
end unless ENV.key?("CI") # TODO: https://github.com/DataDog/dd-trace-rb/issues/4308
-
-
def test_faraday_datadog_distributed_headers_sampling_priority
-
set_datadog
-
uri = URI(build_uri("/status/200"))
-
-
sampling_priority = 10
-
response = trace_with_sampling_priority(sampling_priority) do
-
faraday_connection.get(uri)
-
end
-
-
verify_status(response, 200)
-
-
assert !fetch_spans.empty?, "expected to have spans"
-
span = fetch_spans.last
-
verify_instrumented_request(response.status, span: span, verb: "GET", uri: uri)
-
verify_distributed_headers(request_headers(response), span: span, sampling_priority: sampling_priority)
-
verify_analytics_headers(span)
-
1
end unless ENV.key?("CI") # TODO: https://github.com/DataDog/dd-trace-rb/issues/4308
-
-
1
def test_faraday_datadog_analytics_enabled
-
1
set_datadog(analytics_enabled: true)
-
1
uri = URI(build_uri("/status/200"))
-
-
1
response = faraday_connection.get(uri)
-
1
verify_status(response, 200)
-
-
1
assert !fetch_spans.empty?, "expected to have spans"
-
1
span = fetch_spans.last
-
1
verify_instrumented_request(response.status, span: span, verb: "GET", uri: uri)
-
1
verify_analytics_headers(span, sample_rate: 1.0)
-
end
-
-
1
def test_faraday_datadog_analytics_sample_rate
-
1
set_datadog(analytics_enabled: true, analytics_sample_rate: 0.5)
-
1
uri = URI(build_uri("/status/200"))
-
-
1
response = faraday_connection.get(uri)
-
1
verify_status(response, 200)
-
-
1
assert !fetch_spans.empty?, "expected to have spans"
-
1
span = fetch_spans.last
-
1
verify_instrumented_request(response.status, span: span, verb: "GET", uri: uri)
-
1
verify_analytics_headers(span, sample_rate: 0.5)
-
end
-
-
1
private
-
-
1
def setup
-
9
super
-
9
Datadog.registry[:faraday].reset_configuration!
-
end
-
-
1
def teardown
-
9
super
-
9
Datadog.registry[:faraday].reset_configuration!
-
end
-
-
1
def datadog_service_name
-
25
:faraday
-
end
-
-
1
def origin(orig = httpbin)
-
8
"http://#{orig}"
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "logger"
-
1
require "stringio"
-
1
require "sentry-ruby"
-
1
require "test_helper"
-
1
require "support/http_helpers"
-
1
require "httpx/adapters/sentry"
-
-
1
class SentryTest < Minitest::Test
-
1
include HTTPHelpers
-
-
1
DUMMY_DSN = "http://12345:67890@sentry.localdomain/sentry/42"
-
-
1
def test_sentry_send_yes_pii
-
1
before_pii = Sentry.configuration.send_default_pii
-
begin
-
1
Sentry.configuration.send_default_pii = true
-
-
1
transaction = Sentry.start_transaction
-
1
Sentry.get_current_scope.set_span(transaction)
-
-
1
uri = build_uri("/get")
-
-
1
response = HTTPX.get(uri, params: { "foo" => "bar" })
-
-
1
verify_status(response, 200)
-
1
verify_spans(transaction, response, description: "GET #{uri}?foo=bar")
-
1
crumb = Sentry.get_current_scope.breadcrumbs.peek
-
1
assert crumb.category == "httpx"
-
1
assert crumb.data == { status: 200, method: "GET", url: "#{uri}?foo=bar" }
-
ensure
-
1
Sentry.configuration.send_default_pii = before_pii
-
end
-
end
-
-
1
def test_sentry_send_no_pii
-
1
before_pii = Sentry.configuration.send_default_pii
-
begin
-
1
Sentry.configuration.send_default_pii = false
-
-
1
transaction = Sentry.start_transaction
-
1
Sentry.get_current_scope.set_span(transaction)
-
-
1
uri = build_uri("/get")
-
-
1
response = HTTPX.get(uri, params: { "foo" => "bar" })
-
-
1
verify_status(response, 200)
-
1
verify_spans(transaction, response, description: "GET #{uri}")
-
1
crumb = Sentry.get_current_scope.breadcrumbs.peek
-
1
assert crumb.category == "httpx"
-
1
assert crumb.data == { status: 200, method: "GET", url: uri }
-
ensure
-
1
Sentry.configuration.send_default_pii = before_pii
-
end
-
end
-
-
1
def test_sentry_post_request
-
1
before_pii = Sentry.configuration.send_default_pii
-
begin
-
1
Sentry.configuration.send_default_pii = true
-
1
transaction = Sentry.start_transaction
-
1
Sentry.get_current_scope.set_span(transaction)
-
-
1
uri = build_uri("/post")
-
1
response = HTTPX.post(uri, form: { foo: "bar" })
-
1
verify_status(response, 200)
-
1
verify_spans(transaction, response, verb: "POST")
-
-
1
crumb = Sentry.get_current_scope.breadcrumbs.peek
-
1
assert crumb.category == "httpx"
-
1
assert crumb.data == { status: 200, method: "POST", url: uri, body: "foo=bar" }
-
ensure
-
1
Sentry.configuration.send_default_pii = before_pii
-
end
-
end
-
-
1
def test_sentry_multiple_requests
-
1
transaction = Sentry.start_transaction
-
1
Sentry.get_current_scope.set_span(transaction)
-
-
1
responses = HTTPX.get(build_uri("/status/200"), build_uri("/status/404"))
-
1
verify_status(responses[0], 200)
-
1
verify_status(responses[1], 404)
-
1
verify_spans(transaction, *responses)
-
end
-
-
1
def test_sentry_server_error_request
-
1
transaction = Sentry.start_transaction
-
1
Sentry.get_current_scope.set_span(transaction)
-
-
1
uri = URI("http://unexisting/")
-
-
1
response = HTTPX.get(uri)
-
-
1
verify_error_response(response, /name or service not known/)
-
1
assert response.is_a?(HTTPX::ErrorResponse), "response should contain errors"
-
1
verify_spans(transaction, response, verb: "GET")
-
1
crumb = Sentry.get_current_scope.breadcrumbs.peek
-
1
assert crumb.category == "httpx"
-
1
assert crumb.data == { error: "name or service not known", method: "GET", url: uri.to_s }
-
end
-
-
1
private
-
-
1
def verify_spans(transaction, *responses, verb: nil, description: nil)
-
5
assert transaction.span_recorder.spans.count == responses.size + 1
-
5
assert transaction.span_recorder.spans[0] == transaction
-
-
5
response_spans = transaction.span_recorder.spans[1..-1]
-
-
5
responses.each_with_index do |response, idx|
-
6
request_span = response_spans[idx]
-
6
assert request_span.op == "httpx.client"
-
6
assert !request_span.start_timestamp.nil?
-
6
assert !request_span.timestamp.nil?
-
6
assert request_span.start_timestamp != request_span.timestamp
-
6
assert request_span.description == (description || "#{verb || "GET"} #{response.uri}")
-
6
if response.is_a?(HTTPX::ErrorResponse)
-
1
assert request_span.data == { error: response.error.message }
-
else
-
5
assert request_span.data == { status: response.status }
-
end
-
end
-
end
-
-
1
def setup
-
5
super
-
-
5
mock_io = StringIO.new
-
5
mock_logger = Logger.new(mock_io)
-
-
5
Sentry.init do |config|
-
5
config.traces_sample_rate = 1.0
-
5
config.sdk_logger = mock_logger
-
5
config.dsn = DUMMY_DSN
-
5
config.transport.transport_class = Sentry::DummyTransport
-
5
config.background_worker_threads = 0
-
5
config.breadcrumbs_logger = [:http_logger]
-
5
config.enabled_patches << :httpx
-
# so the events will be sent synchronously for testing
-
end
-
end
-
-
1
def origin
-
5
"https://#{httpbin}"
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "webmock/minitest"
-
1
require "httpx/adapters/webmock"
-
1
require "test_helper"
-
1
require "support/http_helpers"
-
-
1
class WebmockTest < Minitest::Test
-
1
include HTTPHelpers
-
1
include FiberSchedulerTestHelpers
-
-
1
MOCK_URL_HTTP = "http://www.example.com"
-
1
MOCK_URL_HTTP_SAME_ORIGIN = "http://www.example.com/other"
-
1
MOCK_URL_HTTP_OTHER_ORIGIN = "http://www.example2.com"
-
1
MOCK_URL_HTTP_EXCEPTION = "http://exception.example.com"
-
1
MOCK_URL_HTTP_TIMEOUT = "http://timeout.example.com"
-
1
MOCK_URL_HTTP_TIMEOUT_RETRIES = "http://timeout-x-times.example.com"
-
-
1
def setup
-
36
super
-
36
WebMock.enable!
-
36
WebMock.disable_net_connect!
-
36
@stub_http = stub_http_request(:any, MOCK_URL_HTTP)
-
36
@stub_http_same_origin = stub_http_request(:any, MOCK_URL_HTTP_SAME_ORIGIN)
-
36
@stub_http_other_origin = stub_http_request(:any, MOCK_URL_HTTP_OTHER_ORIGIN)
-
-
36
@exception_class = Class.new(StandardError)
-
36
@stub_exception = stub_http_request(:any, MOCK_URL_HTTP_EXCEPTION).to_raise(@exception_class.new("exception message"))
-
36
@stub_timeout = stub_http_request(:any, MOCK_URL_HTTP_TIMEOUT).to_timeout
-
36
@stub_timeout_retries = stub_http_request(:any, MOCK_URL_HTTP_TIMEOUT_RETRIES).to_timeout.times(2).then.to_return(body: "body")
-
end
-
-
1
def teardown
-
36
super
-
36
WebMock.reset!
-
36
WebMock.allow_net_connect!
-
36
WebMock.disable!
-
end
-
-
1
def test_assert_requested_with_stub_and_block_raises_error
-
1
assert_raises ArgumentError do
-
1
assert_requested(@stub_http) {}
-
end
-
end
-
-
1
def test_assert_not_requested_with_stub_and_block_raises_error
-
1
assert_raises ArgumentError do
-
1
assert_not_requested(@stub_http) {}
-
end
-
end
-
-
1
def test_to_raise
-
1
response = http_request(:get, MOCK_URL_HTTP_EXCEPTION)
-
1
assert_requested(@stub_exception)
-
1
assert_equal(@exception_class.new("exception message"), response.error)
-
end
-
-
1
def test_response_not_decoded
-
1
request = stub_request(:get, MOCK_URL_HTTP).to_return(body: "body", headers: { content_encoding: "gzip" })
-
1
response = HTTPX.get(MOCK_URL_HTTP)
-
-
1
assert !response.body.empty?
-
1
assert_equal("body", response.body.to_s)
-
1
assert_requested(request)
-
1
assert response.mocked?
-
end
-
-
1
def test_to_timeout
-
1
response = http_request(:get, MOCK_URL_HTTP_TIMEOUT)
-
1
assert_requested(@stub_timeout)
-
1
assert_equal(HTTPX::TimeoutError.new(1, "Timed out"), response.error)
-
end
-
-
1
def test_to_timeout_with_retries
-
1
response = HTTPX.plugin(:retries, max_retries: 3).get(MOCK_URL_HTTP_TIMEOUT_RETRIES)
-
1
assert_requested(@stub_timeout_retries, times: 3)
-
1
assert !response.is_a?(HTTPX::ErrorResponse)
-
1
assert_equal("body", response.to_s)
-
1
assert response.mocked?
-
end
-
-
1
def test_error_on_non_stubbed_request
-
1
assert_raise_with_message(WebMock::NetConnectNotAllowedError, Regexp.new(
-
"Real HTTP connections are disabled. " \
-
"Unregistered request: GET http://www.example.net/ with headers"
-
)) do
-
1
http_request(:get, "http://www.example.net/")
-
end
-
end
-
-
1
def test_verification_that_expected_request_occured
-
1
http_request(:get, "#{MOCK_URL_HTTP}/")
-
1
assert_requested(:get, MOCK_URL_HTTP, times: 1)
-
1
assert_requested(:get, MOCK_URL_HTTP)
-
end
-
-
1
def test_verification_that_expected_stub_occured
-
1
http_request(:get, "#{MOCK_URL_HTTP}/")
-
1
assert_requested(@stub_http, times: 1)
-
1
assert_requested(@stub_http)
-
end
-
-
1
def test_multi_same_url
-
1
http_request(:get, "#{MOCK_URL_HTTP}/", "#{MOCK_URL_HTTP}/")
-
1
assert_requested(@stub_http, times: 2)
-
end
-
-
1
def test_multi_same_origin
-
1
http_request(:get, "#{MOCK_URL_HTTP}/", MOCK_URL_HTTP_SAME_ORIGIN)
-
1
assert_requested(@stub_http)
-
1
assert_requested(@stub_http_same_origin)
-
end
-
-
1
def test_multi_other_origin
-
1
http_request(:get, "#{MOCK_URL_HTTP}/", "#{MOCK_URL_HTTP_OTHER_ORIGIN}/")
-
1
assert_requested(@stub_http)
-
1
assert_requested(@stub_http_other_origin)
-
end
-
-
1
if Fiber.respond_to?(:set_scheduler) && RUBY_VERSION >= "3.1.0"
-
1
def test_multi_fiber_same_url
-
1
http = HTTPX.plugin(:fiber_concurrency)
-
-
1
with_test_fiber_scheduler do
-
1
2.times do
-
2
Fiber.schedule do
-
2
http.get("#{MOCK_URL_HTTP}/")
-
end
-
end
-
1
assert_requested(@stub_http, times: 2)
-
end
-
end
-
-
1
def test_multi_fiber_same_origin
-
1
http = HTTPX.plugin(:fiber_concurrency)
-
-
1
with_test_fiber_scheduler do
-
1
Fiber.schedule do
-
1
http.get("#{MOCK_URL_HTTP}/")
-
end
-
1
Fiber.schedule do
-
1
http.get(MOCK_URL_HTTP_SAME_ORIGIN)
-
end
-
1
assert_requested(@stub_http)
-
1
assert_requested(@stub_http_same_origin)
-
end
-
end
-
-
1
def test_multi_fiber_other_origin
-
1
http = HTTPX.plugin(:fiber_concurrency)
-
-
1
with_test_fiber_scheduler do
-
1
Fiber.schedule do
-
1
http.get("#{MOCK_URL_HTTP}/")
-
end
-
1
Fiber.schedule do
-
1
http.get(MOCK_URL_HTTP_OTHER_ORIGIN)
-
end
-
1
assert_requested(@stub_http)
-
1
assert_requested(@stub_http_other_origin)
-
end
-
end
-
end
-
-
1
def test_verification_that_expected_request_didnt_occur
-
1
expected_message = "The request GET #{MOCK_URL_HTTP}/ was expected to execute 1 time but it executed 0 times" \
-
"\n\nThe following requests were made:\n\nNo requests were made.\n" \
-
"============================================================"
-
1
assert_raise_with_message(Minitest::Assertion, expected_message) do
-
1
assert_requested(:get, MOCK_URL_HTTP)
-
end
-
end
-
-
1
def test_verification_that_expected_stub_didnt_occur
-
1
expected_message = "The request ANY #{MOCK_URL_HTTP}/ was expected to execute 1 time but it executed 0 times" \
-
"\n\nThe following requests were made:\n\nNo requests were made.\n" \
-
"============================================================"
-
1
assert_raise_with_message(Minitest::Assertion, expected_message) do
-
1
assert_requested(@stub_http)
-
end
-
end
-
-
1
def test_verification_that_expected_request_occured_with_body_and_headers
-
1
http_request(:get, "#{MOCK_URL_HTTP}/",
-
body: "abc", headers: { "A" => "a" })
-
1
assert_requested(:get, MOCK_URL_HTTP,
-
body: "abc", headers: { "A" => "a" })
-
end
-
-
1
def test_verification_that_expected_request_occured_with_query_params
-
1
stub_request(:any, MOCK_URL_HTTP).with(query: hash_including("a" => %w[b c]))
-
1
http_request(:get, "#{MOCK_URL_HTTP}/?a[]=b&a[]=c&x=1")
-
1
assert_requested(:get, MOCK_URL_HTTP,
-
query: hash_including("a" => %w[b c]))
-
end
-
-
1
def test_verification_that_requests_with_query_parameters_correctly_called
-
1
stub_request(:get, MOCK_URL_HTTP).to_return(body: "1")
-
1
stub_request(:get, "#{MOCK_URL_HTTP}/?a[]=b&a[]=c").to_return(body: "2")
-
1
stub_request(:get, "#{MOCK_URL_HTTP}/?test=value").to_return(body: "3")
-
-
1
response_1 = http_request(:get, MOCK_URL_HTTP)
-
1
response_2 = http_request(:get, MOCK_URL_HTTP, params: { "a" => %w[b c] })
-
1
response_3 = http_request(:get, MOCK_URL_HTTP, params: { test: "value" })
-
-
1
assert response_1.mocked?
-
1
assert response_2.mocked?
-
1
assert response_3.mocked?
-
-
1
assert_equal "1", response_1.body.to_s
-
1
assert_equal "2", response_2.body.to_s
-
1
assert_equal "3", response_3.body.to_s
-
-
1
assert_requested(:get, MOCK_URL_HTTP)
-
1
assert_requested(:get, "#{MOCK_URL_HTTP}/?a[]=b&a[]=c")
-
1
assert_requested(:get, "#{MOCK_URL_HTTP}/?test=value")
-
end
-
-
1
def test_verification_that_expected_request_not_occured_with_query_params
-
1
stub_request(:any, MOCK_URL_HTTP).with(query: hash_including(a: %w[b c]))
-
1
stub_request(:any, MOCK_URL_HTTP).with(query: hash_excluding(a: %w[b c]))
-
1
http_request(:get, "#{MOCK_URL_HTTP}/?a[]=b&a[]=c&x=1")
-
1
assert_not_requested(:get, MOCK_URL_HTTP, query: hash_excluding("a" => %w[b c]))
-
end
-
-
1
def test_verification_that_expected_request_occured_with_excluding_query_params
-
1
stub_request(:any, MOCK_URL_HTTP).with(query: hash_excluding("a" => %w[b c]))
-
1
http_request(:get, "#{MOCK_URL_HTTP}/?a[]=x&a[]=y&x=1")
-
1
assert_requested(:get, MOCK_URL_HTTP, query: hash_excluding("a" => %w[b c]))
-
end
-
-
1
def test_verification_that_expected_request_with_hash_as_body
-
1
stub_request(:post, MOCK_URL_HTTP).with(body: { foo: "bar" })
-
1
http_request(:post, MOCK_URL_HTTP, form: { foo: "bar" })
-
1
assert_requested(:post, MOCK_URL_HTTP, body: { foo: "bar" })
-
end
-
-
1
def test_verification_that_expected_request_occured_with_form_file
-
1
file = File.new(fixture_file_path)
-
1
stub_request(:post, MOCK_URL_HTTP)
-
1
http_request(:post, MOCK_URL_HTTP, form: { file: file })
-
# TODO: webmock does not support matching multipart request body
-
1
assert_requested(:post, MOCK_URL_HTTP)
-
end
-
-
1
def test_verification_that_expected_request_occured_with_form_tempfile
-
1
stub_request(:post, MOCK_URL_HTTP)
-
1
Tempfile.open("tmp") do |file|
-
1
http_request(:post, MOCK_URL_HTTP, form: { file: file })
-
end
-
# TODO: webmock does not support matching multipart request body
-
1
assert_requested(:post, MOCK_URL_HTTP)
-
end
-
-
1
def test_verification_that_non_expected_request_didnt_occur
-
1
expected_message = Regexp.new(
-
"The request GET #{MOCK_URL_HTTP}/ was not expected to execute but it executed 1 time\n\n" \
-
"The following requests were made:\n\nGET #{MOCK_URL_HTTP}/ with headers .+ was made 1 time\n\n" \
-
"============================================================"
-
)
-
1
assert_raise_with_message(Minitest::Assertion, expected_message) do
-
1
http_request(:get, "http://www.example.com/")
-
1
assert_not_requested(:get, "http://www.example.com")
-
end
-
end
-
-
1
def test_refute_requested_alias
-
1
expected_message = Regexp.new(
-
"The request GET #{MOCK_URL_HTTP}/ was not expected to execute but it executed 1 time\n\n" \
-
"The following requests were made:\n\nGET #{MOCK_URL_HTTP}/ with headers .+ was made 1 time\n\n" \
-
"============================================================"
-
)
-
1
assert_raise_with_message(Minitest::Assertion, expected_message) do
-
1
http_request(:get, "#{MOCK_URL_HTTP}/")
-
1
refute_requested(:get, MOCK_URL_HTTP)
-
end
-
end
-
-
1
def test_verification_that_non_expected_stub_didnt_occur
-
1
expected_message = Regexp.new(
-
"The request ANY #{MOCK_URL_HTTP}/ was not expected to execute but it executed 1 time\n\n" \
-
"The following requests were made:\n\nGET #{MOCK_URL_HTTP}/ with headers .+ was made 1 time\n\n" \
-
"============================================================"
-
)
-
1
assert_raise_with_message(Minitest::Assertion, expected_message) do
-
1
http_request(:get, "#{MOCK_URL_HTTP}/")
-
1
assert_not_requested(@stub_http)
-
end
-
end
-
-
1
def test_webmock_allows_real_request
-
1
WebMock.allow_net_connect!
-
1
uri = build_uri("/get?foo=bar")
-
1
response = HTTPX.get(uri)
-
1
verify_status(response, 200)
-
1
verify_body_length(response)
-
1
assert_requested(:get, uri, query: { "foo" => "bar" })
-
1
assert !response.mocked?
-
end
-
-
1
def test_webmock_allows_real_request_with_body
-
1
WebMock.allow_net_connect!
-
1
uri = build_uri("/post")
-
1
response = HTTPX.post(uri, form: { foo: "bar" })
-
1
verify_status(response, 200)
-
1
verify_body_length(response)
-
1
assert_requested(:post, uri, headers: { "Content-Type" => "application/x-www-form-urlencoded" }, body: "foo=bar")
-
1
assert !response.mocked?
-
end
-
-
1
def test_webmock_allows_real_request_with_file_body
-
1
WebMock.allow_net_connect!
-
1
uri = build_uri("/post")
-
1
response = HTTPX.post(uri, form: { image: File.new(fixture_file_path) })
-
1
verify_status(response, 200)
-
1
verify_body_length(response)
-
1
body = json_body(response)
-
1
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
1
verify_uploaded_image(body, "image", "image/jpeg")
-
1
assert !response.mocked?
-
# TODO: webmock does not support matching multipart request body
-
# assert_requested(:post, uri, headers: { "Content-Type" => "multipart/form-data" }, form: { "image" => File.new(fixture_file_path) })
-
end
-
-
1
def test_webmock_mix_mock_and_real_request
-
1
WebMock.allow_net_connect!
-
-
1
@stub_http.to_return(status: 200)
-
-
# test webmock callback as well
-
1
responses = {}
-
1
WebMock.after_request do |request_signature, response|
-
2
responses[request_signature.uri.to_s] = response
-
end
-
-
# this one ain't stubbed
-
1
real_request_uri = build_uri("/get", "http://#{httpbin}")
-
1
http_request(:get, "#{MOCK_URL_HTTP}/", real_request_uri)
-
-
1
assert responses.size == 2, "received #{responses.size} instead (#{responses.keys})"
-
3
assert(responses.values.all? { |r| r.status.first == 200 })
-
ensure
-
1
WebMock.reset_callbacks
-
1
WebMock.disable_net_connect!
-
end
-
-
1
def test_webmock_disable_after_enable
-
1
WebMock.disable!
-
-
# WebMock is disabled so this will make a real http request
-
1
http_request(:get, "http://#{httpbin}")
-
-
# WebMock is disabled so it should not have registered the request
-
1
assert_not_requested(:get, "http://#{httpbin}")
-
end
-
-
1
def test_webmock_follow_redirects_with_stream_plugin_each
-
1
session = HTTPX.plugin(:follow_redirects).plugin(:stream)
-
1
redirect_url = "#{MOCK_URL_HTTP}/redirect"
-
1
initial_request = stub_request(:get, MOCK_URL_HTTP).to_return(status: 302, headers: { location: redirect_url }, body: "redirecting")
-
1
redirect_request = stub_request(:get, redirect_url).to_return(status: 200, body: "body")
-
-
1
response = session.get(MOCK_URL_HTTP, stream: true)
-
1
body = "".b
-
1
response.each do |chunk|
-
2
next if (300..399).cover?(response.status)
-
-
1
body << chunk
-
end
-
1
assert_equal("body", body)
-
1
assert_requested(initial_request)
-
1
assert_requested(redirect_request)
-
end
-
-
1
def test_webmock_with_stream_plugin_each
-
1
session = HTTPX.plugin(:stream)
-
1
request = stub_request(:get, MOCK_URL_HTTP).to_return(body: "body")
-
-
1
body = "".b
-
1
response = session.get(MOCK_URL_HTTP, stream: true)
-
1
response.each do |chunk|
-
1
next if (300..399).cover?(response.status)
-
-
1
body << chunk
-
end
-
-
1
assert_equal("body", body)
-
1
assert_requested(request)
-
end
-
-
1
def test_webmock_with_stream_plugin_each_line
-
1
session = HTTPX.plugin(:stream)
-
1
request = stub_request(:get, MOCK_URL_HTTP).to_return(body: "First line\nSecond line")
-
-
1
response = session.get(MOCK_URL_HTTP, stream: true)
-
1
assert_equal(["First line", "Second line"], response.each_line.to_a)
-
1
assert_requested(request)
-
end
-
-
1
private
-
-
1
def assert_raise_with_message(e, message, &block)
-
6
e = assert_raises(e, &block)
-
6
if message.is_a?(Regexp)
-
4
assert_match(message, e.message)
-
else
-
2
assert_equal(message, e.message)
-
end
-
end
-
-
1
def http_request(meth, *uris, **options)
-
23
HTTPX.__send__(meth, *uris, **options)
-
end
-
-
1
def scheme
-
3
"http://"
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "httpx/version"
-
-
# Top-Level Namespace
-
#
-
1
module HTTPX
-
1
EMPTY = [].freeze
-
1
EMPTY_HASH = {}.freeze
-
-
# All plugins should be stored under this module/namespace. Can register and load
-
# plugins.
-
#
-
1
module Plugins
-
1
@plugins = {}
-
1
@plugins_mutex = Thread::Mutex.new
-
-
# Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
-
# it from the load path under "httpx/plugins/" directory.
-
#
-
1
def self.load_plugin(name)
-
61
h = @plugins
-
61
m = @plugins_mutex
-
122
unless (plugin = m.synchronize { h[name] })
-
7
require "httpx/plugins/#{name}"
-
14
raise "Plugin #{name} hasn't been registered" unless (plugin = m.synchronize { h[name] })
-
end
-
61
plugin
-
end
-
-
# Registers a plugin (+mod+) in the central store indexed by +name+.
-
#
-
1
def self.register_plugin(name, mod)
-
8
h = @plugins
-
8
m = @plugins_mutex
-
16
m.synchronize { h[name] = mod }
-
end
-
end
-
end
-
-
1
require "httpx/extensions"
-
-
1
require "httpx/errors"
-
1
require "httpx/utils"
-
1
require "httpx/punycode"
-
1
require "httpx/domain_name"
-
1
require "httpx/altsvc"
-
1
require "httpx/callbacks"
-
1
require "httpx/loggable"
-
1
require "httpx/transcoder"
-
1
require "httpx/timers"
-
1
require "httpx/pool"
-
1
require "httpx/headers"
-
1
require "httpx/request"
-
1
require "httpx/response"
-
1
require "httpx/options"
-
1
require "httpx/chainable"
-
-
1
require "httpx/session"
-
1
require "httpx/session_extensions"
-
-
# load integrations when possible
-
-
1
require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog::Tracing)
-
1
require "httpx/adapters/sentry" if defined?(Sentry)
-
1
require "httpx/adapters/webmock" if defined?(WebMock)
-
# frozen_string_literal: true
-
-
1
require "datadog/tracing/contrib/integration"
-
1
require "datadog/tracing/contrib/configuration/settings"
-
1
require "datadog/tracing/contrib/patcher"
-
-
1
module Datadog::Tracing
-
1
module Contrib
-
1
module HTTPX
-
1
DATADOG_VERSION = defined?(::DDTrace) ? ::DDTrace::VERSION : ::Datadog::VERSION
-
-
1
METADATA_MODULE = Datadog::Tracing::Metadata
-
-
1
TYPE_OUTBOUND = Datadog::Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
-
-
1
TAG_BASE_SERVICE = if Gem::Version.new(DATADOG_VERSION::STRING) < Gem::Version.new("1.15.0")
-
"_dd.base_service"
-
1
elsif Gem::Version.new(DATADOG_VERSION::STRING) < Gem::Version.new("2.34.0")
-
1
Datadog::Tracing::Contrib::Ext::Metadata::TAG_BASE_SERVICE
-
else
-
Datadog::Tracing::Metadata::Ext::TAG_BASE_SERVICE
-
end
-
1
TAG_PEER_HOSTNAME = Datadog::Tracing::Metadata::Ext::TAG_PEER_HOSTNAME
-
1
TAG_PEER_SERVICE = Datadog::Tracing::Metadata::Ext::TAG_PEER_SERVICE
-
-
1
TAG_KIND = Datadog::Tracing::Metadata::Ext::TAG_KIND
-
1
TAG_CLIENT = Datadog::Tracing::Metadata::Ext::SpanKind::TAG_CLIENT
-
1
TAG_COMPONENT = Datadog::Tracing::Metadata::Ext::TAG_COMPONENT
-
1
TAG_OPERATION = Datadog::Tracing::Metadata::Ext::TAG_OPERATION
-
1
TAG_URL = Datadog::Tracing::Metadata::Ext::HTTP::TAG_URL
-
1
TAG_METHOD = Datadog::Tracing::Metadata::Ext::HTTP::TAG_METHOD
-
1
TAG_TARGET_HOST = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_HOST
-
1
TAG_TARGET_PORT = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_PORT
-
-
1
TAG_STATUS_CODE = Datadog::Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE
-
-
# HTTPX Datadog Plugin
-
#
-
# Enables tracing for httpx requests.
-
#
-
# A span will be created for each request transaction; the span is created lazily only when
-
# buffering a request, and it is fed the start time stored inside the tracer object.
-
#
-
1
module Plugin
-
1
module RequestTracer
-
1
extend Contrib::HttpAnnotationHelper
-
-
1
module_function
-
-
1
SPAN_REQUEST = "httpx.request"
-
-
1
def enabled?(request)
-
14
configuration(request).enabled
-
end
-
-
1
def start(request)
-
16
request.datadog_span = initialize_span(request, request.init_time)
-
end
-
-
1
def reset(request)
-
4
request.datadog_span = nil
-
end
-
-
1
def finish(request, response)
-
16
request.datadog_span ||= initialize_span(request, request.init_time) if request.init_time
-
-
16
finish_span(response, request.datadog_span)
-
end
-
-
1
def finish_span(response, span)
-
16
if response.is_a?(::HTTPX::ErrorResponse)
-
1
span.set_error(response.error)
-
else
-
15
span.set_tag(TAG_STATUS_CODE, response.status.to_s)
-
-
15
span.set_error(::HTTPX::HTTPError.new(response)) if response.status.between?(400, 599)
-
-
span.set_tags(
-
Datadog.configuration.tracing.header_tags.response_tags(response.headers.to_h)
-
15
) if Datadog.configuration.tracing.respond_to?(:header_tags)
-
end
-
-
16
span.finish
-
end
-
-
# return a span initialized with the +@request+ state.
-
1
def initialize_span(request, start_time)
-
17
verb = request.verb
-
17
uri = request.uri
-
-
17
config = configuration(request)
-
-
17
span = create_span(request, config, start_time)
-
-
17
span.resource = verb
-
-
# Tag original global service name if not used
-
17
span.set_tag(TAG_BASE_SERVICE, Datadog.configuration.service) if span.service != Datadog.configuration.service
-
-
17
span.set_tag(TAG_KIND, TAG_CLIENT)
-
-
17
span.set_tag(TAG_COMPONENT, "httpx")
-
17
span.set_tag(TAG_OPERATION, "request")
-
-
17
span.set_tag(TAG_URL, request.path)
-
17
span.set_tag(TAG_METHOD, verb)
-
-
17
span.set_tag(TAG_TARGET_HOST, uri.host)
-
17
span.set_tag(TAG_TARGET_PORT, uri.port)
-
-
17
span.set_tag(TAG_PEER_HOSTNAME, uri.host)
-
-
# Tag as an external peer service
-
17
if (peer_service = config[:peer_service])
-
span.set_tag(TAG_PEER_SERVICE, peer_service)
-
end
-
-
17
if config[:distributed_tracing]
-
16
propagate_trace_http(
-
Datadog::Tracing.active_trace,
-
request.headers
-
)
-
end
-
-
# Set analytics sample rate
-
17
if Contrib::Analytics.enabled?(config[:analytics_enabled])
-
2
Contrib::Analytics.set_sample_rate(span, config[:analytics_sample_rate])
-
end
-
-
span.set_tags(
-
Datadog.configuration.tracing.header_tags.request_tags(request.headers.to_h)
-
17
) if Datadog.configuration.tracing.respond_to?(:header_tags)
-
-
17
span
-
rescue StandardError => e
-
Datadog.logger.error("error preparing span for http request: #{e}")
-
Datadog.logger.error(e.backtrace)
-
end
-
-
1
def configuration(request)
-
31
Datadog.configuration.tracing[:httpx, request.uri.host]
-
end
-
-
1
if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
-
def propagate_trace_http(trace, headers)
-
Datadog::Tracing::Contrib::HTTP.inject(trace, headers)
-
end
-
-
def create_span(request, configuration, start_time)
-
Datadog::Tracing.trace(
-
SPAN_REQUEST,
-
service: service_name(request.uri.host, configuration),
-
type: TYPE_OUTBOUND,
-
start_time: start_time
-
)
-
end
-
else
-
1
def propagate_trace_http(trace, headers)
-
16
Datadog::Tracing::Propagation::HTTP.inject!(trace.to_digest, headers)
-
end
-
-
1
def create_span(request, configuration, start_time)
-
17
Datadog::Tracing.trace(
-
SPAN_REQUEST,
-
service: service_name(request.uri.host, configuration),
-
span_type: TYPE_OUTBOUND,
-
start_time: start_time
-
)
-
end
-
end
-
end
-
-
1
class << self
-
1
def load_dependencies(klass)
-
1
klass.plugin(:tracing)
-
end
-
-
1
def extra_options(options)
-
1
options.merge(tracer: RequestTracer)
-
end
-
end
-
-
1
module RequestMethods
-
1
attr_accessor :datadog_span
-
end
-
end
-
-
# patches httpx debug logs to include datadog correlation ids
-
1
module LoggablePatch
-
1
def log_identifiers
-
"#{super} #{Datadog::Tracing.log_correlation}"
-
end
-
end
-
-
1
module Configuration
-
# Default settings for httpx
-
#
-
1
class Settings < Datadog::Tracing::Contrib::Configuration::Settings
-
1
DEFAULT_ERROR_HANDLER = lambda do |response|
-
Datadog::Ext::HTTP::ERROR_RANGE.cover?(response.status)
-
end
-
-
1
option :service_name, default: "httpx"
-
1
option :distributed_tracing, default: true
-
1
option :split_by_domain, default: false
-
-
1
if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
-
1
option :enabled do |o|
-
1
o.type :bool
-
1
o.env "DD_TRACE_HTTPX_ENABLED"
-
1
o.default true
-
end
-
-
1
option :analytics_enabled do |o|
-
1
o.type :bool
-
1
o.env "DD_TRACE_HTTPX_ANALYTICS_ENABLED"
-
1
o.default false
-
end
-
-
1
option :analytics_sample_rate do |o|
-
1
o.type :float
-
1
o.env "DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE"
-
1
o.default 1.0
-
end
-
-
1
option :peer_service do |o|
-
1
o.type :string, nilable: true
-
1
o.env "DD_TRACE_HTTPX_PEER_SERVICE"
-
end
-
else
-
option :enabled do |o|
-
o.default { env_to_bool("DD_TRACE_HTTPX_ENABLED", true) }
-
o.lazy
-
end
-
-
option :analytics_enabled do |o|
-
o.default { env_to_bool(%w[DD_TRACE_HTTPX_ANALYTICS_ENABLED DD_HTTPX_ANALYTICS_ENABLED], false) }
-
o.lazy
-
end
-
-
option :analytics_sample_rate do |o|
-
o.default { env_to_float(%w[DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE DD_HTTPX_ANALYTICS_SAMPLE_RATE], 1.0) }
-
o.lazy
-
end
-
-
option :peer_service do |o|
-
o.default { env_to_string("DD_TRACE_HTTPX_PEER_SERVICE", nil) }
-
o.lazy
-
end
-
end
-
-
1
if defined?(Datadog::Tracing::Contrib::SpanAttributeSchema)
-
1
option :service_name do |o|
-
1
o.default do
-
11
Datadog::Tracing::Contrib::SpanAttributeSchema.fetch_service_name(
-
"DD_TRACE_HTTPX_SERVICE_NAME",
-
"httpx"
-
)
-
end
-
1
o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
-
end
-
else
-
option :service_name do |o|
-
o.default do
-
ENV.fetch("DD_TRACE_HTTPX_SERVICE_NAME", "httpx")
-
end
-
o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
-
end
-
end
-
-
1
option :distributed_tracing, default: true
-
-
1
if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.15.0")
-
1
option :error_handler do |o|
-
1
o.type :proc
-
1
o.default_proc(&DEFAULT_ERROR_HANDLER)
-
end
-
elsif Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
-
option :error_handler do |o|
-
o.type :proc
-
o.experimental_default_proc(&DEFAULT_ERROR_HANDLER)
-
end
-
else
-
option :error_handler, default: DEFAULT_ERROR_HANDLER
-
end
-
end
-
end
-
-
# Patcher enables patching of 'httpx' with datadog components.
-
#
-
1
module Patcher
-
1
include Datadog::Tracing::Contrib::Patcher
-
-
1
module_function
-
-
1
def target_version
-
2
Integration.version
-
end
-
-
# loads a session instannce with the datadog plugin, and replaces the
-
# base HTTPX::Session with the patched session class.
-
1
def patch
-
1
datadog_session = ::HTTPX.plugin(Plugin)
-
-
1
::HTTPX.send(:remove_const, :Session)
-
1
::HTTPX.send(:const_set, :Session, datadog_session.class)
-
-
1
::HTTPX::Loggable.singleton_class.prepend(LoggablePatch)
-
end
-
end
-
-
# Datadog Integration for HTTPX.
-
#
-
1
class Integration
-
1
include Contrib::Integration
-
-
1
MINIMUM_VERSION = Gem::Version.new("0.10.2")
-
-
1
register_as :httpx
-
-
1
def self.version
-
41
Gem.loaded_specs["httpx"] && Gem.loaded_specs["httpx"].version
-
end
-
-
1
def self.loaded?
-
13
defined?(::HTTPX::Request)
-
end
-
-
1
def self.compatible?
-
13
super && version >= MINIMUM_VERSION
-
end
-
-
1
def new_configuration
-
26
Configuration::Settings.new
-
end
-
-
1
def patcher
-
26
Patcher
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "delegate"
-
1
require "httpx"
-
1
require "faraday"
-
-
1
module Faraday
-
1
class Adapter
-
1
class HTTPX < Faraday::Adapter
-
1
def initialize(app = nil, opts = {}, &block)
-
9
@connection = @bind = nil
-
9
super
-
end
-
-
1
module RequestMixin
-
1
def build_connection(env)
-
9
return @connection if @connection
-
-
9
@connection = ::HTTPX.plugin(:persistent).plugin(ReasonPlugin)
-
9
@connection = @connection.with(@connection_options) unless @connection_options.empty?
-
9
connection_opts = options_from_env(env)
-
-
9
if (bind = env.request.bind)
-
@bind = TCPSocket.new(bind[:host], bind[:port])
-
connection_opts[:io] = @bind
-
end
-
9
@connection = @connection.with(connection_opts)
-
-
9
if (proxy = env.request.proxy)
-
proxy_options = { uri: proxy.uri }
-
proxy_options[:username] = proxy.user if proxy.user
-
proxy_options[:password] = proxy.password if proxy.password
-
-
@connection = @connection.plugin(:proxy).with(proxy: proxy_options)
-
end
-
9
@connection = @connection.plugin(OnDataPlugin) if env.request.stream_response?
-
-
9
@connection = @config_block.call(@connection) || @connection if @config_block
-
9
@connection
-
end
-
-
1
def close
-
9
@connection.close if @connection
-
9
@bind.close if @bind
-
end
-
-
1
private
-
-
1
def connect(env, &blk)
-
9
connection(env, &blk)
-
rescue ::HTTPX::TLSError => e
-
raise Faraday::SSLError, e
-
rescue Errno::ECONNABORTED,
-
Errno::ECONNREFUSED,
-
Errno::ECONNRESET,
-
Errno::EHOSTUNREACH,
-
Errno::EINVAL,
-
Errno::ENETUNREACH,
-
Errno::EPIPE,
-
::HTTPX::ConnectionError => e
-
raise Faraday::ConnectionFailed, e
-
rescue ::HTTPX::TimeoutError => e
-
raise Faraday::TimeoutError, e
-
end
-
-
1
def build_request(env)
-
9
meth = env[:method]
-
-
request_options = {
-
9
headers: env.request_headers,
-
body: env.body,
-
**options_from_env(env),
-
}
-
9
[meth.to_s.upcase, env.url, request_options]
-
end
-
-
1
def options_from_env(env)
-
18
timeout_options = {}
-
18
req_opts = env.request
-
18
if (sec = request_timeout(:read, req_opts))
-
timeout_options[:read_timeout] = sec
-
end
-
-
18
if (sec = request_timeout(:write, req_opts))
-
timeout_options[:write_timeout] = sec
-
end
-
-
18
if (sec = request_timeout(:open, req_opts))
-
timeout_options[:connect_timeout] = sec
-
end
-
-
{
-
18
ssl: ssl_options_from_env(env),
-
timeout: timeout_options,
-
}
-
end
-
-
1
if defined?(::OpenSSL)
-
1
def ssl_options_from_env(env)
-
18
ssl_options = {}
-
-
18
unless env.ssl.verify.nil?
-
ssl_options[:verify_mode] = env.ssl.verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
-
end
-
-
18
ssl_options[:ca_file] = env.ssl.ca_file if env.ssl.ca_file
-
18
ssl_options[:ca_path] = env.ssl.ca_path if env.ssl.ca_path
-
18
ssl_options[:cert_store] = env.ssl.cert_store if env.ssl.cert_store
-
18
ssl_options[:cert] = env.ssl.client_cert if env.ssl.client_cert
-
18
ssl_options[:key] = env.ssl.client_key if env.ssl.client_key
-
18
ssl_options[:ssl_version] = env.ssl.version if env.ssl.version
-
18
ssl_options[:verify_depth] = env.ssl.verify_depth if env.ssl.verify_depth
-
18
ssl_options[:min_version] = env.ssl.min_version if env.ssl.min_version
-
18
ssl_options[:max_version] = env.ssl.max_version if env.ssl.max_version
-
18
ssl_options
-
end
-
else
-
# simplecov:disable
-
def ssl_options_from_env(*)
-
{}
-
end
-
# simplecov:enable
-
end
-
end
-
-
1
include RequestMixin
-
-
1
module OnDataPlugin
-
1
module RequestMethods
-
1
attr_writer :response_on_data
-
-
1
def response=(response)
-
super
-
-
return unless @response
-
-
return if @response.is_a?(::HTTPX::ErrorResponse)
-
-
@response.body.on_data = @response_on_data
-
end
-
end
-
-
1
module ResponseBodyMethods
-
1
attr_writer :on_data
-
-
1
def write(chunk)
-
return super unless @on_data
-
-
@on_data.call(chunk, chunk.bytesize)
-
end
-
end
-
end
-
-
1
module ReasonPlugin
-
1
def self.load_dependencies(*)
-
9
require "net/http/status"
-
end
-
-
1
module ResponseMethods
-
1
def reason
-
8
Net::HTTP::STATUS_CODES.fetch(@status, "Non-Standard status code")
-
end
-
end
-
end
-
-
1
class ParallelManager
-
1
class ResponseHandler < SimpleDelegator
-
1
attr_reader :env
-
-
1
def initialize(env)
-
@env = env
-
super
-
end
-
-
1
def on_response(&blk)
-
if blk
-
@on_response = ->(response) do
-
blk.call(response)
-
end
-
self
-
else
-
@on_response
-
end
-
end
-
end
-
-
1
include RequestMixin
-
-
1
def initialize(options)
-
@handlers = []
-
@connection_options = options
-
end
-
-
1
def enqueue(request)
-
handler = ResponseHandler.new(request)
-
@handlers << handler
-
handler
-
end
-
-
1
def run
-
return unless @handlers.last
-
-
env = @handlers.last.env
-
-
connect(env) do |session|
-
requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
-
-
if env.request.stream_response?
-
requests.each do |request|
-
request.response_on_data = env.request.on_data
-
end
-
end
-
-
responses = session.request(*requests)
-
Array(responses).each_with_index do |response, index|
-
handler = @handlers[index]
-
handler.on_response.call(response)
-
end
-
end
-
end
-
-
1
private
-
-
# from Faraday::Adapter#connection
-
1
def connection(env)
-
conn = build_connection(env)
-
return conn unless block_given?
-
-
yield conn
-
end
-
-
# from Faraday::Adapter#request_timeout
-
1
def request_timeout(type, options)
-
key = Faraday::Adapter::TIMEOUT_KEYS[type]
-
options[key] || options[:timeout]
-
end
-
end
-
-
1
self.supports_parallel = true
-
-
1
class << self
-
1
def setup_parallel_manager(options = {})
-
ParallelManager.new(options)
-
end
-
end
-
-
1
def call(env)
-
9
super
-
9
if parallel?(env)
-
handler = env[:parallel_manager].enqueue(env)
-
handler.on_response do |response|
-
if response.is_a?(::HTTPX::Response)
-
save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
-
response_headers.merge!(response.headers)
-
end
-
else
-
env[:error] = response.error
-
save_response(env, 0, "", {}, nil)
-
end
-
end
-
return handler
-
end
-
-
9
response = connect_and_request(env)
-
8
save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
-
8
response_headers.merge!(response.headers)
-
end
-
8
@app.call(env)
-
end
-
-
1
private
-
-
1
def connect_and_request(env)
-
9
connect(env) do |session|
-
9
request = session.build_request(*build_request(env))
-
-
9
request.response_on_data = env.request.on_data if env.request.stream_response?
-
-
9
response = session.request(request)
-
# do not call #raise_for_status for HTTP 4xx or 5xx, as faraday has a middleware for that.
-
9
response.raise_for_status unless response.is_a?(::HTTPX::Response)
-
8
response
-
end
-
end
-
-
1
def parallel?(env)
-
9
env[:parallel_manager]
-
end
-
end
-
-
1
register_middleware httpx: HTTPX
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "sentry-ruby"
-
-
1
module HTTPX::Plugins
-
1
module Sentry
-
1
module Tracer
-
1
module_function
-
-
1
def call(request)
-
20
sentry_span = start_sentry_span
-
-
20
return unless sentry_span
-
-
20
set_sentry_trace_header(request, sentry_span)
-
-
20
request.on(:response, &method(:finish_sentry_span).curry(3)[sentry_span, request])
-
end
-
-
1
def start_sentry_span
-
20
return unless ::Sentry.initialized? && (span = ::Sentry.get_current_scope.get_span)
-
20
return if span.sampled == false
-
-
20
span.start_child(op: "httpx.client", start_timestamp: ::Sentry.utc_now.to_f)
-
end
-
-
1
def set_sentry_trace_header(request, sentry_span)
-
20
return unless sentry_span
-
-
20
config = ::Sentry.configuration
-
20
url = request.uri.to_s
-
-
40
return unless config.propagate_traces && config.trace_propagation_targets.any? { |target| url.match?(target) }
-
-
20
trace = sentry_span.to_sentry_trace
-
20
request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
-
end
-
-
1
def finish_sentry_span(span, request, response)
-
22
return unless ::Sentry.initialized?
-
-
22
record_sentry_breadcrumb(request, response)
-
22
record_sentry_span(request, response, span)
-
end
-
-
1
def record_sentry_breadcrumb(req, res)
-
22
return unless ::Sentry.configuration.breadcrumbs_logger.include?(:http_logger)
-
-
22
request_info = extract_request_info(req)
-
-
22
data = if res.is_a?(HTTPX::ErrorResponse)
-
2
{ error: res.error.message, **request_info }
-
else
-
20
{ status: res.status, **request_info }
-
end
-
-
22
crumb = ::Sentry::Breadcrumb.new(
-
level: :info,
-
category: "httpx",
-
type: :info,
-
data: data
-
)
-
22
::Sentry.add_breadcrumb(crumb)
-
end
-
-
1
def record_sentry_span(req, res, sentry_span)
-
22
return unless sentry_span
-
-
22
request_info = extract_request_info(req)
-
22
sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
-
22
if res.is_a?(HTTPX::ErrorResponse)
-
2
sentry_span.set_data(:error, res.error.message)
-
else
-
20
sentry_span.set_data(:status, res.status)
-
end
-
22
sentry_span.set_timestamp(::Sentry.utc_now.to_f)
-
end
-
-
1
def extract_request_info(req)
-
44
uri = req.uri
-
-
result = {
-
44
method: req.verb,
-
}
-
-
44
if ::Sentry.configuration.send_default_pii
-
4
uri += "?#{req.query}" unless req.query.empty?
-
4
result[:body] = req.body.to_s unless req.body.empty? || req.body.unbounded_body?
-
end
-
-
44
result[:url] = uri.to_s
-
-
44
result
-
end
-
end
-
-
1
module RequestMethods
-
1
def __sentry_enable_trace!
-
22
return if @__sentry_enable_trace
-
-
20
Tracer.call(self)
-
20
@__sentry_enable_trace = true
-
end
-
end
-
-
1
module ConnectionMethods
-
1
def send(request)
-
22
request.__sentry_enable_trace!
-
-
22
super
-
end
-
end
-
end
-
end
-
-
1
Sentry.register_patch(:httpx) do
-
5
sentry_session = HTTPX.plugin(HTTPX::Plugins::Sentry)
-
-
5
HTTPX.send(:remove_const, :Session)
-
5
HTTPX.send(:const_set, :Session, sentry_session.class)
-
end
-
# frozen_string_literal: true
-
-
1
module WebMock
-
1
module HttpLibAdapters
-
1
require "net/http/status"
-
1
HTTP_REASONS = Net::HTTP::STATUS_CODES
-
-
#
-
# HTTPX plugin for webmock.
-
#
-
# Requests are "hijacked" at the session, before they're distributed to a connection.
-
#
-
1
module Plugin
-
1
class << self
-
1
def build_webmock_request_signature(request)
-
43
uri = WebMock::Util::URI.heuristic_parse(request.uri)
-
43
uri.query = request.query
-
43
uri.path = uri.normalized_path.gsub("[^:]//", "/")
-
-
43
WebMock::RequestSignature.new(
-
request.verb.downcase.to_sym,
-
uri.to_s,
-
body: request.body.to_s,
-
headers: request.headers.to_h
-
)
-
end
-
-
1
def build_webmock_response(_request, response)
-
1
webmock_response = WebMock::Response.new
-
1
webmock_response.status = [response.status, HTTP_REASONS[response.status]]
-
1
webmock_response.body = response.body.to_s
-
1
webmock_response.headers = response.headers.to_h
-
1
webmock_response
-
end
-
-
1
def build_from_webmock_response(request, webmock_response)
-
38
return build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
-
-
35
return build_error_response(request, webmock_response.exception) if webmock_response.exception
-
-
34
request
-
.options
-
.response_class
-
.new(
-
request,
-
webmock_response.status[0],
-
"2.0",
-
webmock_response.headers
-
).tap(&:mock!)
-
end
-
-
1
def build_error_response(request, exception)
-
4
HTTPX::ErrorResponse.new(request, exception)
-
end
-
end
-
-
1
module InstanceMethods
-
1
private
-
-
1
def do_init_connection(connection, selector)
-
38
super
-
-
38
connection.once(:unmock_connection) do
-
4
next unless connection.current_session == self
-
-
4
unless connection.addresses?
-
# reset Happy Eyeballs, fail early
-
4
connection.sibling = nil
-
-
4
deselect_connection(connection, selector)
-
end
-
4
resolve_connection(connection, selector)
-
end
-
end
-
end
-
-
1
module ResponseMethods
-
1
def initialize(*)
-
38
super
-
38
@mocked = false
-
end
-
-
1
def mock!
-
34
@mocked = true
-
34
@body.mock!
-
end
-
-
1
def mocked?
-
8
@mocked
-
end
-
end
-
-
1
module ResponseBodyMethods
-
1
def mock!
-
34
@inflaters = nil
-
end
-
end
-
-
1
module ConnectionMethods
-
1
def initialize(*)
-
38
super
-
38
@mocked = true
-
end
-
-
1
def open?
-
42
return true if @mocked
-
-
4
super
-
end
-
-
1
def interests
-
48
return if @mocked
-
-
46
super
-
end
-
-
1
def terminate
-
34
force_reset
-
end
-
-
1
def send(request)
-
43
request_signature = Plugin.build_webmock_request_signature(request)
-
43
WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
-
-
43
if (mock_response = WebMock::StubRegistry.instance.response_for_request(request_signature))
-
38
response = Plugin.build_from_webmock_response(request, mock_response)
-
38
WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, request_signature, mock_response)
-
38
log { "mocking #{request.uri} with #{mock_response.inspect}" }
-
38
request.transition(:headers)
-
38
request.transition(:body)
-
38
request.transition(:trailers)
-
38
request.transition(:done)
-
38
response.finish!
-
38
request.response = response
-
38
request.emit_response(response)
-
38
request_signature.headers = request.headers.to_h
-
-
38
response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)
-
5
elsif WebMock.net_connect_allowed?(request_signature.uri)
-
4
if WebMock::CallbackRegistry.any_callbacks?
-
1
request.on(:response) do |resp|
-
1
unless resp.is_a?(HTTPX::ErrorResponse)
-
1
webmock_response = Plugin.build_webmock_response(request, resp)
-
1
WebMock::CallbackRegistry.invoke_callbacks(
-
{ lib: :httpx, real_request: true }, request_signature,
-
webmock_response
-
)
-
end
-
end
-
end
-
4
@mocked = false
-
4
emit(:unmock_connection, self)
-
4
super
-
else
-
1
raise WebMock::NetConnectNotAllowedError, request_signature
-
end
-
end
-
-
1
private
-
-
1
def connect
-
8
super unless @mocked
-
end
-
end
-
end
-
-
1
class HttpxAdapter < HttpLibAdapter
-
1
adapter_for :httpx
-
-
1
class << self
-
1
def enable!
-
73
@original_session ||= HTTPX::Session
-
-
73
webmock_session = HTTPX.plugin(Plugin)
-
-
73
HTTPX.send(:remove_const, :Session)
-
73
HTTPX.send(:const_set, :Session, webmock_session.class)
-
end
-
-
1
def disable!
-
73
return unless @original_session
-
-
72
HTTPX.send(:remove_const, :Session)
-
72
HTTPX.send(:const_set, :Session, @original_session)
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "strscan"
-
-
1
module HTTPX
-
1
module AltSvc
-
# makes connections able to accept requests destined to primary service.
-
1
module ConnectionMixin
-
1
using URIExtensions
-
-
1
H2_ALTSVC_SCHEMES = %w[https h2].freeze
-
-
1
ALTSVC_IGNORE_IVARS = %i[@ssl].freeze
-
-
1
def send(request)
-
request.headers["alt-used"] = @origin.authority if @parser && !@write_buffer.full? && match_altsvcs?(request.uri)
-
-
super
-
end
-
-
1
def match?(uri, options)
-
return false if !used? && (@state == :closing || @state == :closed)
-
-
match_altsvcs?(uri) && match_altsvc_options?(uri, options)
-
end
-
-
1
private
-
-
# checks if this is connection is an alternative service of
-
# +uri+
-
1
def match_altsvcs?(uri)
-
@origins.any? { |origin| altsvc_match?(uri, origin) } ||
-
AltSvc.cached_altsvc(@origin).any? do |altsvc|
-
origin = altsvc["origin"]
-
altsvc_match?(origin, uri.origin)
-
end
-
end
-
-
1
def match_altsvc_options?(uri, options)
-
return @options.connection_options_match?(options) unless @options.ssl.all? do |k, v|
-
v == (k == :hostname ? uri.host : options.ssl[k])
-
end
-
-
@options.connection_options_match?(options, ALTSVC_IGNORE_IVARS)
-
end
-
-
1
def altsvc_match?(uri, other_uri)
-
other_uri = URI(other_uri) #: http_uri
-
-
uri.origin == other_uri.origin || begin
-
case uri.scheme
-
when "h2"
-
H2_ALTSVC_SCHEMES.include?(other_uri.scheme) &&
-
uri.host == other_uri.host &&
-
uri.port == other_uri.port
-
else
-
false
-
end
-
end
-
end
-
end
-
-
1
@altsvc_mutex = Thread::Mutex.new
-
1
@altsvcs = Hash.new { |h, k| h[k] = [] }
-
-
1
module_function
-
-
1
def cached_altsvc(origin)
-
now = Utils.now
-
@altsvc_mutex.synchronize do
-
lookup(origin, now)
-
end
-
end
-
-
1
def cached_altsvc_set(origin, entry)
-
now = Utils.now
-
@altsvc_mutex.synchronize do
-
return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
-
-
entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
-
@altsvcs[origin] << entry
-
entry
-
end
-
end
-
-
1
def lookup(origin, ttl)
-
return [] unless @altsvcs.key?(origin)
-
-
@altsvcs[origin] = @altsvcs[origin].select do |entry|
-
!entry.key?("TTL") || entry["TTL"] > ttl
-
end
-
@altsvcs[origin].reject { |entry| entry["noop"] }
-
end
-
-
1
def emit(request, response)
-
33
return unless response.respond_to?(:headers)
-
# Alt-Svc
-
33
return unless response.headers.key?("alt-svc")
-
-
origin = request.origin
-
host = request.uri.host
-
-
altsvc = response.headers["alt-svc"]
-
-
# https://datatracker.ietf.org/doc/html/rfc7838#section-3
-
# A field value containing the special value "clear" indicates that the
-
# origin requests all alternatives for that origin to be invalidated
-
# (including those specified in the same response, in case of an
-
# invalid reply containing both "clear" and alternative services).
-
if altsvc == "clear"
-
@altsvc_mutex.synchronize do
-
@altsvcs[origin].clear
-
end
-
-
return
-
end
-
-
parse(altsvc) do |alt_origin, alt_params|
-
alt_origin.host ||= host
-
yield(alt_origin, origin, alt_params)
-
end
-
end
-
-
1
def parse(altsvc)
-
return enum_for(__method__, altsvc) unless block_given?
-
-
scanner = StringScanner.new(altsvc)
-
until scanner.eos?
-
alt_service = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
-
-
alt_params = []
-
loop do
-
alt_param = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
-
alt_params << alt_param.strip if alt_param
-
scanner.skip(/;/)
-
break if scanner.eos? || scanner.scan(/ *, */)
-
end
-
alt_params = Hash[alt_params.map { |field| field.split("=", 2) }]
-
-
alt_proto, alt_authority = alt_service.split("=", 2)
-
alt_origin = parse_altsvc_origin(alt_proto, alt_authority)
-
return unless alt_origin
-
-
yield(alt_origin, alt_params.merge("proto" => alt_proto))
-
end
-
end
-
-
1
def parse_altsvc_scheme(alt_proto)
-
case alt_proto
-
when "h2c"
-
"http"
-
when "h2"
-
"https"
-
end
-
end
-
-
1
def parse_altsvc_origin(alt_proto, alt_origin)
-
alt_scheme = parse_altsvc_scheme(alt_proto)
-
-
return unless alt_scheme
-
-
alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
-
-
URI.parse("#{alt_scheme}://#{alt_origin}")
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
if RUBY_VERSION < "3.3.0"
-
1
require "base64"
-
elsif !defined?(Base64)
-
module HTTPX
-
# require "base64" will not be a default gem after ruby 3.4.0
-
module Base64
-
module_function
-
-
def decode64(str)
-
str.unpack1("m")
-
end
-
-
def strict_encode64(bin)
-
[bin].pack("m0")
-
end
-
-
def urlsafe_encode64(bin, padding: true)
-
str = strict_encode64(bin)
-
str.chomp!("==") or str.chomp!("=") unless padding
-
str.tr!("+/", "-_")
-
str
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "forwardable"
-
-
1
module HTTPX
-
# Internal class to abstract a string buffer, by wrapping a string and providing the
-
# minimum possible API and functionality required.
-
#
-
# buffer = Buffer.new(640)
-
# buffer.full? #=> false
-
# buffer << "aa"
-
# buffer.capacity #=> 638
-
#
-
1
class Buffer
-
1
extend Forwardable
-
-
1
def_delegator :@buffer, :to_s
-
-
1
def_delegator :@buffer, :to_str
-
-
1
def_delegator :@buffer, :empty?
-
-
1
def_delegator :@buffer, :bytesize
-
-
1
def_delegator :@buffer, :clear
-
-
1
def_delegator :@buffer, :replace
-
-
1
attr_reader :limit
-
-
1
if RUBY_VERSION >= "3.4.0"
-
def initialize(limit)
-
@buffer = String.new("", encoding: Encoding::BINARY, capacity: limit)
-
@limit = limit
-
end
-
-
def <<(chunk)
-
@buffer.append_as_bytes(chunk)
-
end
-
else
-
1
def initialize(limit)
-
164
@buffer = "".b
-
164
@limit = limit
-
end
-
-
1
def_delegator :@buffer, :<<
-
end
-
-
1
def full?
-
143
@buffer.bytesize >= @limit
-
end
-
-
1
def capacity
-
@limit - @buffer.bytesize
-
end
-
-
1
def shift!(fin)
-
52
@buffer = @buffer.byteslice(fin..-1) || "".b
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Callbacks
-
1
def on(type, &action)
-
901
callbacks(type) << action
-
901
action
-
end
-
-
1
def once(type, &block)
-
378
on(type) do |*args, &callback|
-
187
block.call(*args, &callback)
-
187
:delete
-
end
-
end
-
-
1
def emit(type, *args)
-
614
log { "emit #{type.inspect} callbacks" } if respond_to?(:log)
-
933
callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
-
end
-
-
1
def callbacks_for?(type)
-
2
@callbacks && @callbacks.key?(type) && @callbacks[type].any?
-
end
-
-
1
protected
-
-
1
def callbacks(type = nil)
-
1530
return @callbacks unless type
-
-
2556
@callbacks ||= Hash.new { |h, k| h[k] = [] }
-
1530
@callbacks[type]
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
# Session mixin, implements most of the APIs that the users call.
-
# delegates to a default session when extended.
-
1
module Chainable
-
1
%w[head get post put delete trace options connect patch].each do |meth|
-
9
class_eval(<<-MOD, __FILE__, __LINE__ + 1)
-
def #{meth}(*uri, **options) # def get(*uri, **options)
-
request("#{meth.upcase}", uri, **options) # request("GET", uri, **options)
-
end # end
-
MOD
-
end
-
-
# delegates to the default session (see HTTPX::Session#request).
-
1
def request(*args, **options)
-
44
branch(default_options).request(*args, **options)
-
end
-
-
1
def accept(type)
-
with(headers: { "accept" => String(type) })
-
end
-
-
# delegates to the default session (see HTTPX::Session#wrap).
-
1
def wrap(&blk)
-
branch(default_options).wrap(&blk)
-
end
-
-
# returns a new instance loaded with the +pl+ plugin and +options+.
-
1
def plugin(pl, options = nil, &blk)
-
106
klass = is_a?(S) ? self.class : Session
-
106
klass = Class.new(klass)
-
106
klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
-
106
klass.plugin(pl, options, &blk).new
-
end
-
-
# returns a new instance loaded with +options+.
-
1
def with(options, &blk)
-
9
branch(default_options.merge(options), &blk)
-
end
-
-
1
private
-
-
# returns default instance of HTTPX::Options.
-
1
def default_options
-
159
@options || Session.default_options
-
end
-
-
# returns a default instance of HTTPX::Session.
-
1
def branch(options, &blk)
-
53
return self.class.new(options, &blk) if is_a?(S)
-
-
44
Session.new(options, &blk)
-
end
-
-
1
def method_missing(meth, *args, **options, &blk)
-
case meth
-
when /\Awith_(.+)/
-
-
option = Regexp.last_match(1)
-
-
return super unless option
-
-
with(option.to_sym => args.first || options)
-
when /\Aon_(.+)/
-
callback = Regexp.last_match(1)
-
-
return super unless %w[
-
connection_opened connection_closed
-
request_error
-
request_started request_body_chunk request_completed
-
response_started response_body_chunk response_completed
-
].include?(callback)
-
-
warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
-
"Use `HTTPX.plugin(:callbacks).#{meth}` instead."
-
-
plugin(:callbacks).__send__(meth, *args, **options, &blk)
-
else
-
super
-
end
-
end
-
-
1
def respond_to_missing?(meth, *)
-
case meth
-
when /\Awith_(.+)/
-
option = Regexp.last_match(1)
-
-
default_options.respond_to?(option) || super
-
when /\Aon_(.+)/
-
callback = Regexp.last_match(1)
-
-
%w[
-
connection_opened connection_closed
-
request_error
-
request_started request_body_chunk request_completed
-
response_started response_body_chunk response_completed
-
].include?(callback) || super
-
else
-
super
-
end
-
end
-
end
-
-
1
extend Chainable
-
end
-
# frozen_string_literal: true
-
-
1
require "resolv"
-
1
require "forwardable"
-
1
require "httpx/io"
-
1
require "httpx/buffer"
-
-
1
module HTTPX
-
# The Connection can be watched for IO events.
-
#
-
# It contains the +io+ object to read/write from, and knows what to do when it can.
-
#
-
# It defers connecting until absolutely necessary. Connection should be triggered from
-
# the IO selector (until then, any request will be queued).
-
#
-
# A connection boots up its parser after connection is established. All pending requests
-
# will be redirected there after connection.
-
#
-
# A connection can be prevented from closing by the parser, that is, if there are pending
-
# requests. This will signal that the connection was prematurely closed, due to a possible
-
# number of conditions:
-
#
-
# * Remote peer closed the connection ("Connection: close");
-
# * Remote peer doesn't support pipelining;
-
#
-
# A connection may also route requests for a different host for which the +io+ was connected
-
# to, provided that the IP is the same and the port and scheme as well. This will allow to
-
# share the same socket to send HTTP/2 requests to different hosts.
-
#
-
1
class Connection
-
1
extend Forwardable
-
1
include Loggable
-
1
include Callbacks
-
-
1
using URIExtensions
-
-
1
def_delegator :@write_buffer, :empty?
-
-
1
attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
-
-
1
attr_writer :current_selector
-
-
1
attr_accessor :current_session, :family
-
-
1
protected :ssl_session, :sibling
-
-
1
def initialize(uri, options)
-
@current_session = @current_selector = @max_concurrent_requests =
-
@parser = @sibling = @coalesced_connection = @altsvc_connection =
-
@ping_timer = @family = @io = @ssl_session =
-
66
@timeout = @connected_at = @response_received_at = nil
-
-
66
@exhausted = @cloned = @main_sibling = false
-
-
66
@options = Options.new(options)
-
66
@type = initialize_type(uri, @options)
-
66
@origins = [uri.origin]
-
66
@origin = Utils.to_uri(uri.origin)
-
66
@window_size = @options.window_size
-
66
@read_buffer = Buffer.new(@options.buffer_size)
-
66
@write_buffer = Buffer.new(@options.buffer_size)
-
66
@pending = []
-
66
@inflight = 0
-
66
@keep_alive_timeout = @options.timeout[:keep_alive_timeout]
-
66
@no_more_requests_counter = 0
-
-
66
if @options.io
-
# if there's an already open IO, get its
-
# peer address, and force-initiate the parser
-
transition(:already_open)
-
@io = build_socket
-
parser
-
else
-
66
transition(:idle)
-
end
-
66
self.addresses = @options.addresses if @options.addresses
-
end
-
-
1
def peer
-
77
@origin
-
end
-
-
# this is a semi-private method, to be used by the resolver
-
# to initiate the io object.
-
1
def addresses=(addrs)
-
29
if @io
-
@io.add_addresses(addrs)
-
else
-
29
@io = build_socket(addrs)
-
end
-
end
-
-
1
def addresses
-
61
@io && @io.addresses
-
end
-
-
1
def addresses?
-
76
@io && @io.addresses?
-
end
-
-
1
def match?(uri, options)
-
11
return false if !used? && (@state == :closing || @state == :closed)
-
-
11
@origins.include?(uri.origin) &&
-
# if there is more than one origin to match, it means that this connection
-
# was the result of coalescing. To prevent blind trust in the case where the
-
# origin came from an ORIGIN frame, we're going to verify the hostname with the
-
# SSL certificate
-
9
(@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host))) &&
-
@options.connection_options_match?(options)
-
end
-
-
1
def mergeable?(connection)
-
3
return false if @state == :closing || @state == :closed || !@io
-
-
return false unless connection.addresses
-
-
(
-
(open? && @origin == connection.origin) ||
-
!(@io.addresses & (connection.addresses || [])).empty?
-
) && @options.connection_options_match?(connection.options)
-
end
-
-
# coalesces +self+ into +connection+.
-
1
def coalesce!(connection)
-
@coalesced_connection = connection
-
-
close_sibling
-
connection.merge(self)
-
end
-
-
1
def coalesced?
-
38
@coalesced_connection
-
end
-
-
# coalescable connections need to be mergeable!
-
# but internally, #mergeable? is called before #coalescable?
-
1
def coalescable?(connection)
-
if @io.protocol == "h2" &&
-
@origin.scheme == "https" &&
-
connection.origin.scheme == "https" &&
-
@io.can_verify_peer?
-
@io.verify_hostname(connection.origin.host)
-
else
-
@origin == connection.origin
-
end
-
end
-
-
1
def merge(connection)
-
@origins |= connection.instance_variable_get(:@origins)
-
if @ssl_session.nil? && (ssl_session = connection.ssl_session)
-
@ssl_session = ssl_session
-
# the socket only needs the merged session if it can still resume it,
-
# i.e. if TLS hasn't been negotiated yet.
-
@io.ssl_session = ssl_session if @io.is_a?(SSL) && !@io.connected?
-
end
-
connection.purge_pending do |req|
-
req.transition(:idle)
-
send(req)
-
end
-
end
-
-
1
def purge_pending(&block)
-
if @parser
-
pending = @parser.pending
-
@inflight -= pending.size
-
pending.reject! do |req|
-
block.call(req)
-
true
-
end
-
end
-
@pending.reject! do |req|
-
block.call(req)
-
true
-
end
-
end
-
-
1
def io_connected?
-
return @coalesced_connection.io_connected? if @coalesced_connection
-
-
@io && @io.state == :connected
-
end
-
-
1
def connecting?
-
462
@state == :idle
-
end
-
-
1
def inflight?
-
38
@parser && (
-
# parser may be dealing with other requests (possibly started from a different fiber)
-
4
!@parser.empty? ||
-
# connection may be doing connection termination handshake
-
!@write_buffer.empty?
-
)
-
end
-
-
1
def interests
-
# connecting
-
312
if connecting?
-
32
connect
-
-
32
return @io.interests if connecting?
-
end
-
-
304
return @parser.interests if @parser
-
-
nil
-
rescue Error => e
-
on_error(e)
-
nil
-
end
-
-
1
def to_io
-
69
@io.to_io
-
end
-
-
1
def call
-
124
case @state
-
when :idle
-
63
return if no_more_requests?
-
-
36
connect
-
-
# when opening the tcp or ssl socket fails
-
36
return if @state == :closed
-
-
36
consume
-
when :closed
-
return if no_more_requests?
-
-
# there are pending requests to send, restart the state machine.
-
idling
-
-
# @fiber-switch-guard
-
# fiber may have switch after ensuring that @io is closed.
-
return unless @state == :idle
-
-
call
-
when :closing
-
consume
-
transition(:closed)
-
-
# @fiber-switch-guard
-
# fiber may have switch while closing @io.
-
return if @state == :closed &&
-
# only remain here if there are pending requests.
-
@pending.empty?
-
-
call
-
when :open
-
61
consume
-
end
-
nil
-
rescue Errno::ECONNRESET,
-
Errno::EINVAL,
-
SocketError,
-
IOError,
-
TLSError => e
-
@write_buffer.clear
-
on_io_error(e)
-
rescue Error => e
-
@write_buffer.clear
-
on_error(e)
-
rescue Exception => e # rubocop:disable Lint/RescueException
-
force_close(true)
-
raise e
-
end
-
-
1
def initial_call
-
55
call
-
end
-
-
1
def close
-
4
transition(:active) if @state == :inactive
-
-
4
@parser.close if @parser
-
end
-
-
1
def terminate
-
4
case @state
-
when :idle
-
purge_after_closed
-
-
# @fiber-switch-guard
-
if @io.can_disconnect? && @pending.empty?
-
disconnect
-
return
-
end
-
when :closed
-
@connected_at = nil
-
end
-
-
4
close
-
end
-
-
# bypasses state machine rules while setting the connection in the
-
# :closed state.
-
1
def force_close(delete_pending = false)
-
force_purge
-
return unless @state == :closed
-
-
if delete_pending
-
@pending.clear
-
elsif (parser = @parser)
-
enqueue_pending_requests_from_parser(parser)
-
end
-
-
return unless @pending.empty?
-
-
disconnect
-
emit(:force_closed, delete_pending)
-
end
-
-
# bypasses the state machine to force closing of connections still connecting.
-
# **only** used for Happy Eyeballs v2.
-
1
def force_reset(cloned = false)
-
34
@state = :closing
-
34
@cloned = cloned
-
34
transition(:closed)
-
end
-
-
1
def reset
-
35
return if @state == :closing || @state == :closed
-
-
# do not reset a connection which may have restarted back to :idle, such when the parser resets
-
# (example: HTTP/1 parser disabling pipelining)
-
35
return if @state == :idle && @pending.any?
-
-
35
if @ping_timer
-
@ping_timer.cancel
-
@ping_timer = nil
-
end
-
-
35
parser = @parser
-
-
35
if parser && parser.respond_to?(:max_concurrent_requests)
-
# if connection being reset has at some downgraded the number of concurrent
-
# requests, such as in the case where an attempt to use HTTP/1 pipelining failed,
-
# keep that information around.
-
28
@max_concurrent_requests = parser.max_concurrent_requests
-
end
-
-
35
transition(:closing)
-
-
35
transition(:closed)
-
end
-
-
1
def send(request)
-
36
return @coalesced_connection.send(request) if @coalesced_connection
-
-
36
if @parser && !@write_buffer.full?
-
if @response_received_at && @keep_alive_timeout &&
-
Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
-
# when pushing a request into an existing connection, we have to check whether there
-
# is the possibility that the connection might have extended the keep alive timeout.
-
# for such cases, we want to ping for availability before deciding to shovel requests.
-
log(level: 3) { "keep alive timeout expired, pinging connection..." }
-
@pending << request
-
transition(:active) if @state == :inactive
-
request.ping!
-
ping(request)
-
return
-
end
-
-
send_request_to_parser(request)
-
else
-
36
@pending << request
-
end
-
end
-
-
1
def timeout
-
71
return if @state == :closed || @state == :inactive
-
-
71
return @timeout if @timeout
-
-
33
return @options.timeout[:connect_timeout] if @state == :idle
-
-
33
@options.timeout[:operation_timeout]
-
end
-
-
1
def idling
-
3
purge_after_closed
-
-
3
return unless @state == :closed
-
-
3
@write_buffer.clear
-
3
transition(:idle)
-
3
return unless @parser
-
-
3
enqueue_pending_requests_from_parser(parser)
-
3
@parser = nil
-
end
-
-
1
def used?
-
49
@connected_at
-
end
-
-
1
def deactivate
-
transition(:inactive)
-
end
-
-
1
def open?
-
32
@state == :open || @state == :inactive
-
end
-
-
1
def handle_socket_timeout(interval)
-
error = OperationTimeoutError.new(interval, "timed out while waiting on select")
-
error.set_backtrace(caller)
-
on_error(error)
-
end
-
-
1
def sibling=(connection)
-
4
@sibling = connection
-
-
4
return unless connection
-
-
@main_sibling = connection.sibling.nil?
-
-
return unless @main_sibling
-
-
connection.sibling = self
-
end
-
-
1
def handle_connect_error(error)
-
3
return on_error(error) unless @sibling && @sibling.connecting?
-
-
@sibling.merge(self)
-
-
force_reset(true)
-
end
-
-
# disconnects from the current session it's attached to
-
1
def disconnect
-
68
return if @exhausted # it'll reset
-
-
68
return unless (current_session = @current_session) && (current_selector = @current_selector)
-
-
68
@current_session = @current_selector = nil
-
-
68
current_session.deselect_connection(self, current_selector, @cloned)
-
end
-
-
1
def on_connect_error(e)
-
# connect errors, exit gracefully
-
error = ConnectionError.new(e.message)
-
error.set_backtrace(e.backtrace)
-
handle_connect_error(error) if connecting?
-
force_close
-
end
-
-
1
def on_io_error(e)
-
on_error(e)
-
-
# do not force close if parser resets the connection.
-
# can happen i.e. when HTTP/1.1 pipelining is disabled.
-
return if @state == :idle && @pending.any?
-
-
force_close(true)
-
end
-
-
1
def on_error(error, request = nil)
-
3
if error.is_a?(OperationTimeoutError)
-
-
# inactive connections do not contribute to the select loop, therefore
-
# they should not fail due to such errors.
-
return if @state == :inactive
-
-
if @timeout
-
@timeout -= error.timeout
-
return unless @timeout <= 0
-
-
@timeout = nil
-
end
-
-
error = error.to_connection_error if connecting?
-
end
-
3
handle_error(error, request)
-
3
reset
-
end
-
-
# simplecov:disable
-
1
def inspect
-
"#<#{self.class}:#{object_id} " \
-
"@origin=#{@origin} " \
-
"@state=#{@state} " \
-
"@pending=#{@pending.size} " \
-
"@io=#{@io}>"
-
end
-
# simplecov:enable
-
-
1
private
-
-
1
def connect
-
68
transition(:open)
-
end
-
-
1
def consume
-
101
return unless @io
-
-
101
catch(:called) do
-
101
epiped = false
-
101
loop do
-
# connection may have
-
139
return if @state == :idle
-
-
111
parser.consume
-
-
# we exit if there's no more requests to process
-
#
-
# this condition takes into account:
-
#
-
# * the number of pending requests
-
# * the number of inflight requests
-
# * whether the write buffer has bytes (i.e. for close handshake)
-
111
if no_more_requests? && @write_buffer.empty?
-
4
no_more_requests_loop_check if @parser && @parser.pending.any?
-
-
# terminate if an altsvc connection has been established
-
4
terminate if @altsvc_connection
-
-
4
return
-
end
-
-
107
@timeout = @current_timeout
-
-
107
read_drained = false
-
107
write_drained = nil
-
-
#
-
# tight read loop.
-
#
-
# read as much of the socket as possible.
-
#
-
# this tight loop reads all the data it can from the socket and pipes it to
-
# its parser.
-
#
-
loop do
-
84
siz = @io.read(@window_size, @read_buffer)
-
84
log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
-
84
unless siz
-
@write_buffer.clear
-
-
ex = EOFError.new("descriptor closed")
-
ex.set_backtrace(caller)
-
on_error(ex)
-
return
-
end
-
-
# socket has been drained. mark and exit the read loop.
-
84
if siz.zero?
-
37
read_drained = @read_buffer.empty?
-
37
epiped = false
-
37
break
-
end
-
-
47
parser << @read_buffer.to_s
-
-
# continue reading if possible.
-
19
break if interests == :w && !epiped
-
-
# exit the read loop if connection is preparing to be closed
-
15
break if @state == :closing || @state == :closed
-
-
# exit #consume altogether if all outstanding requests have been dealt with
-
15
if no_more_requests? && @write_buffer.empty? # rubocop:disable Style/Next
-
4
no_more_requests_loop_check if @parser && @parser.pending.any?
-
-
# terminate if an altsvc connection has been established
-
4
terminate if @altsvc_connection
-
-
4
return
-
end
-
107
end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
-
-
#
-
# tight write loop.
-
#
-
# flush as many bytes as the sockets allow.
-
#
-
loop do
-
# buffer has been drained, mark and exit the write loop.
-
44
if @write_buffer.empty?
-
# we only mark as drained on the first loop
-
2
write_drained = write_drained.nil? && @inflight.positive?
-
-
2
break
-
end
-
-
begin
-
42
siz = @io.write(@write_buffer)
-
rescue Errno::EPIPE
-
# this can happen if we still have bytes in the buffer to send to the server, but
-
# the server wants to respond immediately with some message, or an error. An example is
-
# when one's uploading a big file to an unintended endpoint, and the server stops the
-
# consumption, and responds immediately with an authorization of even method not allowed error.
-
# at this point, we have to let the connection switch to read-mode.
-
log(level: 2) { "pipe broken, could not flush buffer..." }
-
epiped = true
-
read_drained = false
-
break
-
end
-
42
log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
-
42
unless siz
-
@write_buffer.clear
-
-
ex = EOFError.new("descriptor closed")
-
ex.set_backtrace(caller)
-
on_error(ex)
-
return
-
end
-
-
# socket closed for writing. mark and exit the write loop.
-
42
if siz.zero?
-
write_drained = !@write_buffer.empty?
-
break
-
end
-
-
# exit write loop if marked to consume from peer, or is closing.
-
42
break if interests == :r || @state == :closing || @state == :closed
-
-
2
write_drained = false
-
75
end unless (ints = interests) == :r
-
-
75
send_pending if @state == :open
-
-
# return if socket is drained
-
75
next unless (ints != :r || read_drained) && (ints != :w || write_drained)
-
-
# gotta go back to the event loop. It happens when:
-
#
-
# * the socket is drained of bytes or it's not the interest of the conn to read;
-
# * theres nothing more to write, or it's not in the interest of the conn to write;
-
37
log(level: 3) { "(#{ints}): WAITING FOR EVENTS..." }
-
37
return
-
end
-
end
-
end
-
-
1
def send_pending
-
240
while !@write_buffer.full? && (request = @pending.shift)
-
34
send_request_to_parser(request)
-
end
-
end
-
-
1
def parser
-
227
@parser ||= build_parser
-
end
-
-
1
def send_request_to_parser(request)
-
34
@inflight += 1
-
34
request.peer_address = @io.ip && @io.ip.address
-
34
set_request_timeouts(request)
-
-
34
parser.send(request)
-
-
34
return unless @state == :inactive
-
-
transition(:active)
-
# mark request as ping, as this inactive connection may have been
-
# closed by the server, and we don't want that to influence retry
-
# bookkeeping.
-
request.ping!
-
end
-
-
1
def enqueue_pending_requests_from_parser(parser)
-
31
parser.reset_requests # move sequential requests back to pending queue.
-
31
parser_pending_requests = parser.pending
-
-
31
return if parser_pending_requests.empty?
-
-
# the connection will be reused, so parser requests must come
-
# back to the pending list before the parser is reset.
-
1
@inflight -= parser_pending_requests.size
-
1
@pending.unshift(*parser_pending_requests)
-
-
1
parser.pending.clear
-
end
-
-
1
def build_parser(protocol = @io.protocol)
-
32
parser = parser_type(protocol).new(@write_buffer, @options)
-
32
set_parser_callbacks(parser)
-
32
parser.max_concurrent_requests = @max_concurrent_requests if @max_concurrent_requests && parser.respond_to?(:max_concurrent_requests=)
-
32
parser
-
end
-
-
1
def set_parser_callbacks(parser)
-
32
parser.on(:response) do |request, response|
-
33
AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
-
build_altsvc_connection(alt_origin, origin, alt_params)
-
end
-
33
@response_received_at = Utils.now
-
33
@no_more_requests_counter = 0
-
33
@inflight -= 1
-
33
response.finish!
-
33
request.emit_response(response)
-
end
-
32
parser.on(:altsvc) do |alt_origin, origin, alt_params|
-
build_altsvc_connection(alt_origin, origin, alt_params)
-
end
-
-
32
parser.on(:pong, &method(:pong))
-
-
32
parser.on(:promise) do |request, stream|
-
request.emit(:promise, parser, stream)
-
end
-
32
parser.on(:exhausted) do
-
enqueue_pending_requests_from_parser(parser)
-
-
@exhausted = true
-
parser.close
-
-
# @fiber-switch-guard
-
# fiber may have switched while closing @io, check whether still in the exhausted loop.
-
next unless @exhausted
-
-
idling
-
-
@exhausted = false
-
end
-
32
parser.on(:origin) do |origin|
-
@origins |= [origin]
-
end
-
32
parser.on(:close) do
-
4
reset
-
end
-
32
parser.on(:close_handshake) do
-
consume unless @state == :closed
-
end
-
32
parser.on(:reset) do
-
28
enqueue_pending_requests_from_parser(parser)
-
-
28
reset
-
-
28
next unless @state == :closed
-
-
# :reset event only fired in http/1.1, so this guarantees
-
# that the connection will be closed here.
-
28
idling unless @pending.empty?
-
end
-
32
parser.on(:current_timeout) do
-
4
@current_timeout = @timeout = parser.timeout
-
end
-
32
parser.on(:timeout) do |tout|
-
4
@timeout = tout
-
end
-
32
parser.on(:error) do |request, error|
-
case error
-
when :http_1_1_required
-
current_session = @current_session
-
current_selector = @current_selector
-
parser.close
-
-
other_connection = current_session.find_connection(@origin, current_selector,
-
@options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
-
other_connection.merge(self)
-
request.transition(:idle)
-
other_connection.send(request)
-
next
-
when OperationTimeoutError
-
# request level timeouts should take precedence
-
next unless request.active_timeouts.empty?
-
end
-
-
@inflight -= 1
-
response = ErrorResponse.new(request, error)
-
request.response = response
-
request.emit_response(response)
-
end
-
end
-
-
1
def transition(nextstate)
-
241
handle_transition(nextstate)
-
rescue Errno::ECONNABORTED,
-
Errno::ECONNREFUSED,
-
Errno::ECONNRESET,
-
Errno::EADDRNOTAVAIL,
-
Errno::EHOSTUNREACH,
-
Errno::EINVAL,
-
Errno::ENETUNREACH,
-
Errno::EPIPE,
-
Errno::ENOENT,
-
SocketError,
-
IOError => e
-
on_connect_error(e)
-
rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
-
# connect errors, exit gracefully
-
handle_error(e)
-
handle_connect_error(e) if connecting?
-
force_close
-
end
-
-
1
def handle_transition(nextstate)
-
241
case nextstate
-
when :idle
-
69
@timeout = @current_timeout = @options.timeout[:connect_timeout]
-
-
69
@connected_at = @response_received_at = nil
-
when :open
-
68
return if @state == :closed
-
-
68
@io.connect
-
68
close_sibling if @io.state == :connected
-
-
68
return unless @io.connected?
-
-
32
@connected_at = Utils.now
-
-
32
send_pending
-
-
32
@timeout = @current_timeout = parser.timeout
-
32
emit(:open)
-
when :inactive
-
return unless @state == :open
-
-
# @type ivar @parser: HTTP1 | HTTP2
-
-
# do not deactivate connection in use
-
return if @inflight.positive? || @parser.waiting_for_ping?
-
when :closing
-
35
return unless connecting? || @state == :open
-
when :closed
-
69
return unless @state == :closing
-
69
return unless @write_buffer.empty?
-
-
69
purge_after_closed
-
-
# @fiber-switch-guard
-
69
return unless @state == :closing && (@io.nil? || @io.can_disconnect?)
-
when :already_open
-
nextstate = :open
-
# the first check for given io readiness must still use a timeout.
-
# connect is the reasonable choice in such a case.
-
@timeout = @options.timeout[:connect_timeout]
-
send_pending
-
when :active
-
return unless @state == :inactive
-
-
nextstate = :open
-
-
# activate
-
@current_session.select_connection(self, @current_selector)
-
end
-
205
log(level: 3) { "#{@state} -> #{nextstate}" }
-
205
@state = nextstate
-
# post state change
-
205
case nextstate
-
when :inactive
-
disconnect
-
when :closing
-
35
return if @write_buffer.empty?
-
-
# try flushing termination handshakes
-
4
consume
-
4
@write_buffer.clear
-
when :closed
-
# TODO: should this raise an error instead?
-
69
return unless @pending.empty?
-
-
68
disconnect
-
end
-
end
-
-
1
def force_purge
-
return if @state == :closed
-
-
@state = :closed
-
@write_buffer.clear
-
begin
-
purge_after_closed
-
rescue IOError
-
# may be raised when closing the socket.
-
# due to connection reuse / fiber scheduling, it may
-
# have been reopened, to bail out in that case.
-
end
-
end
-
-
1
def close_sibling
-
32
sibling = @sibling
-
-
32
return unless sibling
-
-
if sibling.io_connected?
-
reset
-
# TODO: transition connection to closed
-
end
-
-
unless sibling.state == :closed
-
merge(sibling) unless @main_sibling
-
sibling.force_reset(true)
-
end
-
-
@sibling = nil
-
end
-
-
1
def purge_after_closed
-
72
if @io
-
35
@io.close
-
-
# @fiber-switch-guard
-
# due to fiber scheduler, multiple fibers may be listening on the same connection
-
# and moving the state machine forward; in such cases, when the control flow reaches
-
# this line, the io object may not be closed anymore.
-
35
return unless @io&.can_disconnect?
-
end
-
72
@read_buffer.clear
-
72
@timeout = nil
-
end
-
-
1
def initialize_type(uri, options)
-
66
options.transport || begin
-
66
case uri.scheme
-
when "http"
-
62
"tcp"
-
when "https"
-
4
"ssl"
-
else
-
raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
-
end
-
end
-
end
-
-
# returns an HTTPX::Connection for the negotiated Alternative Service (or none).
-
1
def build_altsvc_connection(alt_origin, origin, alt_params)
-
return if @altsvc_connection
-
-
# do not allow security downgrades on altsvc negotiation
-
return if @origin.scheme == "https" && alt_origin.scheme != "https"
-
-
altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
-
-
# altsvc already exists, somehow it wasn't advertised, probably noop
-
return unless altsvc
-
-
alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
-
-
connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
-
-
# advertised altsvc is the same origin being used, ignore
-
return if connection == self
-
-
connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
-
-
@altsvc_connection = connection
-
-
log(level: 1) { "#{origin}: alt-svc connection##{connection.object_id} established to #{alt_origin}" }
-
-
connection.merge(self)
-
rescue UnsupportedSchemeError
-
altsvc["noop"] = true
-
nil
-
end
-
-
1
def build_socket(addrs = nil)
-
29
case @type
-
when "tcp"
-
25
TCP.new(peer, addrs, @options)
-
when "ssl"
-
4
SSL.new(peer, addrs, @options) do |sock|
-
4
sock.ssl_session = @ssl_session
-
4
sock.session_new_cb do |sess|
-
8
@ssl_session = sess
-
-
8
sock.ssl_session = sess
-
end
-
end
-
when "unix"
-
path = Array(addrs).first
-
-
path = String(path) if path
-
-
UNIX.new(peer, path, @options)
-
else
-
raise Error, "unsupported transport (#{@type})"
-
end
-
end
-
-
1
def ping(_request)
-
return if parser.waiting_for_ping?
-
-
parser.ping
-
-
ping_timeout = @options.timeout[:ping_timeout]
-
-
@ping_timer = @current_selector.after(ping_timeout) do
-
log(level: 3) { "ping timeout expired..." }
-
error = PingTimeoutError.new(ping_timeout, "Timed out after #{ping_timeout} seconds")
-
on_error(error)
-
end
-
-
call
-
end
-
-
1
def pong
-
@ping_timer.cancel
-
@ping_timer = nil
-
@response_received_at = Utils.now
-
@no_more_requests_counter = 0
-
send_pending
-
end
-
-
1
def no_more_requests_loop_check
-
log(level: 3) { "NO MORE REQUESTS..." }
-
@no_more_requests_counter += 1
-
-
return if @no_more_requests_counter < 50
-
-
raise Error, "connection corrupted, aborted after looping for a while, " \
-
"please report this https://gitlab.com/os85/httpx/-/work_items " \
-
"along with debug logs"
-
end
-
-
# true when there are no more pending nor inflight (in parser) requests
-
1
def no_more_requests?
-
189
@pending.empty? && @inflight.zero?
-
end
-
-
# recover internal state and emit all relevant error responses when +error+ was raised.
-
# this takes an optiona +request+ which may have already been handled and can be opted out
-
# in the state recovery process.
-
1
def handle_error(error, request = nil)
-
3
if request
-
@inflight -= 1
-
response = ErrorResponse.new(request, error)
-
request.response = response
-
request.emit_response(response)
-
end
-
-
3
pending = @pending
-
3
if (parser = @parser) && parser.respond_to?(:handle_error)
-
# parser.handle_error may disconnect the connection
-
pending = @pending.dup
-
@pending = []
-
-
parser.handle_error(error, request)
-
end
-
-
9
while (req = pending.shift)
-
3
next if request && req == request
-
-
3
resp = ErrorResponse.new(req, error)
-
3
req.response = resp
-
3
req.emit_response(resp)
-
end
-
end
-
-
1
def set_request_timeouts(request)
-
34
request.connection = self
-
34
set_request_write_timeout(request)
-
34
set_request_read_timeout(request)
-
34
set_request_request_timeout(request)
-
34
set_request_total_request_timeout(request)
-
end
-
-
1
def set_request_read_timeout(request)
-
34
read_timeout = request.read_timeout
-
-
34
return if read_timeout.nil? || read_timeout.infinite?
-
-
34
set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
-
read_timeout_callback(request, read_timeout)
-
end
-
end
-
-
1
def set_request_write_timeout(request)
-
34
write_timeout = request.write_timeout
-
-
34
return if write_timeout.nil? || write_timeout.infinite?
-
-
34
set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
-
write_timeout_callback(request, write_timeout)
-
end
-
end
-
-
1
def set_request_request_timeout(request)
-
34
request_timeout = request.request_timeout
-
-
34
return if request_timeout.nil? || request_timeout.infinite?
-
-
set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
-
read_timeout_callback(request, request_timeout, RequestTimeoutError)
-
end
-
end
-
-
1
def write_timeout_callback(request, timeout)
-
return if request.state == :done
-
-
@write_buffer.clear
-
error = WriteTimeoutError.new(request, nil, timeout)
-
-
request.handle_error(error)
-
end
-
-
1
def read_timeout_callback(request, timeout, error_type = ReadTimeoutError)
-
response = request.response
-
-
return if response && response.finished?
-
-
@write_buffer.clear
-
error = error_type.new(request, response, timeout)
-
-
request.handle_error(error)
-
end
-
-
1
def set_request_total_request_timeout(request)
-
34
return if request.started?
-
-
31
total_request_timeout = request.total_request_timeout
-
-
31
return if total_request_timeout.nil? || total_request_timeout.infinite?
-
-
set_request_timeout(:total_request_timeout, request, total_request_timeout, :headers, :complete) do
-
read_timeout_callback(request, total_request_timeout, TotalRequestTimeoutError)
-
end
-
end
-
-
1
def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
-
68
request.set_timeout_callback(start_event) do
-
68
unless (selector = @current_selector)
-
raise Error, "request has been resend to an out-of-session connection, and this " \
-
"should never happen!!! Please report this error! " \
-
"(state:#{@state}, " \
-
"parser?:#{!!@parser}, " \
-
"bytes in write buffer?:#{!@write_buffer.empty?}, " \
-
"cloned?:#{@cloned}, " \
-
"sibling?:#{!!@sibling}, " \
-
"coalesced?:#{coalesced?})"
-
end
-
-
68
timer = selector.after(timeout, callback)
-
68
timer.label = label
-
68
request.active_timeouts << timer
-
-
68
Array(finish_events).each do |event|
-
# clean up request timeouts if the connection errors out
-
102
request.set_timeout_callback(event) do
-
100
timer.cancel
-
100
request.active_timeouts.delete(timer)
-
end
-
end
-
end
-
end
-
-
1
def parser_type(protocol)
-
32
case protocol
-
4
when "h2" then @options.http2_class
-
28
when "http/1.1" then @options.http1_class
-
else
-
raise Error, "unsupported protocol (##{protocol})"
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "httpx/parser/http1"
-
-
1
module HTTPX
-
1
class Connection::HTTP1
-
1
include Callbacks
-
1
include Loggable
-
-
1
MAX_REQUESTS = 200
-
1
CRLF = "\r\n"
-
-
1
UPCASED = {
-
"www-authenticate" => "WWW-Authenticate",
-
"http2-settings" => "HTTP2-Settings",
-
"content-md5" => "Content-MD5",
-
"last-event-id" => "Last-Event-ID",
-
}.freeze
-
1
attr_reader :pending, :requests
-
-
1
attr_accessor :max_concurrent_requests
-
-
1
def initialize(buffer, options)
-
28
@options = options
-
28
@max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
-
28
@max_requests = @options.max_requests
-
28
@parser = Parser::HTTP1.new(self, options.max_response_headers, options.max_response_header_value_size)
-
28
@buffer = buffer
-
28
@version = [1, 1]
-
28
@pending = []
-
28
@requests = []
-
28
@request = nil
-
28
@handshake_completed = @pipelining = false
-
end
-
-
1
def timeout
-
28
@options.timeout[:operation_timeout]
-
end
-
-
1
def interests
-
234
request = @request || @requests.first
-
-
234
return unless request
-
-
234
return :w if request.interests == :w || !@buffer.empty?
-
-
148
:r
-
end
-
-
1
def reset
-
28
if @ping_timer
-
@ping_timer.cancel
-
@ping_timer = nil
-
end
-
28
@max_requests = @options.max_requests || MAX_REQUESTS
-
28
@parser.reset!
-
28
@handshake_completed = false
-
28
reset_requests
-
end
-
-
1
def reset_requests
-
59
@requests.reverse_each do |request|
-
1
next if request.response
-
-
1
request.transition(:idle)
-
1
@pending.unshift(request)
-
end
-
59
@requests.clear
-
end
-
-
1
def close
-
reset
-
emit(:close)
-
end
-
-
1
def exhausted?
-
!@max_requests.positive?
-
end
-
-
1
def empty?
-
# this means that for every request there's an available
-
# partial response, so there are no in-flight requests waiting.
-
@requests.empty? || (
-
# checking all responses can be time-consuming. Alas, as in HTTP/1, responses
-
# do not come out of order, we can get away with checking first and last.
-
!@requests.first.response.nil? &&
-
(@requests.size == 1 || !@requests.last.response.nil?)
-
)
-
end
-
-
1
def <<(data)
-
33
@parser << data
-
end
-
-
1
def send(request)
-
29
unless @max_requests.positive?
-
@pending << request
-
return
-
end
-
-
29
return if @requests.include?(request)
-
-
29
@requests << request
-
29
@pipelining = @max_concurrent_requests > 1 && @requests.size > 1
-
end
-
-
1
def consume
-
87
requests_limit = [@max_requests, @requests.size].min
-
87
concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
-
87
@requests.each_with_index do |request, idx|
-
90
break if idx >= concurrent_requests_limit
-
90
next unless request.can_buffer?
-
-
31
handle(request)
-
end
-
end
-
-
# HTTP Parser callbacks
-
#
-
# must be public methods, or else they won't be reachable
-
-
1
def on_start
-
28
log(level: 2) { "parsing begins" }
-
end
-
-
1
def on_headers(h)
-
28
request = @request = @requests.first
-
-
28
return if request.response
-
-
28
request.log(level: 2) { "headers received" }
-
28
headers = request.options.headers_class.new(h)
-
28
response = request.options.response_class.new(request,
-
@parser.status_code,
-
@parser.http_version.join("."),
-
headers)
-
28
request.log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
-
28
request.log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{log_redact_headers(v)}" }.join("\n") }
-
-
28
if response.content_length && response.content_length > request.options.max_response_body_size
-
raise HTTPX::Error, "maximum response body size exceeded"
-
end
-
-
28
request.response = response
-
28
on_complete if response.finished?
-
end
-
-
1
def on_trailers(h)
-
request = @request
-
-
return unless request
-
-
response = request.response
-
-
request.log(level: 2) { "trailer headers received" }
-
request.log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{log_redact_headers(v.join(", "))}" }.join("\n") }
-
response.merge_headers(h)
-
end
-
-
1
def on_data(chunk)
-
19
request = @request
-
-
19
return unless request
-
-
19
request.log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
-
19
request.log(level: 2, color: :green) { "-> #{log_redact_body(chunk.inspect)}" }
-
-
19
response = request.response
-
-
19
response << chunk
-
end
-
-
1
def on_complete
-
28
request = @request
-
-
28
return unless request
-
-
28
request.log(level: 2) { "parsing complete" }
-
28
dispatch(request)
-
end
-
-
1
def dispatch(request)
-
28
if request.expects?
-
@parser.reset!
-
return handle(request)
-
end
-
-
28
@request = nil
-
28
@requests.shift
-
28
response = request.response
-
28
emit(:response, request, response)
-
-
28
if @parser.upgrade?
-
response << @parser.upgrade_data
-
@parser.reset!
-
throw(:called)
-
end
-
-
28
@parser.reset!
-
28
@max_requests -= 1
-
28
if response.is_a?(ErrorResponse)
-
disable
-
else
-
28
manage_connection(request, response)
-
end
-
-
if exhausted?
-
@pending.unshift(*@requests)
-
@requests.clear
-
-
emit(:exhausted)
-
else
-
send(@pending.shift) unless @pending.empty?
-
end
-
end
-
-
1
def handle_error(ex, request = nil)
-
if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request &&
-
(response = @request.response) && response.is_a?(Response) &&
-
!response.headers.key?("content-length") &&
-
!response.headers.key?("transfer-encoding")
-
# if the response does not contain a content-length header, the server closing the
-
# connnection is the indicator of response consumed.
-
# https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
-
catch(:called) { on_complete }
-
return
-
end
-
-
if @pipelining
-
catch(:called) { disable }
-
else
-
while (req = @requests.shift)
-
next if request && request == req
-
-
emit(:error, req, ex)
-
end
-
while (req = @pending.shift)
-
next if request && request == req
-
-
emit(:error, req, ex)
-
end
-
end
-
end
-
-
1
def ping
-
reset
-
emit(:reset)
-
emit(:exhausted)
-
end
-
-
1
def waiting_for_ping?
-
false
-
end
-
-
1
private
-
-
1
def manage_connection(request, response)
-
28
connection = response.headers["connection"]
-
28
case connection
-
when /keep-alive/i
-
if @handshake_completed
-
if @max_requests.zero?
-
@pending.unshift(*@requests)
-
@requests.clear
-
emit(:exhausted)
-
end
-
return
-
end
-
-
keep_alive = response.headers["keep-alive"]
-
return unless keep_alive
-
-
parameters = Hash[keep_alive.split(/ *, */).map do |pair|
-
pair.split(/ *= */, 2)
-
end]
-
@max_requests = parameters["max"].to_i - 1 if parameters.key?("max")
-
-
if parameters.key?("timeout")
-
keep_alive_timeout = parameters["timeout"].to_i
-
emit(:timeout, keep_alive_timeout)
-
end
-
@handshake_completed = true
-
when /close/i
-
28
disable
-
when nil
-
# In HTTP/1.1, it's keep alive by default
-
return if response.version == "1.1" && request.headers["connection"] != "close"
-
-
disable
-
end
-
end
-
-
1
def disable
-
28
disable_pipelining
-
28
reset
-
28
emit(:reset)
-
28
throw(:called)
-
end
-
-
1
def disable_pipelining
-
# do not disable pipelining if already set to 1 request at a time
-
28
return if @max_concurrent_requests == 1
-
-
25
@requests.each do |r|
-
1
r.transition(:idle) if r.response.nil?
-
-
# when we disable pipelining, we still want to try keep-alive.
-
# only when keep-alive with one request fails, do we fallback to
-
# connection: close.
-
1
r.headers["connection"] = "close" if @max_concurrent_requests == 1
-
end
-
# server doesn't handle pipelining, and probably
-
# doesn't support keep-alive. Fallback to send only
-
# 1 keep alive request.
-
25
@max_concurrent_requests = 1
-
25
@pipelining = false
-
end
-
-
1
def set_protocol_headers(request)
-
29
if !request.headers.key?("content-length") &&
-
request.body.bytesize == Float::INFINITY
-
request.body.chunk!
-
end
-
-
29
extra_headers = {}
-
-
29
unless request.headers.key?("connection")
-
29
connection_value = if request.persistent?
-
# when in a persistent connection, the request can't be at
-
# the edge of a renegotiation
-
8
if @requests.index(request) + 1 < @max_requests
-
8
"keep-alive"
-
else
-
"close"
-
end
-
else
-
# when it's not a persistent connection, it sets "Connection: close" always
-
# on the last request of the possible batch (either allowed max requests,
-
# or if smaller, the size of the batch itself)
-
21
requests_limit = [@max_requests, @requests.size].min
-
21
if request == @requests[requests_limit - 1]
-
20
"close"
-
else
-
1
"keep-alive"
-
end
-
end
-
-
29
extra_headers["connection"] = connection_value
-
end
-
29
extra_headers["host"] = request.authority unless request.headers.key?("host")
-
29
extra_headers
-
end
-
-
1
def handle(request)
-
31
catch(:buffer_full) do
-
31
request.transition(:headers)
-
31
join_headers(request) if request.state == :headers
-
31
request.transition(:body)
-
31
join_body(request) if request.state == :body
-
29
request.transition(:trailers)
-
# HTTP/1.1 trailers should only work for chunked encoding
-
29
join_trailers(request) if request.body.chunked? && request.state == :trailers
-
29
request.transition(:done)
-
end
-
end
-
-
1
def join_headline(request)
-
29
"#{request.verb} #{request.path} HTTP/#{@version.join(".")}"
-
end
-
-
1
def join_headers(request)
-
29
headline = join_headline(request)
-
29
@buffer << headline << CRLF
-
29
request.log(color: :yellow) { "<- HEADLINE: #{headline.chomp.inspect}" }
-
29
extra_headers = set_protocol_headers(request)
-
29
join_headers2(request, request.headers.each(extra_headers))
-
29
request.log { "<- " }
-
29
@buffer << CRLF
-
end
-
-
1
def join_body(request)
-
31
return if request.body.empty?
-
-
16
while (chunk = request.drain_body)
-
6
request.log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
-
6
request.log(level: 2, color: :green) { "<- #{log_redact_body(chunk.inspect)}" }
-
6
@buffer << chunk
-
6
throw(:buffer_full, request) if @buffer.full?
-
end
-
-
4
return unless (error = request.drain_error)
-
-
raise error
-
end
-
-
1
def join_trailers(request)
-
return unless request.trailers? && request.callbacks_for?(:trailers)
-
-
join_headers2(request, request.trailers)
-
request.log { "<- " }
-
@buffer << CRLF
-
end
-
-
1
def join_headers2(request, headers)
-
29
headers.each do |field, value|
-
315
field = capitalized(field)
-
315
request.log(color: :yellow) { "<- HEADER: #{[field, log_redact_headers(value)].join(": ")}" }
-
315
@buffer << "#{field}: #{value}#{CRLF}"
-
end
-
end
-
-
1
def capitalized(field)
-
630
UPCASED.fetch(field) { field.split("-").map(&:capitalize).join("-") }
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "securerandom"
-
1
require "http/2"
-
-
1
module HTTPX
-
1
class Connection::HTTP2
-
1
include Callbacks
-
1
include Loggable
-
-
1
MAX_CONCURRENT_REQUESTS = ::HTTP2::DEFAULT_MAX_CONCURRENT_STREAMS
-
-
1
class Error < Error
-
1
def initialize(id, error)
-
super("stream #{id} closed with error: #{error}")
-
end
-
end
-
-
1
class PingError < Error
-
1
def initialize
-
super(0, :ping_error)
-
end
-
end
-
-
1
class GoawayError < Error
-
1
def initialize(code = :no_error)
-
super(0, code)
-
end
-
end
-
-
1
attr_reader :streams, :pending
-
-
1
def initialize(buffer, options)
-
4
@options = options
-
4
@settings = @options.http2_settings
-
4
@pending = []
-
4
@streams = {}
-
4
@drains = {}
-
4
@pings = []
-
4
@streams_to_close_after_receive = []
-
4
@buffer = buffer
-
4
@handshake_completed = false
-
4
@wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
-
4
@max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
-
4
@max_requests = @options.max_requests
-
4
init_connection
-
end
-
-
1
def timeout
-
8
return @options.timeout[:operation_timeout] if @handshake_completed
-
-
4
@options.timeout[:settings_timeout]
-
end
-
-
1
def interests
-
70
if @connection.closed?
-
12
return unless @handshake_completed
-
-
12
return if @buffer.empty?
-
-
# HTTP/2 GOAWAY frame buffered.
-
8
return :w
-
end
-
-
58
unless @connection.state == :connected && @handshake_completed
-
# HTTP/2 in intermediate state or still completing initialization-
-
24
return @buffer.empty? ? :r : :rw
-
end
-
-
34
unless @connection.send_buffer.empty?
-
# HTTP/2 connection is buffering data chunks and failing to emit DATA frames,
-
# most likely because the flow control window is exhausted.
-
return :rw unless @buffer.empty?
-
-
# waiting for WINDOW_UPDATE frames
-
return :r
-
end
-
-
# there are pending bufferable requests
-
34
return :w if !@pending.empty? && can_buffer_more_requests?
-
-
# there are pending frames from the last run
-
34
return :w unless @drains.empty?
-
-
34
if @buffer.empty?
-
# skip if no more requests or pings to process
-
26
return if @streams.empty? && @pings.empty?
-
-
22
:r
-
else
-
# buffered frames
-
8
:w
-
end
-
end
-
-
1
def close
-
4
unless @connection.closed?
-
4
@connection.goaway
-
4
emit(:timeout, @options.timeout[:close_handshake_timeout])
-
end
-
4
emit(:close)
-
end
-
-
1
def empty?
-
4
@connection.closed? || @streams.empty?
-
end
-
-
1
def exhausted?
-
4
!@max_requests.positive?
-
end
-
-
1
def <<(data)
-
14
@connection << data
-
-
28
while (stream, request, error = @streams_to_close_after_receive.shift)
-
# these streams were marked for cancellation due to errors found while processing the
-
# data received by the peer.
-
emit_stream_error(stream, request, error)
-
end
-
end
-
-
1
def send(request, head = false)
-
10
unless can_buffer_more_requests?
-
5
head ? @pending.unshift(request) : @pending << request
-
5
return false
-
end
-
5
unless (stream = @streams[request])
-
5
stream = @connection.new_stream(**request.http2_stream_options)
-
5
handle_stream(stream, request)
-
5
@streams[request] = stream
-
5
@max_requests -= 1
-
end
-
5
handle(request, stream)
-
5
true
-
rescue ::HTTP2::Error::StreamLimitExceeded
-
@pending.unshift(request)
-
false
-
rescue ::HTTP2::Error::Error, ArgumentError => e
-
emit(:error, request, e)
-
end
-
-
1
def consume
-
24
@streams.each do |request, stream|
-
10
next unless request.can_buffer?
-
-
handle(request, stream)
-
end
-
end
-
-
1
def handle_error(ex, request = nil)
-
if ex.is_a?(OperationTimeoutError) && !@handshake_completed && @connection.state != :closed
-
@connection.goaway(:settings_timeout, "closing due to settings timeout")
-
emit(:close_handshake)
-
settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
-
settings_ex.set_backtrace(ex.backtrace)
-
ex = settings_ex
-
end
-
while (req, _ = @streams.shift)
-
next if request && request == req
-
-
emit(:error, req, ex)
-
end
-
while (req = @pending.shift)
-
next if request && request == req
-
-
emit(:error, req, ex)
-
end
-
end
-
-
1
def ping
-
ping = SecureRandom.gen_random(8)
-
@connection.ping(ping.dup)
-
ensure
-
@pings << ping
-
end
-
-
1
def waiting_for_ping?
-
@pings.any?
-
end
-
-
1
def reset_requests; end
-
-
1
private
-
-
1
def can_buffer_more_requests?
-
10
(@handshake_completed || !@wait_for_handshake) &&
-
@streams.size < @max_concurrent_requests &&
-
@streams.size < @max_requests
-
end
-
-
1
def send_pending
-
13
while (request = @pending.shift)
-
5
break unless send(request, true)
-
end
-
end
-
-
1
def handle(request, stream)
-
5
catch(:buffer_full) do
-
5
request.transition(:headers)
-
5
join_headers(stream, request) if request.state == :headers
-
5
request.transition(:body)
-
5
join_body(stream, request) if request.state == :body
-
5
request.transition(:trailers)
-
5
join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
-
5
request.transition(:done)
-
end
-
end
-
-
1
def init_connection
-
4
@connection = ::HTTP2::Client.new(@settings)
-
4
@connection.on(:frame, &method(:on_frame))
-
4
@connection.on(:frame_sent, &method(:on_frame_sent))
-
4
@connection.on(:frame_received, &method(:on_frame_received))
-
4
@connection.on(:origin, &method(:on_origin))
-
4
@connection.on(:promise, &method(:on_promise))
-
4
@connection.on(:altsvc) { |frame| on_altsvc(frame[:origin], frame) }
-
4
@connection.on(:settings_ack, &method(:on_settings))
-
4
@connection.on(:ack, &method(:on_pong))
-
4
@connection.on(:goaway, &method(:on_close))
-
#
-
# Some servers initiate HTTP/2 negotiation right away, some don't.
-
# As such, we have to check the socket buffer. If there is something
-
# to read, the server initiated the negotiation. If not, we have to
-
# initiate it.
-
#
-
4
@connection.send_connection_preface
-
end
-
-
1
alias_method :reset, :init_connection
-
1
public :reset
-
-
1
def handle_stream(stream, request)
-
5
request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
-
5
stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
-
10
stream.on(:half_close) { on_stream_half_close(stream, request) }
-
5
stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
-
5
stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
-
5
stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
-
end
-
-
1
def set_protocol_headers(request)
-
{
-
5
":scheme" => request.scheme,
-
":method" => request.verb,
-
":path" => request.path,
-
":authority" => request.authority,
-
}
-
end
-
-
1
def join_headers(stream, request)
-
5
extra_headers = set_protocol_headers(request)
-
-
5
if request.headers.key?("host")
-
request.log { "forbidden \"host\" header found (#{log_redact_headers(request.headers["host"])}), will use it as authority..." }
-
extra_headers[":authority"] = request.headers["host"]
-
end
-
-
5
request.log(level: 1, color: :yellow) do
-
"\n#{request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")}"
-
end
-
5
stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
-
end
-
-
1
def join_trailers(stream, request)
-
1
unless request.trailers?
-
1
stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
-
1
return
-
end
-
-
request.log(level: 1, color: :yellow) do
-
request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")
-
end
-
stream.headers(request.trailers.each, end_stream: true)
-
end
-
-
1
def join_body(stream, request)
-
5
return if request.body.empty?
-
-
1
chunk = @drains.delete(request) || request.drain_body
-
1
while chunk
-
1
next_chunk = request.drain_body
-
1
send_chunk(request, stream, chunk, next_chunk)
-
-
1
if next_chunk && (@buffer.full? || request.body.unbounded_body?)
-
@drains[request] = next_chunk
-
throw(:buffer_full)
-
end
-
-
1
chunk = next_chunk
-
end
-
-
1
return unless (error = request.drain_error)
-
-
on_stream_refuse(stream, request, error)
-
end
-
-
1
def send_chunk(request, stream, chunk, next_chunk)
-
1
request.log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
-
1
request.log(level: 2, color: :green) { "#{stream.id}: -> #{log_redact_body(chunk.inspect)}" }
-
1
stream.data(chunk, end_stream: end_stream?(request, next_chunk))
-
end
-
-
1
def end_stream?(request, next_chunk)
-
1
!(next_chunk || request.trailers? || request.callbacks_for?(:trailers))
-
end
-
-
######
-
# HTTP/2 Callbacks
-
######
-
-
1
def on_stream_headers(stream, request, h)
-
5
response = request.response
-
-
5
if response.is_a?(Response) && response.version == "2.0"
-
on_stream_trailers(stream, request, response, h)
-
return
-
end
-
-
5
request.log(color: :yellow) do
-
h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{k == ":status" ? v : log_redact_headers(v)}" }.join("\n")
-
end
-
5
_, status = h.shift
-
5
headers = request.options.headers_class.new(h)
-
-
5
raise HTTPX::Error, "maximum number of response headers exceeded" if h.size > @options.max_response_headers
-
-
5
if (max_header_value_size = @options.max_response_header_value_size)
-
headers.each do |_, v| # rubocop:disable Style/HashEachMethods
-
raise HTTPX::Error, "maximum header value size exceeded" if v.size > max_header_value_size
-
end
-
end
-
-
5
response = request.options.response_class.new(request, status, "2.0", headers)
-
-
5
if response.content_length && response.content_length > request.options.max_response_body_size
-
raise HTTPX::Error.new, "maximum response body size exceeded"
-
end
-
-
5
request.response = response
-
5
@streams[request] = stream
-
-
5
handle(request, stream) if request.expects?
-
rescue HTTPX::Error => e
-
@streams_to_close_after_receive << [stream, request, e]
-
end
-
-
1
def on_stream_trailers(stream, request, response, h)
-
request.log(color: :yellow) do
-
h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")
-
end
-
response.merge_headers(h)
-
end
-
-
1
def on_stream_data(stream, request, data)
-
7
request.log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
-
7
request.log(level: 2, color: :green) { "#{stream.id}: <- #{log_redact_body(data.inspect)}" }
-
-
7
return unless request.response
-
-
7
request.response << data
-
rescue HTTPX::Error => e
-
if stream.state == :closing
-
# there won't be any more chunks to process after this one.
-
emit_stream_error(stream, request, e)
-
else
-
# defer until the last chunk from the payload is processed.
-
@streams_to_close_after_receive << [stream, request, e]
-
end
-
end
-
-
1
def on_stream_refuse(stream, request, error)
-
on_stream_close(stream, request, error)
-
stream.close
-
end
-
-
1
def on_stream_half_close(stream, request)
-
5
unless stream.send_buffer.empty?
-
stream.send_buffer.clear
-
stream.data("", end_stream: true)
-
end
-
-
# TODO: omit log line if response already here
-
5
request.log(level: 2) { "#{stream.id}: waiting for response..." }
-
end
-
-
1
def on_stream_close(stream, request, error)
-
5
return if error == :stream_closed && !@streams.key?(request)
-
-
5
log(level: 2) { "#{stream.id}: closing stream" }
-
5
teardown(request)
-
-
5
if error
-
case error
-
when :http_1_1_required
-
emit(:error, request, error)
-
else
-
ex = Error.new(stream.id, error)
-
ex.set_backtrace(caller)
-
response = ErrorResponse.new(request, ex)
-
request.response = response
-
emit(:response, request, response)
-
end
-
else
-
5
if (response = request.response)
-
5
if response.is_a?(Response) && response.status == 421
-
emit(:error, request, :http_1_1_required)
-
else
-
5
emit(:response, request, response)
-
end
-
end
-
end
-
5
send(@pending.shift) unless @pending.empty?
-
-
5
return unless @streams.empty? && exhausted?
-
-
if @pending.empty?
-
close
-
else
-
emit(:exhausted)
-
end
-
end
-
-
1
def on_frame(bytes)
-
22
@buffer << bytes
-
end
-
-
1
def on_settings(*)
-
4
@handshake_completed = true
-
4
emit(:current_timeout)
-
4
@max_concurrent_requests = [@max_concurrent_requests, @connection.remote_settings[:settings_max_concurrent_streams]].min
-
4
send_pending
-
end
-
-
1
def on_close(_last_frame, error, _payload)
-
is_connection_closed = @connection.closed?
-
if error
-
@buffer.clear if is_connection_closed
-
case error
-
when :http_1_1_required
-
while (request = @pending.shift)
-
emit(:error, request, error)
-
end
-
else
-
ex = GoawayError.new(error)
-
ex.set_backtrace(caller)
-
-
handle_error(ex)
-
teardown
-
-
end
-
end
-
return unless is_connection_closed && @streams.empty?
-
-
emit(:close) if is_connection_closed
-
end
-
-
1
def on_frame_sent(frame)
-
18
log(level: 2) { "#{frame[:stream]}: frame was sent!" }
-
18
log(level: 2, color: :blue) { "#{frame[:stream]}: #{frame_with_extra_info(frame)}" }
-
end
-
-
1
def on_frame_received(frame)
-
20
log(level: 2) { "#{frame[:stream]}: frame was received!" }
-
20
log(level: 2, color: :magenta) { "#{frame[:stream]}: #{frame_with_extra_info(frame)}" }
-
end
-
-
1
def frame_with_extra_info(frame)
-
flags_bits = frame.fetch(:flags, 0)
-
case frame[:type]
-
when :data
-
flags = [] #: Array[Symbol]
-
flags << :end_stream if flags_bits.anybits?(0b0001)
-
flags << :padded if flags_bits.anybits?(0b1000)
-
frame.merge(payload: frame[:payload].bytesize, flags: flags)
-
when :push_promise, :headers
-
flags = [] #: Array[Symbol]
-
flags << :end_stream if flags_bits.anybits?(0b0001)
-
flags << :priority if flags_bits.anybits?(0b0010)
-
flags << :end_headers if flags_bits.anybits?(0b0100)
-
flags << :padded if flags_bits.anybits?(0b1000)
-
frame.merge(payload: log_redact_headers(frame[:payload]), flags: flags)
-
when :ping
-
flags = [] #: Array[Symbol]
-
flags << :ack if flags_bits.anybits?(0b0001)
-
frame.merge(payload: log_redact_headers(frame[:payload]), flags: flags)
-
when :settings
-
flags = [] #: Array[Symbol]
-
flags << :ack if flags_bits.anybits?(0b0001)
-
frame.merge(flags: flags)
-
when :window_update
-
connection_or_stream = if (id = frame[:stream]).zero?
-
@connection
-
else
-
@streams.each_value.find { |s| s.id == id }
-
end
-
if connection_or_stream
-
frame.merge(
-
local_window: connection_or_stream.local_window,
-
remote_window: connection_or_stream.remote_window,
-
buffered_amount: connection_or_stream.buffered_amount,
-
stream_state: connection_or_stream.state,
-
)
-
else
-
frame
-
end
-
else
-
frame
-
end.merge(connection_state: @connection.state)
-
end
-
-
1
def on_altsvc(origin, frame)
-
log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
-
log(level: 2) { "#{frame[:stream]}: #{log_redact_headers(frame.inspect)}" }
-
alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
-
params = { "ma" => frame[:max_age] }
-
emit(:altsvc, origin, alt_origin, origin, params)
-
end
-
-
1
def on_promise(stream)
-
emit(:promise, @streams.key(stream.parent), stream)
-
end
-
-
1
def on_origin(origin)
-
emit(:origin, origin)
-
end
-
-
1
def on_pong(ping)
-
raise PingError unless @pings.delete(ping.to_s)
-
-
emit(:pong)
-
end
-
-
1
def emit_stream_error(stream, request, error)
-
teardown(request)
-
stream.close
-
emit(:error, request, error)
-
end
-
-
1
def teardown(request = nil)
-
5
if request
-
5
@drains.delete(request)
-
5
@streams.delete(request)
-
else
-
@drains.clear
-
@streams.clear
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
#
-
# domain_name.rb - Domain Name manipulation library for Ruby
-
#
-
# Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
-
#
-
# Redistribution and use in source and binary forms, with or without
-
# modification, are permitted provided that the following conditions
-
# are met:
-
# 1. Redistributions of source code must retain the above copyright
-
# notice, this list of conditions and the following disclaimer.
-
# 2. Redistributions in binary form must reproduce the above copyright
-
# notice, this list of conditions and the following disclaimer in the
-
# documentation and/or other materials provided with the distribution.
-
#
-
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-
# SUCH DAMAGE.
-
-
1
require "ipaddr"
-
-
1
module HTTPX
-
# Represents a domain name ready for extracting its registered domain
-
# and TLD.
-
1
class DomainName
-
1
include Comparable
-
-
# The full host name normalized, ASCII-ized and downcased using the
-
# Unicode NFC rules and the Punycode algorithm. If initialized with
-
# an IP address, the string representation of the IP address
-
# suitable for opening a connection to.
-
1
attr_reader :hostname
-
-
# The Unicode representation of the #hostname property.
-
#
-
# :attr_reader: hostname_idn
-
-
# The least "universally original" domain part of this domain name.
-
# For example, "example.co.uk" for "www.sub.example.co.uk". This
-
# may be nil if the hostname does not have one, like when it is an
-
# IP address, an effective TLD or higher itself, or of a
-
# non-canonical domain.
-
1
attr_reader :domain
-
-
1
class << self
-
1
def new(domain)
-
return domain if domain.is_a?(self)
-
-
super
-
end
-
-
# Normalizes a _domain_ using the Punycode algorithm as necessary.
-
# The result will be a downcased, ASCII-only string.
-
1
def normalize(domain)
-
unless domain.ascii_only?
-
domain = domain.chomp(".").unicode_normalize(:nfc)
-
domain = Punycode.encode_hostname(domain)
-
end
-
-
domain.downcase
-
end
-
end
-
-
# Parses _hostname_ into a DomainName object. An IP address is also
-
# accepted. An IPv6 address may be enclosed in square brackets.
-
1
def initialize(hostname)
-
hostname = String(hostname)
-
-
raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
-
-
begin
-
@ipaddr = IPAddr.new(hostname)
-
@hostname = @ipaddr.to_s
-
return
-
rescue IPAddr::Error
-
nil
-
end
-
-
@hostname = DomainName.normalize(hostname)
-
tld = if (last_dot = @hostname.rindex("."))
-
@hostname[(last_dot + 1)..-1]
-
else
-
@hostname
-
end
-
-
# unknown/local TLD
-
@domain = if last_dot
-
# fallback - accept cookies down to second level
-
# cf. http://www.dkim-reputation.org/regdom-libs/
-
if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
-
@hostname[(penultimate_dot + 1)..-1]
-
else
-
@hostname
-
end
-
else
-
# no domain part - must be a local hostname
-
tld
-
end
-
end
-
-
# Checks if the server represented by this domain is qualified to
-
# send and receive cookies with a domain attribute value of
-
# _domain_. A true value given as the second argument represents
-
# cookies without a domain attribute value, in which case only
-
# hostname equality is checked.
-
1
def cookie_domain?(domain, host_only = false)
-
# RFC 6265 #5.3
-
# When the user agent "receives a cookie":
-
return self == @domain if host_only
-
-
domain = DomainName.new(domain)
-
-
# RFC 6265 #5.1.3
-
# Do not perform subdomain matching against IP addresses.
-
@hostname == domain.hostname if @ipaddr
-
-
# RFC 6265 #4.1.1
-
# Domain-value must be a subdomain.
-
@domain && self <= domain && domain <= @domain
-
end
-
-
1
def <=>(other)
-
other = DomainName.new(other)
-
othername = other.hostname
-
if othername == @hostname
-
0
-
elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
-
# The other is higher
-
-1
-
else
-
# The other is lower
-
1
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
# the default exception class for exceptions raised by HTTPX.
-
1
class Error < StandardError; end
-
-
1
class UnsupportedSchemeError < Error; end
-
-
1
class ConnectionError < Error; end
-
-
# Error raised when there was a timeout. Its subclasses allow for finer-grained
-
# control of which timeout happened.
-
1
class TimeoutError < Error
-
# The timeout value which caused this error to be raised.
-
1
attr_reader :timeout
-
-
# initializes the timeout exception with the +timeout+ causing the error, and the
-
# error +message+ for it.
-
1
def initialize(timeout, message)
-
4
@timeout = timeout
-
4
super(message)
-
end
-
-
# clones this error into a HTTPX::ConnectionTimeoutError.
-
1
def to_connection_error
-
ex = ConnectTimeoutError.new(@timeout, message)
-
ex.set_backtrace(backtrace)
-
ex
-
end
-
end
-
-
# Raise when it can't acquire a connection from the pool.
-
1
class PoolTimeoutError < TimeoutError; end
-
-
# Error raised when there was a timeout establishing the connection to a server.
-
# This may be raised due to timeouts during TCP and TLS (when applicable) connection
-
# establishment.
-
1
class ConnectTimeoutError < TimeoutError; end
-
-
# Error raised when there was a timeout while sending a request, or receiving a response
-
# from the server.
-
1
class RequestTimeoutError < TimeoutError
-
# The HTTPX::Request request object this exception refers to.
-
1
attr_reader :request
-
-
# initializes the exception with the +request+ and +response+ it refers to, and the
-
# +timeout+ causing the error, and the
-
1
def initialize(request, response, timeout)
-
@request = request
-
@response = response
-
super(timeout, "Timed out after #{timeout} seconds")
-
end
-
-
1
def marshal_dump
-
[message]
-
end
-
end
-
-
# Error raised when there was a timeout while receiving a response from the server.
-
1
class ReadTimeoutError < RequestTimeoutError; end
-
-
# Error raised when there was a timeout while sending a request from the server.
-
1
class WriteTimeoutError < RequestTimeoutError; end
-
-
# Error raised when a response couldn't be received for a request after multiple interactions.
-
# This error should not be retriable.
-
1
class TotalRequestTimeoutError < RequestTimeoutError; end
-
-
# Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
-
1
class SettingsTimeoutError < TimeoutError; end
-
-
# Error raised when there was a timeout while resolving a domain to an IP.
-
1
class ResolveTimeoutError < TimeoutError; end
-
-
# Error raised when there was a timeout waiting for readiness of the socket the request is related to.
-
1
class OperationTimeoutError < TimeoutError; end
-
-
# Error raised when a connection liveness probe (aka ping) times out.
-
1
class PingTimeoutError < TimeoutError; end
-
-
# Error raised when there was an error while resolving a domain to an IP.
-
1
class ResolveError < Error; end
-
-
# Error raised when there was an error while resolving a domain to an IP
-
# using a HTTPX::Resolver::Native resolver.
-
1
class NativeResolveError < ResolveError
-
1
attr_reader :host
-
-
1
attr_accessor :connection
-
-
# initializes the exception with the +connection+ it refers to, the +host+ domain
-
# which failed to resolve, and the error +message+.
-
1
def initialize(connection, host, message = "Can't resolve #{host}")
-
3
@connection = connection
-
3
@host = host
-
3
super(message)
-
end
-
end
-
-
# The exception class for HTTP responses with 4xx or 5xx status.
-
1
class HTTPError < Error
-
# The HTTPX::Response response object this exception refers to.
-
1
attr_reader :response
-
-
# Creates the instance and assigns the HTTPX::Response +response+.
-
1
def initialize(response)
-
5
@response = response
-
5
super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
-
end
-
-
# The HTTP response status.
-
#
-
# error.status #=> 404
-
1
def status
-
@response.status
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "uri"
-
-
1
module HTTPX
-
1
module ArrayExtensions
-
1
module Intersect
-
refine Array do
-
# Ruby 3.1 backport
-
def intersect?(arr)
-
if size < arr.size
-
smaller = self
-
else
-
smaller, arr = arr, self
-
end
-
(arr & smaller).size > 0
-
end
-
1
end unless Array.method_defined?(:intersect?)
-
end
-
end
-
-
1
module URIExtensions
-
# uri 0.11 backport, ships with ruby 3.1
-
1
refine URI::Generic do
-
-
1
def non_ascii_hostname
-
4
@non_ascii_hostname
-
end
-
-
1
def non_ascii_hostname=(hostname)
-
@non_ascii_hostname = hostname
-
end
-
-
def authority
-
return host if port == default_port
-
-
"#{host}:#{port}"
-
1
end unless URI::HTTP.method_defined?(:authority)
-
-
def origin
-
"#{scheme}://#{authority}"
-
1
end unless URI::HTTP.method_defined?(:origin)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
class Headers
-
1
class << self
-
1
def new(headers = nil)
-
118
return headers if headers.is_a?(self)
-
-
69
super
-
end
-
end
-
-
1
def initialize(headers = nil)
-
69
if headers.nil? || headers.empty?
-
33
@headers = headers.to_h
-
33
return
-
end
-
-
36
@headers = {}
-
-
36
headers.each do |field, value|
-
264
field = downcased(field)
-
-
264
value = array_value(value)
-
-
264
current = @headers[field]
-
-
264
if current.nil?
-
264
@headers[field] = value
-
else
-
current.concat(value)
-
end
-
end
-
end
-
-
# cloned initialization
-
1
def initialize_clone(orig, **kwargs)
-
super
-
@headers = orig.instance_variable_get(:@headers).clone(**kwargs)
-
end
-
-
# dupped initialization
-
1
def initialize_dup(orig)
-
98
super
-
98
@headers = orig.instance_variable_get(:@headers).transform_values(&:dup)
-
end
-
-
# freezes the headers hash
-
1
def freeze
-
171
@headers.each_value(&:freeze).freeze
-
171
super
-
end
-
-
# merges headers with another header-quack.
-
# the merge rule is, if the header already exists,
-
# ignore what the +other+ headers has. Otherwise, set
-
#
-
1
def merge(other)
-
10
headers = dup
-
10
other.each do |field, value|
-
74
headers[downcased(field)] = value
-
end
-
10
headers
-
end
-
-
# returns the comma-separated values of the header field
-
# identified by +field+, or nil otherwise.
-
#
-
1
def [](field)
-
263
a = @headers[downcased(field)] || return
-
99
a.join(", ")
-
end
-
-
# sets +value+ (if not nil) as single value for the +field+ header.
-
#
-
1
def []=(field, value)
-
210
return unless value
-
-
210
@headers[downcased(field)] = array_value(value)
-
end
-
-
# deletes all values associated with +field+ header.
-
#
-
1
def delete(field)
-
canonical = downcased(field)
-
@headers.delete(canonical) if @headers.key?(canonical)
-
end
-
-
# adds additional +value+ to the existing, for header +field+.
-
#
-
1
def add(field, value)
-
(@headers[downcased(field)] ||= []) << String(value)
-
end
-
-
# helper to be used when adding an header field as a value to another field
-
#
-
# h2_headers.add_header("vary", "accept-encoding")
-
# h2_headers["vary"] #=> "accept-encoding"
-
# h1_headers.add_header("vary", "accept-encoding")
-
# h1_headers["vary"] #=> "Accept-Encoding"
-
#
-
1
alias_method :add_header, :add
-
-
# returns the enumerable headers store in pairs of header field + the values in
-
# the comma-separated string format
-
#
-
1
def each(extra_headers = nil)
-
352
return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
-
-
184
@headers.each do |field, value|
-
1026
yield(field, value.join(", ")) unless value.empty?
-
end
-
-
34
extra_headers.each do |field, value|
-
78
yield(field, value) unless value.empty?
-
184
end if extra_headers
-
end
-
-
1
def ==(other)
-
18
other == to_hash
-
end
-
-
1
def empty?
-
@headers.empty?
-
end
-
-
# the headers store in Hash format
-
1
def to_hash
-
134
Hash[to_a]
-
end
-
1
alias_method :to_h, :to_hash
-
-
# the headers store in array of pairs format
-
1
def to_a
-
134
Array(each)
-
end
-
-
# headers as string
-
1
def to_s
-
6
@headers.to_s
-
end
-
-
# simplecov:disable
-
1
def inspect
-
1
"#<#{self.class}:#{object_id} " \
-
"#{to_hash.inspect}>"
-
end
-
# simplecov:enable
-
-
# this is internal API and doesn't abide to other public API
-
# guarantees, like downcasing strings.
-
# Please do not use this outside of core!
-
#
-
1
def key?(downcased_key)
-
360
@headers.key?(downcased_key)
-
end
-
-
# returns the values for the +field+ header in array format.
-
# This method is more internal, and for this reason doesn't try
-
# to "correct" the user input, i.e. it doesn't downcase the key.
-
#
-
1
def get(field)
-
1
@headers[field] || EMPTY
-
end
-
-
1
private
-
-
1
def array_value(value)
-
474
Array(value)
-
end
-
-
1
def downcased(field)
-
811
String(field).downcase
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "socket"
-
1
require "httpx/io/udp"
-
1
require "httpx/io/tcp"
-
1
require "httpx/io/unix"
-
-
begin
-
1
require "httpx/io/ssl"
-
rescue LoadError
-
end
-
# frozen_string_literal: true
-
-
1
require "openssl"
-
-
1
module HTTPX
-
1
TLSError = OpenSSL::SSL::SSLError
-
-
1
class SSL < TCP
-
1
tls_options = { alpn_protocols: %w[h2 http/1.1].freeze }
-
# TODO: remove when dropping support for jruby-openssl < 0.15.4
-
# https://github.com/jruby/jruby-openssl/issues/284
-
1
tls_options[:verify_hostname] = true if RUBY_ENGINE == "jruby" && JOpenSSL::VERSION < "0.15.4"
-
1
TLS_OPTIONS = tls_options.freeze
-
-
1
attr_writer :ssl_session
-
-
1
def initialize(_, _, options)
-
4
super
-
-
4
@ssl_session = @session_new_cb = nil
-
-
4
ctx_options = TLS_OPTIONS
-
4
ctx_options = ctx_options.merge(options.ssl) if options.ssl && !options.ssl.empty?
-
4
@sni_hostname = (ctx_options.delete(:hostname) if ctx_options.key?(:hostname)) || @hostname
-
-
4
if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
-
# externally initiated ssl socket
-
@ctx = @io.context
-
@state = :negotiated
-
else
-
4
@ctx = OpenSSL::SSL::SSLContext.new
-
4
@ctx.set_params(ctx_options)
-
4
unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
-
4
@ctx.session_cache_mode =
-
OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
-
end
-
4
init_session_new_cb
-
-
4
yield(self) if block_given?
-
end
-
-
4
@verify_hostname = @ctx.verify_hostname
-
end
-
-
1
if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
-
# sets the ssl session callback to be picked up by the ssl context.
-
1
def session_new_cb(&pr)
-
4
@session_new_cb = pr
-
end
-
-
# sets the ssl context's new session callback, which points at @session_new_cb when available.
-
1
def init_session_new_cb
-
12
@ctx.session_new_cb = proc { |_, sess| @session_new_cb&.call(sess) }
-
end
-
else
-
# session_new_cb not implemented under JRuby
-
def session_new_cb; end
-
-
def init_session_new_cb; end
-
end
-
-
1
private :init_session_new_cb
-
-
1
def protocol
-
4
return super unless @io.is_a?(OpenSSL::SSL::SSLSocket)
-
-
4
@io.alpn_protocol || super
-
end
-
-
1
if RUBY_ENGINE == "jruby"
-
# in jruby, alpn_protocol may return ""
-
# https://github.com/jruby/jruby-openssl/issues/287
-
def protocol
-
return super unless @io.is_a?(OpenSSL::SSL::SSLSocket)
-
-
proto = @io.alpn_protocol
-
-
return super if proto.nil? || proto.empty?
-
-
proto
-
end
-
end
-
-
1
def can_verify_peer?
-
@ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
-
end
-
-
1
def verify_hostname(host)
-
return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
return false if !@io.respond_to?(:peer_cert) || (peer_cert = @io.peer_cert).nil?
-
-
OpenSSL::SSL.verify_certificate_identity(peer_cert, host)
-
end
-
-
1
def connected?
-
12
@state == :negotiated
-
end
-
-
1
def ssl_session_expired?
-
4
ssl_session = @ssl_session
-
-
4
ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (ssl_session.time.to_f + ssl_session.timeout)
-
end
-
-
1
def connect
-
12
return if @state == :negotiated
-
-
12
unless @state == :connected
-
8
super
-
8
return unless @state == :connected
-
end
-
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
-
8
unless @io.is_a?(OpenSSL::SSL::SSLSocket)
-
4
if (hostname_is_ip = (@ip == @sni_hostname)) && @ctx.verify_hostname
-
# IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
-
@sni_hostname = @ip.to_string
-
# IP addresses in SNI is not valid per RFC 6066, section 3.
-
@ctx.verify_hostname = false
-
end
-
-
4
ssl = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
-
-
4
ssl.hostname = @sni_hostname unless hostname_is_ip
-
4
ssl.session = @ssl_session unless ssl_session_expired?
-
4
ssl.sync_close = true
-
-
4
@io = ssl
-
end
-
8
try_ssl_connect
-
end
-
-
1
def try_ssl_connect
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
8
ret = @io.connect_nonblock(exception: false)
-
8
log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
-
8
case ret
-
when :wait_readable
-
4
@interests = :r
-
4
return
-
when :wait_writable
-
@interests = :w
-
return
-
end
-
4
@io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
-
4
transition(:negotiated)
-
4
@interests = :w
-
end
-
-
1
private
-
-
1
def transition(nextstate)
-
16
case nextstate
-
when :negotiated
-
4
return unless @state == :connected
-
-
when :closed
-
4
return unless @state == :negotiated ||
-
@state == :connected
-
end
-
16
do_transition(nextstate)
-
end
-
-
1
def log_transition_state(nextstate)
-
return super unless nextstate == :negotiated
-
-
# @type ivar @io: OpenSSL::SSL::SSLSocket
-
-
server_cert = @io.peer_cert #: OpenSSL::X509::Certificate
-
-
"#{super}\n\n" \
-
"SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
-
"ALPN, server accepted to use #{protocol}\n" \
-
"Server certificate:\n " \
-
"subject: #{log_redact(server_cert.subject)}\n " \
-
"start date: #{log_redact(server_cert.not_before)}\n " \
-
"expire date: #{log_redact(server_cert.not_after)}\n " \
-
"issuer: #{log_redact(server_cert.issuer)}\n " \
-
"SSL certificate verify ok."
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "resolv"
-
-
1
module HTTPX
-
1
class TCP
-
1
include Loggable
-
-
1
using URIExtensions
-
-
1
attr_reader :ip, :port, :addresses, :state, :interests
-
-
1
alias_method :host, :ip
-
-
1
def initialize(origin, addresses, options)
-
29
@state = :idle
-
29
@keep_open = false
-
29
@addresses = []
-
29
@ip_index = -1
-
29
@ip = nil
-
29
@hostname = origin.host
-
29
@options = options
-
29
@fallback_protocol = @options.fallback_protocol
-
29
@port = origin.port
-
29
@interests = :w
-
29
if (io = @options.io)
-
io =
-
case io
-
when Hash
-
io[origin.authority]
-
else
-
io
-
end
-
raise Error, "Given IO objects do not match the request authority" unless io
-
-
# @type var io: TCPSocket | OpenSSL::SSL::SSLSocket
-
-
_, _, _, ip = io.addr
-
@io = io
-
@addresses << (@ip = Resolver::Entry.new(ip))
-
@keep_open = true
-
@state = :connected
-
else
-
29
add_addresses(addresses)
-
end
-
29
@ip_index = @addresses.size - 1
-
end
-
-
1
def socket
-
@io
-
end
-
-
1
def add_addresses(addrs)
-
29
return if addrs.empty?
-
-
29
ip_index = @ip_index || (@addresses.size - 1)
-
29
if addrs.first.ipv6?
-
# should be the next in line
-
@addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
-
else
-
29
@addresses.unshift(*addrs)
-
end
-
29
@ip_index += addrs.size
-
end
-
-
# eliminates expired entries and returns whether there are still any left.
-
1
def addresses?
-
2
prev_addr_size = @addresses.size
-
-
2
@addresses.delete_if(&:expired?).sort! do |addr1, addr2|
-
if addr1.ipv6?
-
addr2.ipv6? ? 0 : 1
-
else
-
addr2.ipv6? ? -1 : 0
-
end
-
end
-
-
2
@ip_index = @addresses.size - 1 if prev_addr_size != @addresses.size
-
-
2
@addresses.any?
-
end
-
-
1
def to_io
-
69
@io.to_io
-
end
-
-
1
def protocol
-
28
@fallback_protocol
-
end
-
-
1
def connect
-
64
return unless closed?
-
-
64
if @addresses.empty?
-
# an idle connection trying to connect with no available addresses is a connection
-
# out of the initial context which is back to the DNS resolution loop. This may
-
# happen in a fiber-aware context where a connection reconnects with expired addresses,
-
# and context is passed back to a fiber on the same connection while waiting for the
-
# DNS answer.
-
log { "tried connecting while resolving, skipping..." }
-
-
return
-
end
-
-
64
if !@io || @io.closed?
-
32
transition(:idle)
-
32
@io = build_socket
-
end
-
64
try_connect
-
rescue Errno::EHOSTUNREACH,
-
Errno::ENETUNREACH => e
-
@ip_index -= 1
-
-
raise e if @ip_index.negative?
-
-
log { "failed connecting to #{@ip} (#{e.message}), evict from cache and trying next..." }
-
@options.resolver_cache.evict(@hostname, @ip)
-
-
@io = build_socket
-
retry
-
rescue Errno::ECONNREFUSED,
-
Errno::EADDRNOTAVAIL,
-
SocketError,
-
IOError => e
-
@ip_index -= 1
-
-
raise e if @ip_index.negative?
-
-
log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
-
@io = build_socket
-
retry
-
rescue Errno::ETIMEDOUT => e
-
@ip_index -= 1
-
-
raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index.negative?
-
-
log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
-
-
@io = build_socket
-
retry
-
end
-
-
1
def try_connect
-
64
ret = @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
-
64
log(level: 3, color: :cyan) { "TCP CONNECT: #{ret}..." }
-
64
case ret
-
when :wait_readable
-
@interests = :r
-
return
-
when :wait_writable
-
32
@interests = :w
-
32
return
-
end
-
32
transition(:connected)
-
32
@interests = :w
-
rescue Errno::EALREADY
-
@interests = :w
-
end
-
1
private :try_connect
-
-
1
def read(size, buffer)
-
84
ret = @io.read_nonblock(size, buffer, exception: false)
-
84
if ret == :wait_readable
-
37
buffer.clear
-
37
return 0
-
end
-
47
return if ret.nil?
-
-
47
log { "READ: #{buffer.bytesize} bytes..." }
-
47
buffer.bytesize
-
end
-
-
1
def write(buffer)
-
42
siz = @io.write_nonblock(buffer, exception: false)
-
42
return 0 if siz == :wait_writable
-
42
return if siz.nil?
-
-
42
log { "WRITE: #{siz} bytes..." }
-
-
42
buffer.shift!(siz)
-
42
siz
-
end
-
-
1
def close
-
35
return if @keep_open
-
-
# mark tcp as closed, so that it can be disconnected.
-
# this bypasses the state machine API as not not allow the transition
-
# from idle to closed in normal circumstances.
-
35
@state = :closed if @state == :idle
-
-
35
return if closed?
-
-
begin
-
32
@io.close
-
rescue IOError => e
-
log { "error closing socket" }
-
log { e.full_message(highlight: false) }
-
ensure
-
# @fiber-switch-guard
-
# ensure that all :closed IOs don't leave dangling sockets
-
# behind. This may happen in a fiber scheduler scenario where
-
# connection is reused across fibers.
-
32
transition(:closed) if @io.closed?
-
end
-
end
-
-
# signals that the connection that contains this IO can be checked back into the pool.
-
# that includes sockets opened outside of the scope of the session, or closed IOs.
-
1
def can_disconnect?
-
67
@keep_open || @state == :closed
-
end
-
-
1
def connected?
-
56
@state == :connected
-
end
-
-
1
def closed?
-
99
@state == :idle || @state == :closed
-
end
-
-
# simplecov:disable
-
1
def inspect
-
"#<#{self.class}:#{object_id} " \
-
"#{@ip}:#{@port} " \
-
"@state=#{@state} " \
-
"@hostname=#{@hostname} " \
-
"@addresses=#{@addresses} " \
-
"@state=#{@state}>"
-
end
-
# simplecov:enable
-
-
1
private
-
-
1
def build_socket
-
32
@ip = @addresses[@ip_index]
-
32
Socket.new(@ip.family, :STREAM, 0)
-
end
-
-
1
def transition(nextstate)
-
84
case nextstate
-
# when :idle
-
when :connected
-
28
return unless @state == :idle
-
when :closed
-
28
return unless @state == :connected
-
end
-
84
do_transition(nextstate)
-
end
-
-
1
def do_transition(nextstate)
-
100
log(level: 1) { log_transition_state(nextstate) }
-
100
@state = nextstate
-
end
-
-
1
def log_transition_state(nextstate)
-
label = host
-
label = "#{label}(##{@io.fileno})" if nextstate == :connected
-
"#{label} #{@state} -> #{nextstate}"
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "ipaddr"
-
-
1
module HTTPX
-
1
class UDP
-
1
include Loggable
-
-
1
def initialize(ip, port, options)
-
4
@host = ip
-
4
@port = port
-
4
@io = UDPSocket.new(IPAddr.new(ip).family)
-
4
@options = options
-
end
-
-
1
def to_io
-
10
@io.to_io
-
end
-
-
1
def connect; end
-
-
1
def connected?
-
4
true
-
end
-
-
1
def close
-
4
@io.close
-
end
-
-
1
if RUBY_ENGINE == "jruby"
-
# In JRuby, sendmsg_nonblock is not implemented
-
def write(buffer)
-
siz = @io.send(buffer.to_s, 0, @host, @port)
-
log { "WRITE: #{siz} bytes..." }
-
buffer.shift!(siz)
-
siz
-
end
-
else
-
1
def write(buffer)
-
10
siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
-
10
return 0 if siz == :wait_writable
-
10
return if siz.nil?
-
-
10
log { "WRITE: #{siz} bytes..." }
-
-
10
buffer.shift!(siz)
-
10
siz
-
end
-
end
-
-
1
def read(size, buffer)
-
20
ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
-
20
return 0 if ret == :wait_readable
-
10
return if ret.nil?
-
-
10
log { "READ: #{buffer.bytesize} bytes..." }
-
-
10
buffer.bytesize
-
rescue IOError
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
class UNIX < TCP
-
1
using URIExtensions
-
-
1
attr_reader :path
-
-
1
alias_method :host, :path
-
-
1
def initialize(origin, path, options)
-
@addresses = []
-
@hostname = origin.host
-
@state = :idle
-
@options = options
-
@fallback_protocol = @options.fallback_protocol
-
if (io = @options.io)
-
io =
-
case io
-
when Hash
-
io[origin.authority]
-
else
-
io
-
end
-
raise Error, "Given IO objects do not match the request authority" unless io
-
-
# @type var io: UNIXSocket
-
-
_, @path = io.addr
-
@io = io
-
@keep_open = true
-
@state = :connected
-
elsif path
-
@path = path
-
else
-
raise Error, "No path given where to store the socket"
-
end
-
@io ||= build_socket
-
end
-
-
1
def connect
-
return unless closed?
-
-
begin
-
if @io.closed?
-
transition(:idle)
-
@io = build_socket
-
end
-
@io.connect_nonblock(Socket.sockaddr_un(@path))
-
rescue Errno::EISCONN
-
end
-
transition(:connected)
-
rescue Errno::EINPROGRESS,
-
Errno::EALREADY,
-
IO::WaitReadable
-
end
-
-
# the path is always explicitly passed, so no point in resolving.
-
1
def addresses?
-
true
-
end
-
-
# simplecov:disable
-
1
def inspect
-
"#<#{self.class}:#{object_id} @path=#{@path}) @state=#{@state})>"
-
end
-
# simplecov:enable
-
-
1
private
-
-
1
def build_socket
-
Socket.new(Socket::PF_UNIX, :STREAM, 0)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "fiber" if RUBY_VERSION < "3.0.0"
-
-
1
module HTTPX
-
1
module Loggable
-
1
COLORS = {
-
black: 30,
-
red: 31,
-
green: 32,
-
yellow: 33,
-
blue: 34,
-
magenta: 35,
-
cyan: 36,
-
white: 37,
-
}.freeze
-
-
1
USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
-
-
1
def self.log_identifiers
-
"pid=#{Process.pid} " \
-
"tid=#{Thread.current.object_id} " \
-
"fid=#{Fiber.current.object_id}"
-
end
-
-
1
def log(
-
level: @options.debug_level,
-
color: nil,
-
debug_level: @options.debug_level,
-
debug: @options.debug,
-
&msg
-
)
-
3083
return unless debug_level >= level
-
-
1358
debug_stream = debug || ($stderr if USE_DEBUG_LOG)
-
-
1358
return unless debug_stream
-
-
klass = self.class
-
-
until (class_name = klass.name)
-
klass = klass.superclass
-
end
-
-
message = +"(time=#{Time.now.utc} " \
-
"#{Loggable.log_identifiers} " \
-
"self=#{class_name}##{object_id}) "
-
message << msg.call << "\n"
-
message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
-
debug_stream << message
-
end
-
-
1
def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
-
7
log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
-
end
-
-
1
private
-
-
1
def log_redact_headers(text)
-
log_redact(text, @options.debug_redact == :headers)
-
end
-
-
1
def log_redact_body(text)
-
log_redact(text, @options.debug_redact == :body)
-
end
-
-
1
def log_redact(text, should_redact = nil)
-
should_redact ||= @options.debug_redact == true
-
-
return text.to_s unless should_redact
-
-
"[REDACTED]"
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
# Contains a set of options which are passed and shared across from session to its requests or
-
# responses.
-
1
class Options
-
1
BUFFER_SIZE = 1 << 14
-
1
WINDOW_SIZE = 1 << 14 # 16K
-
1
MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
-
1
KEEP_ALIVE_TIMEOUT = 20
-
1
PING_TIMEOUT = 2
-
1
SETTINGS_TIMEOUT = 10
-
1
CLOSE_HANDSHAKE_TIMEOUT = 10
-
1
CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
-
1
REQUEST_TIMEOUT = OPERATION_TIMEOUT = TOTAL_REQUEST_TIMEOUT = nil
-
1
RESOLVER_TYPES = %i[memory file].freeze
-
# default value used for "user-agent" header, when not overridden.
-
1
USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
-
-
1
@options_names = []
-
-
1
class << self
-
1
attr_reader :options_names
-
-
1
def inherited(klass)
-
1
super
-
1
klass.instance_variable_set(:@options_names, @options_names.dup)
-
end
-
-
1
def new(options = {})
-
# let enhanced options go through
-
84
return options if self == Options && options.class < self
-
58
return options if options.is_a?(self)
-
-
18
super
-
end
-
-
1
def freeze
-
106
@options_names.freeze
-
106
super
-
end
-
-
1
def method_added(meth)
-
237
super
-
-
237
return unless meth =~ /^option_(.+)$/
-
-
110
optname = Regexp.last_match(1) #: String
-
-
110
if optname =~ /^(.+[^_])_+with/
-
# ignore alias method chain generated methods.
-
# this is the case with RBS runtime tests.
-
# it relies on the "_with/_without" separator, which is the most used convention,
-
# however it shouldn't be used in practice in httpx given the plugin architecture
-
# as the main extension API.
-
orig_name = Regexp.last_match(1) #: String
-
-
return if @options_names.include?(orig_name.to_sym)
-
end
-
-
110
optname = optname.to_sym
-
-
110
attr_reader(optname) unless method_defined?(optname)
-
-
110
@options_names << optname unless @options_names.include?(optname)
-
end
-
end
-
-
# creates a new options instance from a given hash, which optionally define the following:
-
#
-
# :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
-
# :debug_level :: the log level of messages (can be 1, 2, or 3).
-
# :debug_redact :: whether header/body payload should be redacted (defaults to <tt>false</tt>).
-
# :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::SSL)
-
# :http2_settings :: a hash of options to be passed to a HTTP2::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
-
# :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
-
# like ALPN (defaults to <tt>"http/1.1"</tt>)
-
# :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
-
# :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
-
# :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
-
# :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
-
# <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>, <tt>:read_timeout</tt>, <tt>:write_timeout</tt>,
-
# <tt>:request_timeout</tt>, <tt>:total_request_timeout</tt> and <tt>:ping_timeout</tt>,
-
# :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
-
# :max_response_body_size :: maximum size (in bytes) that the response body can consume (no threshold by default), after which an
-
# error is raised.
-
# :max_response_headers :: maximum number of header fields that a response can receive, after which an error is raised.
-
# :max_response_header_value_size :: maximum size (in bytes) a header value can have (no threshold by default).
-
# for cases where the value is broken into multiple header fields (such as "cookie" or "set-cookie"),
-
# this is the total aggregated size.
-
# :window_size :: number of bytes to read from a socket
-
# :buffer_size :: internal read and write buffer size in bytes
-
# :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
-
# :request_class :: class used to instantiate a request
-
# :response_class :: class used to instantiate a response
-
# :headers_class :: class used to instantiate headers
-
# :request_body_class :: class used to instantiate a request body
-
# :response_body_class :: class used to instantiate a response body
-
# :connection_class :: class used to instantiate connections
-
# :http1_class :: class used to manage HTTP1 sessions
-
# :http2_class :: class used to imanage HTTP2 sessions
-
# :resolver_native_class :: class used to resolve names using pure ruby DNS implementation
-
# :resolver_system_class :: class used to resolve names using system-based (getaddrinfo) name resolution
-
# :resolver_https_class :: class used to resolve names using DoH
-
# :pool_class :: class used to instantiate the session connection pool
-
# :options_class :: class used to instantiate options
-
# :transport :: type of transport to use (set to "unix" for UNIX sockets)
-
# :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
-
# paths should be used for UNIX sockets instead)
-
# :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
-
# :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
-
# :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
-
# using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class inheriting
-
# from HTTPX::Resolver::Resolver)
-
# :resolver_cache :: strategy to cache DNS results, ignored by the <tt>:system</tt> resolver, can be set to <tt>:memory<tt>
-
# or an instance of a custom class inheriting from HTTPX::Resolver::Cache::Base
-
# :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
-
# :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
-
# :ip_families :: which socket families are supported (system-dependent)
-
# :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
-
# :base_path :: path to prefix given relative paths with (ex: "/v2")
-
# :max_concurrent_requests :: max number of requests which can be set concurrently
-
# :max_requests :: max number of requests which can be made on socket before it reconnects.
-
# :close_on_fork :: whether the session automatically closes when the process is fork (defaults to <tt>false</tt>).
-
# it only works if the session is persistent (and ruby 3.1 or higher is used).
-
#
-
# This list of options are enhanced with each loaded plugin, see the plugin docs for details.
-
1
def initialize(options = EMPTY_HASH)
-
18
options_names = self.class.options_names
-
-
defaults =
-
18
case options
-
when Options
-
16
unknown_options = options.class.options_names - options_names
-
-
16
raise Error, "unknown option: #{unknown_options.first}" unless unknown_options.empty?
-
-
16
DEFAULT_OPTIONS.merge(options)
-
else
-
2
options.each_key do |k|
-
37
raise Error, "unknown option: #{k}" unless options_names.include?(k)
-
end
-
-
2
options.empty? ? DEFAULT_OPTIONS : DEFAULT_OPTIONS.merge(options)
-
end
-
-
18
options_names.each do |k|
-
869
v = defaults[k]
-
-
869
if v.nil?
-
236
instance_variable_set(:"@#{k}", v)
-
-
236
next
-
end
-
-
633
value = __send__(:"option_#{k}", v)
-
633
instance_variable_set(:"@#{k}", value)
-
end
-
-
18
do_initialize
-
18
freeze
-
end
-
-
# returns the class with which to instantiate the DNS resolver.
-
1
def resolver_class
-
50
case @resolver_class
-
when Symbol
-
50
public_send(:"resolver_#{@resolver_class}_class")
-
else
-
@resolver_class
-
end
-
end
-
-
1
def resolver_cache
-
51
cache_type = @resolver_cache
-
-
51
case cache_type
-
when :memory
-
51
Resolver::Cache::Memory.cache(cache_type)
-
when :file
-
Resolver::Cache::File.cache(cache_type)
-
else
-
unless cache_type.respond_to?(:resolve) &&
-
cache_type.respond_to?(:get) &&
-
cache_type.respond_to?(:set) &&
-
cache_type.respond_to?(:evict)
-
raise TypeError, ":resolver_cache must be a compatible resolver cache and implement #get, #set and #evict"
-
end
-
-
cache_type #: Object & Resolver::_Cache
-
end
-
end
-
-
1
def freeze
-
104
self.class.options_names.each do |ivar|
-
# avoid freezing debug option, as when it's set, it's usually an
-
# object which cannot be frozen, like stderr or stdout. It's a
-
# documented exception then, and still does not defeat the purpose
-
# here, which is to make option objects shareable across ractors,
-
# and in most cases debug should be nil, or one of the objects
-
# which will eventually be shareable, like STDOUT or STDERR.
-
4815
next if ivar == :debug
-
-
4711
instance_variable_get(:"@#{ivar}").freeze
-
end
-
104
super
-
end
-
-
1
REQUEST_BODY_IVARS = %i[@headers].freeze
-
-
# checks whether +other+ matches the same connection-level options
-
1
def connection_options_match?(other, ignore_ivars = nil)
-
9
return true if self == other
-
-
# headers and other request options do not play a role, as they are
-
# relevant only for the request.
-
1
ivars = instance_variables
-
51
ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
-
1
ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
-
-
1
other_ivars = other.instance_variables
-
51
other_ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
-
1
other_ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
-
-
1
return false if ivars.size != other_ivars.size
-
-
1
return false if ivars.sort != other_ivars.sort
-
-
1
ivars.all? do |ivar|
-
49
instance_variable_get(ivar) == other.instance_variable_get(ivar)
-
end
-
end
-
-
1
RESOLVER_IVARS = %i[
-
@resolver_class @resolver_cache @resolver_options
-
@resolver_native_class @resolver_system_class @resolver_https_class
-
].freeze
-
-
# checks whether +other+ matches the same resolver-level options
-
1
def resolver_options_match?(other)
-
self == other ||
-
RESOLVER_IVARS.all? do |ivar|
-
instance_variable_get(ivar) == other.instance_variable_get(ivar)
-
end
-
end
-
-
# returns a HTTPX::Options instance resulting of the merging of +other+ with self.
-
# it may return self if +other+ is self or equal to self.
-
1
def merge(other)
-
383
if (is_options = other.is_a?(Options))
-
-
159
return self if eql?(other)
-
-
9
opts_names = other.class.options_names
-
-
450
return self if opts_names.all? { |opt| public_send(opt) == other.public_send(opt) }
-
-
other_opts = opts_names
-
else
-
224
other_opts = other # : Hash[Symbol, untyped]
-
224
other_opts = Hash[other] unless other.is_a?(Hash)
-
-
224
return self if other_opts.empty?
-
-
138
return self if other_opts.all? { |opt, v| !respond_to?(opt) || public_send(opt) == v }
-
end
-
-
56
opts = dup
-
-
56
other_opts.each do |opt, v|
-
84
next unless respond_to?(opt)
-
-
84
v = other.public_send(opt) if is_options
-
84
ivar = :"@#{opt}"
-
-
84
unless v
-
3
opts.instance_variable_set(ivar, v)
-
3
next
-
end
-
-
81
v = opts.__send__(:"option_#{opt}", v)
-
-
81
orig_v = public_send(opt)
-
-
81
v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
-
-
81
opts.instance_variable_set(ivar, v)
-
end
-
-
56
opts
-
end
-
-
1
def to_hash
-
17
instance_variables.each_with_object({}) do |ivar, hs|
-
759
val = instance_variable_get(ivar)
-
-
759
next if val.nil?
-
-
598
hs[ivar[1..-1].to_sym] = val
-
end
-
end
-
-
1
def extend_with_plugin_classes(pl)
-
# extend request class
-
85
if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
-
30
@request_class = @request_class.dup
-
30
SET_TEMPORARY_NAME[@request_class, pl]
-
30
@request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
-
30
@request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
-
end
-
# extend response class
-
85
if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
-
60
@response_class = @response_class.dup
-
60
SET_TEMPORARY_NAME[@response_class, pl]
-
60
@response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
-
60
@response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
-
end
-
# extend headers class
-
85
if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
-
@headers_class = @headers_class.dup
-
SET_TEMPORARY_NAME[@headers_class, pl]
-
@headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
-
@headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
-
end
-
# extend request body class
-
85
if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
-
@request_body_class = @request_body_class.dup
-
SET_TEMPORARY_NAME[@request_body_class, pl]
-
@request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
-
@request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
-
end
-
# extend response body class
-
85
if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
-
40
@response_body_class = @response_body_class.dup
-
40
SET_TEMPORARY_NAME[@response_body_class, pl]
-
40
@response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
-
40
@response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
-
end
-
# extend connection pool class
-
85
if defined?(pl::PoolMethods)
-
@pool_class = @pool_class.dup
-
SET_TEMPORARY_NAME[@pool_class, pl]
-
@pool_class.__send__(:include, pl::PoolMethods)
-
end
-
# extend connection class
-
85
if defined?(pl::ConnectionMethods)
-
52
@connection_class = @connection_class.dup
-
52
SET_TEMPORARY_NAME[@connection_class, pl]
-
52
@connection_class.__send__(:include, pl::ConnectionMethods)
-
end
-
# extend http1 class
-
85
if defined?(pl::HTTP1Methods)
-
12
@http1_class = @http1_class.dup
-
12
SET_TEMPORARY_NAME[@http1_class, pl]
-
12
@http1_class.__send__(:include, pl::HTTP1Methods)
-
end
-
# extend http2 class
-
85
if defined?(pl::HTTP2Methods)
-
12
@http2_class = @http2_class.dup
-
12
SET_TEMPORARY_NAME[@http2_class, pl]
-
12
@http2_class.__send__(:include, pl::HTTP2Methods)
-
end
-
# extend native resolver class
-
85
if defined?(pl::ResolverNativeMethods)
-
12
@resolver_native_class = @resolver_native_class.dup
-
12
SET_TEMPORARY_NAME[@resolver_native_class, pl]
-
12
@resolver_native_class.__send__(:include, pl::ResolverNativeMethods)
-
end
-
# extend system resolver class
-
85
if defined?(pl::ResolverSystemMethods)
-
12
@resolver_system_class = @resolver_system_class.dup
-
12
SET_TEMPORARY_NAME[@resolver_system_class, pl]
-
12
@resolver_system_class.__send__(:include, pl::ResolverSystemMethods)
-
end
-
# extend https resolver class
-
85
if defined?(pl::ResolverHTTPSMethods)
-
@resolver_https_class = @resolver_https_class.dup
-
SET_TEMPORARY_NAME[@resolver_https_class, pl]
-
@resolver_https_class.__send__(:include, pl::ResolverHTTPSMethods)
-
end
-
-
85
return unless defined?(pl::OptionsMethods)
-
-
# extend option class
-
# works around lack of initialize_dup callback
-
16
@options_class = @options_class.dup
-
# (self.class.options_names)
-
16
@options_class.__send__(:include, pl::OptionsMethods)
-
end
-
-
1
private
-
-
# number options
-
1
%i[
-
max_concurrent_requests max_requests window_size buffer_size
-
max_response_body_size max_response_headers max_response_header_value_size
-
body_threshold_size debug_level
-
].each do |option|
-
9
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
# converts +v+ into an Integer before setting the +#{option}+ option.
-
private def option_#{option}(value) # private def option_max_requests(v)
-
value = Integer(value) unless value.respond_to?(:infinite?) && value.infinite?
-
raise TypeError, ":#{option} must be positive" unless value.positive? # raise TypeError, ":max_requests must be positive" unless value.positive?
-
-
value
-
end
-
OUT
-
end
-
-
# hashable options
-
1
%i[ssl http2_settings resolver_options pool_options].each do |option|
-
4
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
# converts +v+ into an Hash before setting the +#{option}+ option.
-
private def option_#{option}(value) # def option_ssl(v)
-
Hash[value]
-
end
-
OUT
-
end
-
-
1
%i[
-
request_class response_class headers_class request_body_class
-
response_body_class connection_class http1_class http2_class
-
resolver_native_class resolver_system_class resolver_https_class options_class pool_class
-
io fallback_protocol debug debug_redact
-
compress_request_body decompress_response_body
-
persistent close_on_fork
-
].each do |method_name|
-
21
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
# sets +v+ as the value of the +#{method_name}+ option
-
private def option_#{method_name}(v); v; end # private def option_smth(v); v; end
-
OUT
-
end
-
-
1
def option_origin(value)
-
URI(value)
-
end
-
-
1
def option_base_path(value)
-
String(value)
-
end
-
-
1
def option_headers(value)
-
18
value = value.dup if value.frozen?
-
-
18
headers_class.new(value)
-
end
-
-
1
def option_timeout(value)
-
42
timeout_hash = Hash[value]
-
-
42
default_timeouts = DEFAULT_OPTIONS[:timeout]
-
-
# Validate keys and values
-
42
timeout_hash.each do |key, val|
-
192
raise TypeError, "invalid timeout: :#{key}" unless default_timeouts.key?(key)
-
-
192
next if val.nil?
-
-
139
raise TypeError, ":#{key} must be numeric" unless val.is_a?(Numeric)
-
end
-
-
42
timeout_hash
-
end
-
-
1
def option_supported_compression_formats(value)
-
18
Array(value).map(&:to_s)
-
end
-
-
1
def option_transport(value)
-
transport = value.to_s
-
raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
-
-
transport
-
end
-
-
1
def option_addresses(value)
-
Array(value).map { |entry| Resolver::Entry.convert(entry) }
-
end
-
-
1
def option_ip_families(value)
-
Array(value)
-
end
-
-
1
def option_resolver_class(resolver_type)
-
18
case resolver_type
-
when Symbol
-
18
meth = :"resolver_#{resolver_type}_class"
-
-
18
raise TypeError, ":resolver_class must be a supported type" unless respond_to?(meth)
-
-
18
resolver_type
-
when Class
-
raise TypeError, ":resolver_class must be a subclass of `#{Resolver::Resolver}`" unless resolver_type < Resolver::Resolver
-
-
resolver_type
-
else
-
raise TypeError, ":resolver_class must be a supported type"
-
end
-
end
-
-
1
def option_resolver_cache(cache_type)
-
18
if cache_type.is_a?(Symbol)
-
18
raise TypeError, ":resolver_cache: #{cache_type} is invalid" unless RESOLVER_TYPES.include?(cache_type)
-
-
18
require "httpx/resolver/cache/file" if cache_type == :file
-
-
else
-
unless cache_type.respond_to?(:resolve) &&
-
cache_type.respond_to?(:get) &&
-
cache_type.respond_to?(:set) &&
-
cache_type.respond_to?(:evict)
-
raise TypeError, ":resolver_cache must be a compatible resolver cache and implement #resolve, #get, #set and #evict"
-
end
-
end
-
-
18
cache_type
-
end
-
-
# called after all options are initialized
-
1
def do_initialize
-
18
hs = @headers
-
-
# initialized default request headers
-
18
hs["user-agent"] = USER_AGENT unless hs.key?("user-agent")
-
18
hs["accept"] = "*/*" unless hs.key?("accept")
-
18
if hs.key?("range")
-
hs.delete("accept-encoding")
-
else
-
18
hs["accept-encoding"] = supported_compression_formats unless hs.key?("accept-encoding")
-
end
-
end
-
-
1
def access_option(obj, k, ivar_map)
-
case obj
-
when Hash
-
obj[ivar_map[k]]
-
else
-
obj.instance_variable_get(k)
-
end
-
end
-
-
# rubocop:disable Lint/UselessConstantScoping
-
# these really need to be defined at the end of the class
-
1
SET_TEMPORARY_NAME = ->(klass, pl = nil) do
-
246
if klass.respond_to?(:set_temporary_name) # ruby 3.4 only
-
name = klass.name || "#{klass.superclass.name}(plugin)"
-
name = "#{name}/#{pl}" if pl
-
klass.set_temporary_name(name)
-
end
-
end
-
-
DEFAULT_OPTIONS = {
-
1
:max_requests => Float::INFINITY,
-
:debug => nil,
-
1
:debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
-
:debug_redact => ENV.key?("HTTPX_DEBUG_REDACT"),
-
:ssl => EMPTY_HASH,
-
:http2_settings => { settings_enable_push: 0 }.freeze,
-
:fallback_protocol => "http/1.1",
-
:supported_compression_formats => %w[gzip deflate],
-
:decompress_response_body => true,
-
:compress_request_body => true,
-
:max_response_headers => 1000,
-
:max_response_header_value_size => nil,
-
:max_response_body_size => Float::INFINITY,
-
:timeout => {
-
connect_timeout: CONNECT_TIMEOUT,
-
settings_timeout: SETTINGS_TIMEOUT,
-
close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
-
operation_timeout: OPERATION_TIMEOUT,
-
keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
-
ping_timeout: PING_TIMEOUT,
-
read_timeout: READ_TIMEOUT,
-
write_timeout: WRITE_TIMEOUT,
-
request_timeout: REQUEST_TIMEOUT,
-
total_request_timeout: TOTAL_REQUEST_TIMEOUT,
-
}.freeze,
-
:headers_class => Class.new(Headers, &SET_TEMPORARY_NAME),
-
:headers => EMPTY_HASH,
-
:window_size => WINDOW_SIZE,
-
:buffer_size => BUFFER_SIZE,
-
:body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
-
:request_class => Class.new(Request, &SET_TEMPORARY_NAME),
-
:response_class => Class.new(Response, &SET_TEMPORARY_NAME),
-
:request_body_class => Class.new(Request::Body, &SET_TEMPORARY_NAME),
-
:response_body_class => Class.new(Response::Body, &SET_TEMPORARY_NAME),
-
:pool_class => Class.new(Pool, &SET_TEMPORARY_NAME),
-
:connection_class => Class.new(Connection, &SET_TEMPORARY_NAME),
-
:http1_class => Class.new(Connection::HTTP1, &SET_TEMPORARY_NAME),
-
:http2_class => Class.new(Connection::HTTP2, &SET_TEMPORARY_NAME),
-
:resolver_native_class => Class.new(Resolver::Native, &SET_TEMPORARY_NAME),
-
:resolver_system_class => Class.new(Resolver::System, &SET_TEMPORARY_NAME),
-
:resolver_https_class => Class.new(Resolver::HTTPS, &SET_TEMPORARY_NAME),
-
:options_class => Class.new(self, &SET_TEMPORARY_NAME),
-
:transport => nil,
-
:addresses => nil,
-
:persistent => false,
-
1
:resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
-
1
:resolver_cache => (ENV["HTTPX_RESOLVER_CACHE"] || :memory).to_sym,
-
:resolver_options => { cache: true }.freeze,
-
:pool_options => EMPTY_HASH,
-
:ip_families => nil,
-
:close_on_fork => false,
-
}.each_value(&:freeze).freeze
-
# rubocop:enable Lint/UselessConstantScoping
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Parser
-
1
class Error < Error; end
-
-
1
class HTTP1
-
1
VERSIONS = %w[1.0 1.1].freeze
-
-
1
attr_reader :status_code, :http_version, :headers
-
-
1
def initialize(observer, max_headers, max_header_value_size)
-
28
@observer = observer
-
28
@state = :idle
-
28
@buffer = "".b
-
28
@headers = {}
-
28
@max_headers = max_headers
-
28
@max_header_value_size = max_header_value_size
-
28
@content_length = nil
-
28
@_has_trailers = @upgrade = false
-
end
-
-
1
def <<(chunk)
-
33
@buffer << chunk
-
33
parse
-
end
-
-
1
def reset!
-
56
@state = :idle
-
56
@headers = {}
-
56
@content_length = nil
-
56
@_has_trailers = @upgrade = false
-
56
@buffer = @buffer.to_s
-
end
-
-
1
def upgrade?
-
28
@upgrade
-
end
-
-
1
def upgrade_data
-
@buffer.to_s
-
end
-
-
1
private
-
-
1
def parse
-
33
loop do
-
75
state = @state
-
75
case @state
-
when :idle
-
28
parse_headline
-
when :headers, :trailers
-
28
parse_headers
-
when :data
-
19
parse_data
-
end
-
47
return if @buffer.empty? || state == @state
-
end
-
end
-
-
1
def parse_headline
-
#: @type ivar @buffer: String
-
28
idx = @buffer.index("\n")
-
28
return unless idx
-
-
28
(m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
-
raise(Error, "wrong head line format")
-
28
version, code, _ = m.captures
-
28
raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
-
-
28
@http_version = version.split(".").map(&:to_i)
-
28
@status_code = code.to_i
-
28
raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
-
-
28
@buffer = @buffer.byteslice((idx + 1)..-1)
-
28
nextstate(:headers)
-
end
-
-
1
def parse_headers
-
28
headers = @headers
-
28
buffer = @buffer
-
-
#: @type var buffer: String
-
-
280
while (idx = buffer.index("\n"))
-
# @type var line: String
-
252
line = buffer.byteslice(0..idx)
-
252
raise Error, "wrong header format" if line.start_with?("\s", "\t")
-
-
252
line.lstrip!
-
252
buffer = @buffer = buffer.byteslice((idx + 1)..-1)
-
252
if line.empty?
-
28
case @state
-
when :headers
-
28
prepare_data(headers)
-
28
@observer.on_headers(headers)
-
15
return unless @state == :headers
-
-
# state might have been reset
-
# in the :headers callback
-
15
nextstate(:data)
-
15
headers.clear
-
when :trailers
-
@observer.on_trailers(headers)
-
headers.clear
-
nextstate(:complete)
-
end
-
15
return
-
end
-
224
separator_index = line.index(":")
-
224
raise Error, "wrong header format" unless separator_index
-
-
# @type var key: String
-
224
key = line.byteslice(0..(separator_index - 1))
-
-
224
key.rstrip! # was lstripped previously!
-
# @type var value: String
-
224
value = line.byteslice((separator_index + 1)..-1)
-
224
value.strip!
-
224
raise Error, "wrong header format" if value.nil?
-
-
224
values = (headers[key.downcase] ||= []) << value
-
-
224
raise Error, "maximum header value size exceeded" if @max_header_value_size && (values.sum(&:size) > @max_header_value_size)
-
-
224
raise Error, "maximum number of response headers exceeded" if headers.size > @max_headers
-
end
-
end
-
-
1
def parse_data
-
19
if @buffer.respond_to?(:each)
-
# @type ivar @buffer: Transcoder::Chunker::Decoder
-
@buffer.each do |chunk|
-
@observer.on_data(chunk)
-
end
-
19
elsif @content_length
-
# @type ivar @buffer: String
-
19
data = @buffer.byteslice(0, @content_length)
-
# @type var data: String
-
19
@buffer = @buffer.byteslice(@content_length..-1) || "".b
-
19
@content_length -= data.bytesize
-
19
@observer.on_data(data)
-
19
data.clear
-
else
-
# @type ivar @buffer: String
-
@observer.on_data(@buffer)
-
@buffer.clear
-
end
-
19
return unless no_more_data?
-
-
15
@buffer = @buffer.to_s
-
15
if @_has_trailers
-
nextstate(:trailers)
-
else
-
15
nextstate(:complete)
-
end
-
end
-
-
1
def prepare_data(headers)
-
28
@upgrade = headers.key?("upgrade")
-
-
28
@_has_trailers = headers.key?("trailer")
-
-
28
if (tr_encodings = headers["transfer-encoding"])
-
tr_encodings.reverse_each do |tr_encoding|
-
tr_encoding.split(/ *, */).each do |encoding|
-
case encoding
-
when "chunked"
-
@buffer = Transcoder::Chunker::Decoder.new(@buffer.to_s, @_has_trailers)
-
end
-
end
-
end
-
else
-
28
@content_length = headers["content-length"][0].to_i if headers.key?("content-length")
-
end
-
end
-
-
1
def no_more_data?
-
19
if @content_length
-
19
@content_length <= 0
-
elsif @buffer.respond_to?(:finished?)
-
# @type ivar @buffer: Transcoder::Chunker::Decoder
-
@buffer.finished?
-
else
-
false
-
end
-
end
-
-
1
def nextstate(state)
-
58
@state = state
-
58
case state
-
when :headers
-
28
@observer.on_start
-
when :complete
-
15
@observer.on_complete
-
reset!
-
nextstate(:idle) unless @buffer.empty?
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Plugins
-
#
-
# This plugin adds suppoort for callbacks around the request/response lifecycle.
-
#
-
# https://gitlab.com/os85/httpx/-/wikis/Events
-
#
-
1
module Callbacks
-
1
CALLBACKS = %i[
-
connection_opened connection_closed
-
request_error
-
request_started request_body_chunk request_completed
-
response_started response_body_chunk response_completed
-
].freeze
-
-
# connection closed user-space errors happen after errors can be surfaced to requests,
-
# so they need to pierce through the scheduler, which is only possible by simulating an
-
# interrupt.
-
1
class CallbackError < Exception; end # rubocop:disable Lint/InheritException
-
-
1
module InstanceMethods
-
1
include HTTPX::Callbacks
-
-
1
CALLBACKS.each do |meth|
-
9
class_eval(<<-MOD, __FILE__, __LINE__ + 1)
-
def on_#{meth}(&blk) # def on_connection_opened(&blk)
-
on(:#{meth}, &blk) # on(:connection_opened, &blk)
-
self # self
-
end # end
-
MOD
-
end
-
-
1
def plugin(*args, &blk)
-
super(*args).tap do |sess|
-
CALLBACKS.each do |cb|
-
next unless callbacks_for?(cb)
-
-
sess.callbacks(cb).concat(callbacks(cb))
-
end
-
-
sess.wrap(&blk) if blk
-
end
-
end
-
-
1
private
-
-
1
def branch(options, &blk)
-
super(options).tap do |sess|
-
CALLBACKS.each do |cb|
-
next unless callbacks_for?(cb)
-
-
sess.callbacks(cb).concat(callbacks(cb))
-
end
-
sess.wrap(&blk) if blk
-
end
-
end
-
-
1
def do_init_connection(connection, selector)
-
super
-
connection.on(:open) do
-
next unless connection.current_session == self
-
-
emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
-
end
-
connection.on(:callback_connection_closed) do
-
next unless connection.current_session == self
-
-
emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
-
end
-
-
connection
-
end
-
-
1
def set_request_callbacks(request)
-
super
-
-
request.on(:headers) do
-
emit_or_callback_error(:request_started, request)
-
end
-
request.on(:body_chunk) do |chunk|
-
emit_or_callback_error(:request_body_chunk, request, chunk)
-
end
-
request.on(:done) do
-
emit_or_callback_error(:request_completed, request)
-
end
-
-
request.on(:response_started) do |res|
-
if res.is_a?(Response)
-
emit_or_callback_error(:response_started, request, res)
-
res.on(:chunk_received) do |chunk|
-
emit_or_callback_error(:response_body_chunk, request, res, chunk)
-
end
-
else
-
emit_or_callback_error(:request_error, request, res.error)
-
end
-
end
-
request.on(:response) do |res|
-
emit_or_callback_error(:response_completed, request, res) if res.is_a?(Response)
-
end
-
end
-
-
1
def emit_or_callback_error(*args)
-
emit(*args)
-
rescue StandardError => e
-
ex = CallbackError.new(e.message)
-
ex.set_backtrace(e.backtrace)
-
raise ex
-
end
-
-
1
def receive_requests(*)
-
super
-
rescue CallbackError => e
-
raise e.cause
-
end
-
-
1
def close(*)
-
super
-
rescue CallbackError => e
-
raise e.cause
-
end
-
end
-
-
1
module RequestMethods
-
1
def drain_body
-
super.tap do |chunk|
-
emit(:body_chunk, chunk) if chunk
-
rescue StandardError => e
-
# in case an error occurs in callback code
-
@drain_error = e
-
nil
-
end
-
end
-
end
-
-
1
module ConnectionMethods
-
1
private
-
-
1
def disconnect
-
return if @exhausted
-
-
return unless @current_session && @current_selector
-
-
emit(:callback_connection_closed)
-
-
super
-
end
-
end
-
end
-
1
register_plugin :callbacks, Callbacks
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Plugins
-
# This plugin makes a session reuse the same selector across all fibers in a given thread.
-
#
-
# This enables integration with fiber scheduler implementations such as [async](https://github.com/async).
-
#
-
# # https://gitlab.com/os85/httpx/wikis/Fiber-Concurrency
-
#
-
1
module FiberConcurrency
-
1
def self.subplugins
-
{
-
30
h2c: FiberConcurrencyH2C,
-
stream: FiberConcurrencyStream,
-
}
-
end
-
-
1
module InstanceMethods
-
1
private
-
-
1
def send_request(request, *)
-
15
request.set_context!
-
-
15
super
-
end
-
-
1
def get_current_selector
-
15
super(&nil) || begin
-
12
return unless block_given?
-
-
12
default = yield
-
-
12
set_current_selector(default)
-
-
12
default
-
end
-
end
-
end
-
-
1
module RequestMethods
-
# the execution context (fiber) this request was sent on.
-
1
attr_reader :context
-
-
1
def initialize(*)
-
15
super
-
15
@context = nil
-
end
-
-
# sets the execution context for this request. the default is the current fiber.
-
1
def set_context!
-
15
@context ||= Fiber.current # rubocop:disable Naming/MemoizedInstanceVariableName
-
end
-
-
# checks whether the current execution context is the one where the request was created.
-
1
def current_context?
-
106
@context == Fiber.current
-
end
-
-
1
def complete!(response = @response)
-
9
@context = nil
-
9
super
-
end
-
end
-
-
1
module ConnectionMethods
-
1
def current_context?
-
41
@pending.any?(&:current_context?) || (
-
6
@sibling && @sibling.pending.any?(&:current_context?)
-
)
-
end
-
-
1
def initial_call
-
13
return unless current_context?
-
-
7
super
-
end
-
-
1
def interests
-
64
return if connecting? && @pending.none?(&:current_context?)
-
-
64
super
-
end
-
-
1
def on_io_error(e)
-
return super unless e.is_a?(IOError) && e.message.include?("stream closed in another thread")
-
-
# @fiber-switch-guard
-
# sockets closed during fiber scheduler switches are raised in separate fibers than the fiber the
-
# socket may be used in. this check verifies that this is actually about this socket.
-
return unless to_io.closed?
-
-
if @state == :closing
-
# @fiber-switch-guard
-
# if the connection is reused across fibers, the socket may have been closed in the other fiber
-
# and switched here during the process, so continue what it was doing and transition to closed
-
# via #call.
-
call
-
elsif !backlog?
-
super
-
end
-
end
-
-
1
def on_connect_error(e)
-
return super unless e.is_a?(IOError) && e.message.include?("stream closed in another thread")
-
-
# @fiber-switch-guard
-
# sockets closed during fiber scheduler switches are raised in separate fibers than the fiber the
-
# socket may be used in. this check verifies that this is actually about this socket.
-
return unless to_io.closed? && !backlog?
-
-
super
-
end
-
-
1
private
-
-
# checks whether the connection has any pending request (which the connection itself may
-
# have stored, or it may be somewhere in the parser).
-
1
def backlog?
-
@pending.any? ||
-
(@parser && (@parser.pending.any? || @parser.requests.any?))
-
end
-
end
-
-
1
module HTTP1Methods
-
1
def interests
-
63
request = @request || @requests.first
-
-
63
return unless request
-
-
63
return unless request.current_context? || @requests.any?(&:current_context?) || @pending.any?(&:current_context?)
-
-
63
super
-
end
-
end
-
-
1
module HTTP2Methods
-
1
def initialize(*)
-
super
-
@contexts = Hash.new { |hs, k| hs[k] = Set.new }
-
end
-
-
1
def interests
-
if @connection.state == :connected && @handshake_completed && !@contexts.key?(Fiber.current)
-
return :w unless @pings.empty?
-
-
return
-
end
-
-
super
-
end
-
-
1
def send(request, *)
-
add_to_context(request)
-
-
super
-
end
-
-
1
private
-
-
1
def on_close(_, error, _)
-
if error == :http_1_1_required
-
# remove all pending requests context
-
@pending.each do |req|
-
clear_from_context(req)
-
end
-
end
-
-
super
-
end
-
-
1
def on_stream_close(_, request, error)
-
clear_from_context(request) if error != :stream_closed && @streams.key?(request)
-
-
super
-
end
-
-
1
def teardown(request = nil)
-
super
-
-
if request
-
clear_from_context(request)
-
else
-
@contexts.clear
-
end
-
end
-
-
1
def add_to_context(request)
-
@contexts[request.context] << request
-
end
-
-
1
def clear_from_context(request)
-
requests = @contexts[request.context]
-
-
requests.delete(request)
-
-
@contexts.delete(request.context) if requests.empty?
-
end
-
end
-
-
1
module ResolverNativeMethods
-
1
def initial_call
-
2
return unless @queries.values.any?(&:current_context?) || @connections.any?(&:current_context?)
-
-
2
super
-
end
-
-
1
def calculate_interests
-
28
return if @queries.empty?
-
-
26
return unless @queries.values.any?(&:current_context?) || @connections.any?(&:current_context?)
-
-
26
super
-
end
-
-
1
def disconnect
-
2
return unless @connections.all?(&:current_context?)
-
-
2
super
-
end
-
-
1
def on_io_error(e)
-
# TODO: return super if this is not stream clsed in another thread
-
-
log { "IO Erroring: #{e.message}, current:#{@name}, queries:#{@queries.size}" }
-
return unless @name
-
-
super
-
end
-
end
-
-
1
module ResolverSystemMethods
-
1
def initial_call
-
return unless current_context?
-
-
super
-
end
-
-
1
def interests
-
return unless current_context?
-
-
super
-
end
-
-
1
private
-
-
1
def current_context?
-
@queries.any? { |_, conn| conn.current_context? }
-
end
-
end
-
-
1
module FiberConcurrencyH2C
-
1
module HTTP2Methods
-
1
def upgrade(request, *)
-
@contexts[request.context] << request
-
-
super
-
end
-
end
-
end
-
-
1
module FiberConcurrencyStream
-
1
module StreamResponseMethods
-
1
def close
-
unless @request.current_context?
-
@request.close
-
-
return
-
end
-
-
super
-
end
-
end
-
end
-
end
-
-
1
register_plugin :fiber_concurrency, FiberConcurrency
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
class InsecureRedirectError < Error
-
end
-
-
1
module Plugins
-
#
-
# This plugin adds support for automatically following redirect (status 30X) responses.
-
#
-
# It has a default upper bound of followed redirects (see *MAX_REDIRECTS* and the *max_redirects* option),
-
# after which it will return the last redirect response. It will **not** raise an exception.
-
#
-
# It doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
-
#
-
# It doesn't propagate authorization related headers to requests redirecting to different origins
-
# (see *allow_auth_to_other_origins*) to override.
-
#
-
# It allows customization of when to redirect via the *redirect_on* callback option).
-
#
-
# https://gitlab.com/os85/httpx/wikis/Follow-Redirects
-
#
-
1
module FollowRedirects
-
1
MAX_REDIRECTS = 3
-
1
REDIRECT_STATUS = (300..399).freeze
-
1
REQUEST_BODY_HEADERS = %w[transfer-encoding content-encoding content-type content-length content-language content-md5 trailer].freeze
-
-
1
using URIExtensions
-
-
# adds support for the following options:
-
#
-
# :max_redirects :: max number of times a request will be redirected (defaults to <tt>3</tt>).
-
# :follow_insecure_redirects :: whether redirects to an "http://" URI, when coming from an "https//", are allowed
-
# (defaults to <tt>false</tt>).
-
# :allow_auth_to_other_origins :: whether auth-related headers, such as "Authorization", are propagated on redirection
-
# (defaults to <tt>false</tt>).
-
# :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
-
1
module OptionsMethods
-
1
private
-
-
1
def option_max_redirects(value)
-
1
num = Integer(value)
-
1
raise TypeError, ":max_redirects must be positive" if num.negative?
-
-
1
num
-
end
-
-
1
def option_follow_insecure_redirects(value)
-
value
-
end
-
-
1
def option_allow_auth_to_other_origins(value)
-
value
-
end
-
-
1
def option_redirect_on(value)
-
raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
-
-
value
-
end
-
end
-
-
1
module InstanceMethods
-
# returns a session with the *max_redirects* option set to +n+
-
1
def max_redirects(n)
-
with(max_redirects: n.to_i)
-
end
-
-
1
private
-
-
1
def fetch_response(request, selector, options)
-
2
redirect_request = request.redirect_request
-
2
response = super(redirect_request, selector, options)
-
2
return unless response
-
-
2
max_redirects = redirect_request.max_redirects
-
-
2
return response unless response.is_a?(Response)
-
2
return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
-
1
return response unless max_redirects.positive?
-
-
1
redirect_uri = __get_location_from_response(response)
-
-
1
if options.redirect_on
-
redirect_allowed = options.redirect_on.call(redirect_uri)
-
return response unless redirect_allowed
-
end
-
-
# build redirect request
-
1
request_body = redirect_request.body
-
1
redirect_method = "GET"
-
1
redirect_params = {}
-
-
1
if response.status == 305 && options.respond_to?(:proxy)
-
request_body.rewind
-
# The requested resource MUST be accessed through the proxy given by
-
# the Location field. The Location field gives the URI of the proxy.
-
redirect_options = options.merge(headers: redirect_request.headers,
-
proxy: { uri: redirect_uri },
-
max_redirects: max_redirects - 1)
-
-
redirect_params[:body] = request_body
-
redirect_uri = redirect_request.uri
-
options = redirect_options
-
else
-
1
redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
-
1
redirect_opts = Hash[options]
-
1
redirect_params[:max_redirects] = max_redirects - 1
-
-
1
unless request_body.empty?
-
if response.status == 307
-
# The method and the body of the original request are reused to perform the redirected request.
-
redirect_method = redirect_request.verb
-
request_body.rewind
-
redirect_params[:body] = request_body
-
else
-
# redirects are **ALWAYS** GET, so remove body-related headers
-
REQUEST_BODY_HEADERS.each do |h|
-
redirect_headers.delete(h)
-
end
-
redirect_params[:body] = nil
-
end
-
end
-
-
1
options = options.class.new(redirect_opts.merge(headers: redirect_headers.to_h))
-
end
-
-
1
redirect_uri = Utils.to_uri(redirect_uri)
-
-
1
if !options.follow_insecure_redirects &&
-
response.uri.scheme == "https" &&
-
redirect_uri.scheme == "http"
-
error = InsecureRedirectError.new(redirect_uri.to_s)
-
error.set_backtrace(caller)
-
return ErrorResponse.new(request, error)
-
end
-
-
1
retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
-
-
1
request.redirect_request = retry_request
-
-
1
redirect_after = response.headers["retry-after"]
-
-
1
if redirect_after
-
# Servers send the "Retry-After" header field to indicate how long the
-
# user agent ought to wait before making a follow-up request.
-
# When sent with any 3xx (Redirection) response, Retry-After indicates
-
# the minimum time that the user agent is asked to wait before issuing
-
# the redirected request.
-
#
-
redirect_after = Utils.parse_retry_after(redirect_after)
-
-
retry_start = Utils.now
-
log { "redirecting after #{redirect_after} secs..." }
-
selector.after(redirect_after) do
-
if (response = request.response)
-
response.finish!
-
retry_request.response = response
-
# request has terminated abruptly meanwhile
-
retry_request.emit_response(response)
-
else
-
log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-
send_request(retry_request, selector, options)
-
end
-
end
-
else
-
1
send_request(retry_request, selector, options)
-
-
# recalling itself, in case an error was triggered by the above, and we can
-
# verify retriability again.
-
1
return fetch_response(request, selector, options)
-
end
-
nil
-
end
-
-
# :nodoc:
-
1
def redirect_request_headers(original_uri, redirect_uri, headers, options)
-
1
headers = headers.dup
-
-
1
return headers if options.allow_auth_to_other_origins
-
-
1
return headers unless headers.key?("authorization")
-
-
return headers if original_uri.origin == redirect_uri.origin
-
-
headers.delete("authorization")
-
-
headers
-
end
-
-
# :nodoc:
-
1
def __get_location_from_response(response)
-
# @type var location_uri: http_uri
-
1
location_uri = URI(response.headers["location"])
-
1
location_uri = response.uri.merge(location_uri) if location_uri.relative?
-
1
location_uri
-
end
-
end
-
-
1
module RequestMethods
-
# returns the top-most original HTTPX::Request from the redirect chain
-
1
attr_accessor :root_request
-
-
1
def initialize(*)
-
2
super
-
2
@redirect_request = nil
-
end
-
-
1
def on_response_arrived=(cb)
-
1
@redirect_request.on_response_arrived = cb if @redirect_request
-
-
1
super
-
end
-
-
# returns the follow-up redirect request, or itself
-
1
def redirect_request
-
2
@redirect_request || self
-
end
-
-
# sets the follow-up redirect request
-
1
def redirect_request=(req)
-
1
@redirect_request = req
-
1
req.root_request = @root_request || self
-
1
req.on_response_arrived = @on_response_arrived
-
1
@response = nil
-
end
-
-
1
def response
-
8
return super unless @redirect_request && @response.nil?
-
-
2
@redirect_request.response
-
end
-
-
1
def max_redirects
-
2
@options.max_redirects || MAX_REDIRECTS
-
end
-
end
-
-
1
module ConnectionMethods
-
1
private
-
-
1
def set_request_request_timeout(request)
-
return unless request.root_request.nil?
-
-
super
-
end
-
end
-
end
-
1
register_plugin :follow_redirects, FollowRedirects
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Plugins
-
#
-
# This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
-
# (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
-
#
-
# https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
-
#
-
1
module H2C
-
1
VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
-
-
1
class << self
-
1
def load_dependencies(klass)
-
klass.plugin(:upgrade)
-
end
-
-
1
def call(connection, request, response)
-
connection.upgrade_to_h2c(request, response)
-
end
-
-
1
def extra_options(options)
-
options.merge(
-
h2c_class: Class.new(options.http2_class) { include(H2CParser) },
-
max_concurrent_requests: 1,
-
upgrade_handlers: options.upgrade_handlers.merge("h2c" => self),
-
)
-
end
-
end
-
-
1
module OptionsMethods
-
1
def option_h2c_class(value)
-
value
-
end
-
end
-
-
1
module RequestMethods
-
1
def valid_h2c_verb?
-
VALID_H2C_VERBS.include?(@verb)
-
end
-
end
-
-
1
module ConnectionMethods
-
1
using URIExtensions
-
-
1
def initialize(*)
-
super
-
@h2c_handshake = false
-
end
-
-
1
def send(request)
-
return super if @h2c_handshake
-
-
return super unless request.valid_h2c_verb? && request.scheme == "http"
-
-
return super if @upgrade_protocol == "h2c"
-
-
@h2c_handshake = true
-
-
# build upgrade request
-
request.headers.add("connection", "upgrade")
-
request.headers.add("connection", "http2-settings")
-
request.headers["upgrade"] = "h2c"
-
request.headers["http2-settings"] = ::HTTP2::Client.settings_header(request.options.http2_settings)
-
-
super
-
end
-
-
1
def upgrade_to_h2c(request, response)
-
enqueue_pending_requests_from_parser(@parser)
-
-
@parser = request.options.h2c_class.new(@write_buffer, @options)
-
set_parser_callbacks(@parser)
-
@inflight += 1 # request is being completed below
-
@parser.upgrade(request, response)
-
@upgrade_protocol = "h2c"
-
end
-
-
1
private
-
-
1
def send_request_to_parser(request)
-
super
-
-
return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
-
-
max_concurrent_requests = parser.max_concurrent_requests
-
-
return if max_concurrent_requests == 1
-
-
parser.max_concurrent_requests = 1
-
request.once(:response) do
-
parser.max_concurrent_requests = max_concurrent_requests
-
end
-
end
-
end
-
-
1
module H2CParser
-
1
def upgrade(request, response)
-
# skip checks, it is assumed that this is the first
-
# request in the connection
-
stream = @connection.upgrade
-
-
# on_settings
-
handle_stream(stream, request)
-
@streams[request] = stream
-
-
# clean up data left behind in the buffer, if the server started
-
# sending frames
-
data = response.read
-
@connection << data
-
end
-
end
-
end
-
1
register_plugin(:h2c, H2C)
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Plugins
-
# This plugin implements a session that persists connections over the duration of the process.
-
#
-
# This will improve connection reuse in a long-running process.
-
#
-
# One important caveat to note is, although this session might not close connections,
-
# other sessions from the same process that don't have this plugin turned on might.
-
#
-
# This session will still be able to work with it, as if, when expecting a connection
-
# terminated by a different session, it will just retry on a new one and keep it open.
-
#
-
# This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
-
# So when you use this, make sure that you don't fall into this trap.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Persistent
-
#
-
1
module Persistent
-
1
class << self
-
1
def load_dependencies(klass)
-
9
klass.plugin(:fiber_concurrency)
-
-
9
max_retries = if klass.default_options.respond_to?(:max_retries)
-
[klass.default_options.max_retries, 1].max
-
else
-
9
1
-
end
-
9
klass.plugin(:retries, max_retries: max_retries)
-
end
-
end
-
-
1
def self.extra_options(options)
-
9
options.merge(persistent: true)
-
end
-
-
1
module InstanceMethods
-
1
def close(*)
-
9
super
-
-
# traverse other threads and unlink respective selector
-
# WARNING: this is not thread safe, make sure that the session isn't being
-
# used anymore, or all non-main threads are stopped.
-
9
Thread.list.each do |th|
-
72
store = thread_selector_store(th)
-
-
72
next unless store && store.key?(self)
-
-
9
selector = store.delete(self)
-
-
9
selector_close(selector)
-
end
-
end
-
-
1
private
-
-
1
def reconnectable_error?(error)
-
Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
-
end
-
-
1
def when_to_retry(request, response, *)
-
return super unless response.is_a?(ErrorResponse)
-
-
error = response.error
-
# allow request to be retried immediately if the request failed right after the keep alive timeout.
-
# the chances are, the request failed because the connect has been dropped by the peer server, so it's
-
# fine to reopen.
-
return if request.ping? && reconnectable_error?(error)
-
-
super
-
end
-
-
1
def retryable_request?(request, response, *)
-
9
super || begin
-
1
return false unless response.is_a?(ErrorResponse)
-
-
error = response.error
-
-
reconnectable_error?(error)
-
end
-
end
-
-
1
def retryable_error?(ex, options)
-
1
super &&
-
# under the persistent plugin rules, requests are only retried for connection related errors,
-
# which do not include request timeout related errors. This only gets overriden if the end user
-
# manually changed +:max_retries+ to something else, which means it is aware of the
-
# consequences.
-
(!ex.is_a?(RequestTimeoutError) || options.max_retries != 1)
-
end
-
end
-
end
-
1
register_plugin :persistent, Persistent
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Plugins
-
#
-
# This plugin adds support for retrying requests when errors happen.
-
#
-
# It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
-
# after which it will return the last response, error or not. It will **not** raise an exception.
-
#
-
# It does not retry which are not considered idempotent (see *retry_change_requests* to override).
-
#
-
# https://gitlab.com/os85/httpx/wikis/Retries
-
#
-
1
module Retries
-
1
MAX_RETRIES = 3
-
# TODO: pass max_retries in a configure/load block
-
-
1
IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
-
-
# subset of retryable errors which are safe to retry when reconnecting
-
RECONNECTABLE_ERRORS = [
-
1
IOError,
-
EOFError,
-
Errno::ECONNRESET,
-
Errno::ECONNABORTED,
-
Errno::EPIPE,
-
Errno::EINVAL,
-
Errno::ETIMEDOUT,
-
ConnectionError,
-
TLSError,
-
Connection::HTTP2::Error,
-
].freeze
-
-
1
RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
-
Parser::Error,
-
TimeoutError,
-
]).freeze
-
-
1
DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }.freeze
-
-
# list of supported backoff algorithms
-
1
BACKOFF_ALGORITHMS = %i[exponential_backoff polynomial_backoff].freeze
-
-
1
class << self
-
1
if ENV.key?("HTTPX_NO_JITTER")
-
1
def extra_options(options)
-
11
options.merge(max_retries: MAX_RETRIES)
-
end
-
else
-
def extra_options(options)
-
options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
-
end
-
end
-
-
# returns the time to wait before resending +request+ as per the polynomial backoff retry strategy,
-
# where base is 1 and exponent is 2.
-
1
def retry_after_polynomial_backoff(request, _)
-
offset = request.options.max_retries - request.retries
-
1 * ((offset - 1)**2)
-
end
-
-
# returns the time to wait before resending +request+ as per the exponential backoff retry strategy,
-
# where base is 2
-
1
def retry_after_exponential_backoff(request, _)
-
offset = request.options.max_retries - request.retries
-
2**(offset - 1)
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
-
# :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
-
# :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
-
# or the name of a supported backoff algorithm (i.e. <tt>:exponential_backoff</tt>).
-
# :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
-
# :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
-
# (i.e. <tt>->(res) { ... }</tt>).
-
1
module OptionsMethods
-
1
private
-
-
1
def option_retry_after(value)
-
if value.respond_to?(:call)
-
value1 = value
-
value1 = value1.method(:call) unless value1.respond_to?(:arity)
-
-
# allow ->(*) arity as well, which is < 0
-
raise TypeError, "`:retry_after` proc has invalid number of parameters" unless value1.arity.negative? || value1.arity.between?(
-
1, 2
-
)
-
-
else
-
case value
-
when Symbol
-
raise TypeError, "`retry_after`: `#{value}` is not a supported backoff algorithm" unless BACKOFF_ALGORITHMS.include?(value)
-
-
value = Retries.method(:"retry_after_#{value}")
-
-
else
-
value = Float(value)
-
raise TypeError, "`:retry_after` must be positive" unless value.positive?
-
end
-
end
-
-
value
-
end
-
-
1
def option_retry_jitter(value)
-
# return early if callable
-
raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
-
-
value
-
end
-
-
1
def option_max_retries(value)
-
21
num = Integer(value)
-
21
raise TypeError, ":max_retries must be positive" unless num >= 0
-
-
21
num
-
end
-
-
1
def option_retry_change_requests(v)
-
v
-
end
-
-
1
def option_retry_on(value)
-
1
raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
-
-
1
value
-
end
-
end
-
-
1
module InstanceMethods
-
# returns a `:retries` plugin enabled session with +n+ maximum retries per request setting.
-
1
def max_retries(n)
-
with(max_retries: n)
-
end
-
-
1
private
-
-
1
def fetch_response(request, selector, options)
-
27
response = super
-
-
27
if response &&
-
request.retries.positive? &&
-
retryable_request?(request, response, options) &&
-
retryable_response?(response, options)
-
4
try_partial_retry(request, response)
-
4
log { "failed to get response, #{request.retries} tries to go..." }
-
4
prepare_to_retry(request, response)
-
-
4
if (retry_after = when_to_retry(request, response, options)) && retry_after.positive?
-
-
retry_start = Utils.now
-
log { "retrying after #{retry_after} secs..." }
-
selector.after(retry_after) do
-
if (response = request.response)
-
response.finish!
-
# request has terminated abruptly meanwhile
-
request.emit_response(response)
-
else
-
log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-
send_request(request, selector, options)
-
end
-
end
-
-
return
-
else
-
4
send_request(request, selector, options)
-
-
# recalling itself, in case an error was triggered by the above, and we can
-
# verify retriability again.
-
4
return fetch_response(request, selector, options)
-
end
-
end
-
23
response
-
end
-
-
# returns whether +request+ can be retried.
-
1
def retryable_request?(request, _, options)
-
14
IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
-
end
-
-
1
def retryable_response?(response, options)
-
13
(response.is_a?(ErrorResponse) && retryable_error?(response.error, options)) || options.retry_on&.call(response)
-
end
-
-
# returns whether the +ex+ exception happend for a retriable request.
-
1
def retryable_error?(ex, _)
-
39
RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) } && !ex.is_a?(TotalRequestTimeoutError)
-
end
-
-
1
def proxy_error?(request, response, _)
-
super && !request.retries.positive?
-
end
-
-
1
def prepare_to_retry(request, _response)
-
4
request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
-
4
request.transition(:idle)
-
end
-
-
1
def when_to_retry(request, response, options)
-
4
retry_after = options.retry_after
-
-
4
return unless retry_after
-
-
retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
-
-
return unless retry_after
-
-
# apply jitter
-
if (jitter = request.options.retry_jitter)
-
retry_after = jitter.call(retry_after)
-
end
-
retry_after
-
end
-
-
#
-
# Attempt to set the request to perform a partial range request.
-
# This happens if the peer server accepts byte-range requests, and
-
# the last response contains some body payload.
-
#
-
1
def try_partial_retry(request, response)
-
4
response = response.response if response.is_a?(ErrorResponse)
-
-
4
return unless response
-
-
2
unless response.headers.key?("accept-ranges") &&
-
response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
-
(original_body = response.body)
-
2
response.body.close
-
2
return
-
end
-
-
request.partial_response = response
-
-
size = original_body.bytesize
-
-
request.headers["range"] = "bytes=#{size}-"
-
end
-
end
-
-
1
module RequestMethods
-
# number of retries left.
-
1
attr_accessor :retries
-
-
# a response partially received before.
-
1
attr_writer :partial_response
-
-
# initializes the request instance, sets the number of retries for the request.
-
1
def initialize(*args)
-
11
super
-
11
@retries = @options.max_retries
-
11
@partial_response = nil
-
end
-
-
1
def response=(response)
-
15
if (partial_response = @partial_response)
-
if response.is_a?(Response) && response.status == 206
-
response.from_partial_response(partial_response)
-
else
-
partial_response.close
-
end
-
@partial_response = nil
-
end
-
-
15
super
-
end
-
end
-
-
1
module ResponseMethods
-
1
def from_partial_response(response)
-
@status = response.status
-
@headers = response.headers
-
@body = response.body
-
end
-
end
-
end
-
1
register_plugin :retries, Retries
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
class StreamResponse
-
1
attr_reader :request
-
-
1
def initialize(request, session)
-
3
@request = request
-
3
@options = @request.options
-
3
@session = session
-
3
@response_enum = nil
-
3
@buffered_chunks = []
-
end
-
-
1
def each(&block)
-
3
return enum_for(__method__) unless block
-
-
3
if (response_enum = @response_enum)
-
@response_enum = nil
-
# streaming already started, let's finish it
-
-
while (chunk = @buffered_chunks.shift)
-
block.call(chunk)
-
end
-
-
# consume enum til the end
-
begin
-
while (chunk = response_enum.next)
-
block.call(chunk)
-
end
-
rescue StopIteration
-
return
-
end
-
end
-
-
3
@request.stream = self
-
-
begin
-
3
@on_chunk = block
-
-
3
response = @session.request(@request)
-
-
3
response.raise_for_status
-
ensure
-
3
@on_chunk = nil
-
end
-
end
-
-
1
def each_line(&block)
-
2
return enum_for(__method__) unless block
-
-
1
line = "".b
-
-
1
each do |chunk|
-
1
line << chunk
-
-
3
while (idx = line.index("\n"))
-
1
if idx.zero?
-
yield ""
-
else
-
1
yield line.byteslice(0..(idx - 1))
-
end
-
-
1
line = line.byteslice((idx + 1)..-1)
-
end
-
end
-
-
1
yield line unless line.empty?
-
end
-
-
# This is a ghost method. It's to be used ONLY internally, when processing streams
-
1
def on_chunk(chunk)
-
4
raise NoMethodError unless @on_chunk
-
-
4
@on_chunk.call(chunk)
-
end
-
-
# simplecov:disable
-
1
def inspect
-
"#<#{self.class}:#{object_id}>"
-
end
-
# simplecov:enable
-
-
1
def to_s
-
if @request.response
-
@request.response.to_s
-
else
-
@buffered_chunks.join
-
end
-
end
-
-
1
private
-
-
1
def response
-
6
@request.response || begin
-
response_enum = each
-
while (chunk = response_enum.next)
-
@buffered_chunks << chunk
-
break if @request.response
-
end
-
@response_enum = response_enum
-
@request.response
-
end
-
end
-
-
1
def respond_to_missing?(meth, include_private)
-
if (response = @request.response)
-
response.respond_to_missing?(meth, include_private)
-
else
-
@options.response_class.method_defined?(meth) || (include_private && @options.response_class.private_method_defined?(meth))
-
end || super
-
end
-
-
1
def method_missing(meth, *args, **kwargs, &block)
-
3
return super unless response.respond_to?(meth)
-
-
3
response.__send__(meth, *args, **kwargs, &block)
-
end
-
end
-
-
1
module Plugins
-
#
-
# This plugin adds support for streaming a response (useful for i.e. "text/event-stream" payloads).
-
#
-
# https://gitlab.com/os85/httpx/wikis/Stream
-
#
-
1
module Stream
-
1
STREAM_REQUEST_OPTIONS = { timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 }.freeze }.freeze
-
-
1
def self.extra_options(options)
-
3
options.merge(
-
stream: false,
-
timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 },
-
stream_response_class: Class.new(StreamResponse, &Options::SET_TEMPORARY_NAME).freeze
-
)
-
end
-
-
# adds support for the following options:
-
#
-
# :stream :: whether the request to process should be handled as a stream (defaults to <tt>false</tt>).
-
# :stream_response_class :: Class used to build the stream response object.
-
1
module OptionsMethods
-
1
def option_stream(val)
-
4
val
-
end
-
-
1
def option_stream_response_class(value)
-
4
value
-
end
-
-
1
def extend_with_plugin_classes(pl)
-
return super unless defined?(pl::StreamResponseMethods)
-
-
@stream_response_class = @stream_response_class.dup
-
Options::SET_TEMPORARY_NAME[@stream_response_class, pl]
-
@stream_response_class.__send__(:include, pl::StreamResponseMethods) if defined?(pl::StreamResponseMethods)
-
-
super
-
end
-
end
-
-
1
module InstanceMethods
-
1
def request(*args, **options)
-
6
if args.first.is_a?(Request)
-
3
requests = args
-
-
3
request = requests.first
-
-
3
unless request.options.stream && !request.stream
-
3
if options[:stream]
-
warn "passing `stream: true` with a request object is not supported anymore. " \
-
"You can instead build the request object with `stream :true`"
-
end
-
3
return super
-
end
-
else
-
3
return super unless options[:stream]
-
-
3
requests = build_requests(*args, options)
-
-
3
request = requests.first
-
end
-
-
3
raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
-
-
3
@options.stream_response_class.new(request, self)
-
end
-
-
1
def build_request(verb, uri, params = EMPTY_HASH, options = @options)
-
4
return super unless params[:stream]
-
-
3
super(verb, uri, params, options.merge(STREAM_REQUEST_OPTIONS.merge(stream: true)))
-
end
-
end
-
-
1
module RequestMethods
-
1
attr_accessor :stream
-
end
-
-
1
module ResponseMethods
-
1
def stream
-
4
request = @request.root_request if @request.respond_to?(:root_request)
-
4
request ||= @request
-
-
4
request.stream
-
end
-
end
-
-
1
module ResponseBodyMethods
-
1
def initialize(*)
-
4
super
-
4
@stream = @response.stream
-
end
-
-
1
def write(chunk)
-
4
return super unless @stream
-
-
4
return 0 if chunk.empty?
-
-
4
chunk = decode_chunk(chunk)
-
-
4
@stream.on_chunk(chunk.dup)
-
-
4
chunk.bytesize
-
end
-
-
1
private
-
-
1
def transition(*)
-
return if @stream
-
-
super
-
end
-
end
-
end
-
1
register_plugin :stream, Stream
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX::Plugins
-
#
-
# This plugin adds a simple interface to integrate request tracing SDKs.
-
#
-
# An example of such an integration is the datadog adapter.
-
#
-
# https://gitlab.com/os85/httpx/wikis/Tracing
-
#
-
1
module Tracing
-
1
class Wrapper
-
1
attr_reader :tracers
-
1
protected :tracers
-
-
1
def initialize(*tracers)
-
@tracers = tracers.flat_map do |tracer|
-
case tracer
-
when Wrapper
-
tracer.tracers
-
else
-
tracer
-
end
-
end.uniq
-
end
-
-
1
def merge(tracer)
-
Wrapper.new(*@tracers, *tracer.tracers)
-
end
-
-
1
def freeze
-
@tracers.each(&:freeze).freeze
-
super
-
end
-
-
1
%i[start finish reset enabled?].each do |callback|
-
4
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-
# proxies ##{callback} calls to wrapper tracers.
-
def #{callback}(*args) # def start(*args)
-
@tracers.each { |t| t.#{callback}(*args) } # @tracers.each { |t| t.start(*args) }
-
end # end
-
OUT
-
end
-
end
-
-
# adds support for the following options:
-
#
-
# :tracer :: object which responds to #start, #finish and #reset.
-
1
module OptionsMethods
-
1
private
-
-
1
def option_tracer(tracer)
-
2
unless tracer.respond_to?(:start) &&
-
tracer.respond_to?(:finish) &&
-
tracer.respond_to?(:reset) &&
-
tracer.respond_to?(:enabled?)
-
raise TypeError, "#{tracer} must to respond to `#start(r)`, `#finish` and `#reset` and `#enabled?"
-
end
-
-
2
tracer = Wrapper.new(@tracer, tracer) if @tracer
-
2
tracer
-
end
-
end
-
-
1
module RequestMethods
-
1
attr_accessor :init_time
-
-
# intercepts request initialization to inject the tracing logic.
-
1
def initialize(*)
-
14
super
-
-
14
@init_time = nil
-
-
14
tracer = @options.tracer
-
-
14
return unless tracer && tracer.enabled?(self)
-
-
14
on(:idle) do
-
4
tracer.reset(self)
-
-
# request is reset when it's retried.
-
4
@init_time = nil
-
end
-
14
on(:headers) do
-
# the usual request init time (when not including the connection handshake)
-
# should be the time the request is buffered the first time.
-
16
@init_time ||= ::Time.now.utc
-
-
16
tracer.start(self)
-
end
-
30
on(:response) { |response| tracer.finish(self, response) }
-
end
-
-
1
def response=(*)
-
# init_time should be set when it's send to a connection.
-
# However, there are situations where connection initialization fails.
-
# Example is the :ssrf_filter plugin, which raises an error on
-
# initialize if the host is an IP which matches against the known set.
-
# in such cases, we'll just set here right here.
-
16
@init_time ||= ::Time.now.utc
-
-
16
super
-
end
-
end
-
-
# Connection mixin
-
1
module ConnectionMethods
-
1
def initialize(*)
-
13
super
-
-
13
@init_time = ::Time.now.utc
-
end
-
-
1
def send_request_to_parser(request)
-
16
if connecting?
-
# request span timeframe should include the time it took to connect.
-
16
request.init_time ||= @init_time
-
end
-
-
16
super
-
end
-
-
1
def idling
-
3
super
-
-
# time of initial request(s) is accounted from the moment
-
# the connection is back to :idle, and ready to connect again.
-
3
@init_time = ::Time.now.utc
-
end
-
-
1
private
-
-
1
def ping(request)
-
# if a connection is probed for liveness, the request timeframe should include
-
# it too.
-
request.init_time ||= ::Time.now.utc
-
-
super
-
end
-
end
-
end
-
1
register_plugin :tracing, Tracing
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module ResponsePatternMatchExtensions
-
1
def deconstruct
-
[@status, @headers, @body]
-
end
-
-
1
def deconstruct_keys(_keys)
-
{ status: @status, headers: @headers, body: @body }
-
end
-
end
-
-
1
module ErrorResponsePatternMatchExtensions
-
1
def deconstruct
-
[@error]
-
end
-
-
1
def deconstruct_keys(_keys)
-
{ error: @error }
-
end
-
end
-
-
1
module HeadersPatternMatchExtensions
-
1
def deconstruct
-
to_a
-
end
-
end
-
-
1
Headers.include HeadersPatternMatchExtensions
-
1
Response.include ResponsePatternMatchExtensions
-
1
ErrorResponse.include ErrorResponsePatternMatchExtensions
-
end
-
# frozen_string_literal: true
-
-
1
require "httpx/selector"
-
1
require "httpx/connection"
-
1
require "httpx/connection/http2"
-
1
require "httpx/connection/http1"
-
1
require "httpx/resolver"
-
-
1
module HTTPX
-
1
class Pool
-
1
using URIExtensions
-
-
1
POOL_TIMEOUT = 5
-
-
# Sets up the connection pool with the given +options+, which can be the following:
-
#
-
# :max_connections:: the maximum number of connections held in the pool.
-
# :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
-
# :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
-
#
-
1
def initialize(options)
-
159
@max_connections = options.fetch(:max_connections, Float::INFINITY)
-
159
@max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
-
159
@pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
-
191
@resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
-
159
@resolver_mtx = Thread::Mutex.new
-
159
@connections = []
-
159
@connection_mtx = Thread::Mutex.new
-
159
@connections_counter = 0
-
159
@max_connections_cond = ConditionVariable.new
-
159
@origin_counters = Hash.new(0)
-
191
@origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
-
end
-
-
# connections returned by this function are not expected to return to the connection pool.
-
1
def pop_connection
-
92
@connection_mtx.synchronize do
-
92
drop_connection
-
end
-
end
-
-
# opens a connection to the IP reachable through +uri+.
-
# Many hostnames are reachable through the same IP, so we try to
-
# maximize pipelining by opening as few connections as possible.
-
#
-
1
def checkout_connection(uri, options)
-
68
return checkout_new_connection(uri, options) if options.io
-
-
68
@connection_mtx.synchronize do
-
68
acquire_connection(uri, options) || begin
-
66
if @connections_counter == @max_connections
-
# this takes precedence over per-origin
-
-
expires_at = Utils.now + @pool_timeout
-
-
loop do
-
@max_connections_cond.wait(@connection_mtx, @pool_timeout)
-
-
if (conn = acquire_connection(uri, options))
-
return conn
-
end
-
-
# if one can afford to create a new connection, do it
-
break unless @connections_counter == @max_connections
-
-
# if no matching usable connection was found, the pool will make room and drop a closed connection.
-
if (conn = @connections.find { |c| c.state == :closed })
-
drop_connection(conn)
-
break
-
end
-
-
# happens when a condition was signalled, but another thread snatched the available connection before
-
# context was passed back here.
-
next if Utils.now < expires_at
-
-
raise PoolTimeoutError.new(@pool_timeout,
-
"Timed out after #{@pool_timeout} seconds while waiting for a connection")
-
end
-
-
end
-
-
66
if @origin_counters[uri.origin] == @max_connections_per_origin
-
-
expires_at = Utils.now + @pool_timeout
-
-
loop do
-
@origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
-
-
if (conn = acquire_connection(uri, options))
-
return conn
-
end
-
-
# happens when a condition was signalled, but another thread snatched the available connection before
-
# context was passed back here.
-
next if Utils.now < expires_at
-
-
raise(PoolTimeoutError.new(@pool_timeout,
-
"Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
-
end
-
end
-
-
66
@connections_counter += 1
-
66
@origin_counters[uri.origin] += 1
-
-
66
checkout_new_connection(uri, options)
-
end
-
end
-
end
-
-
1
def checkin_connection(connection)
-
38
return if connection.options.io
-
-
38
@connection_mtx.synchronize do
-
38
if connection.coalesced? || connection.state == :idle
-
# when connections coalesce
-
4
drop_connection(connection)
-
-
4
return
-
end
-
-
34
@connections << connection
-
-
34
@max_connections_cond.signal
-
34
@origin_conds[connection.origin.to_s].signal
-
end
-
end
-
-
1
def checkout_mergeable_connection(connection)
-
67
return if connection.options.io
-
-
67
@connection_mtx.synchronize do
-
67
idx = @connections.find_index do |ch|
-
ch != connection && ch.mergeable?(connection)
-
end
-
67
@connections.delete_at(idx) if idx
-
end
-
end
-
-
1
def reset_resolvers
-
128
@resolver_mtx.synchronize { @resolvers.clear }
-
end
-
-
1
def checkout_resolver(options)
-
32
resolver_type = options.resolver_class
-
-
32
@resolver_mtx.synchronize do
-
32
resolvers = @resolvers[resolver_type]
-
-
32
idx = resolvers.find_index do |res|
-
res.options.resolver_options_match?(options)
-
end
-
32
resolvers.delete_at(idx) if idx
-
end || checkout_new_resolver(resolver_type, options)
-
end
-
-
1
def checkin_resolver(resolver)
-
32
if resolver.is_a?(Resolver::Multi)
-
28
resolver_class = resolver.resolvers.first.class
-
else
-
4
resolver_class = resolver.class
-
-
4
resolver = resolver.multi
-
end
-
-
# a multi requires all sub-resolvers being closed in order to be
-
# correctly checked back in.
-
32
return unless resolver.closed?
-
-
32
@resolver_mtx.synchronize do
-
32
resolvers = @resolvers[resolver_class]
-
-
32
resolvers << resolver unless resolvers.include?(resolver)
-
end
-
end
-
-
# simplecov:disable
-
1
def inspect
-
"#<#{self.class}:#{object_id} " \
-
"@max_connections=#{@max_connections} " \
-
"@max_connections_per_origin=#{@max_connections_per_origin} " \
-
"@pool_timeout=#{@pool_timeout} " \
-
"@connections=#{@connections.size}>"
-
end
-
# simplecov:enable
-
-
1
private
-
-
1
def acquire_connection(uri, options)
-
68
idx = @connections.find_index do |connection|
-
2
connection.match?(uri, options)
-
end
-
-
68
return unless idx
-
-
2
@connections.delete_at(idx)
-
end
-
-
1
def checkout_new_connection(uri, options)
-
66
connection = options.connection_class.new(uri, options)
-
66
connection.log(level: 2) { "created connection##{connection.object_id} in pool##{object_id}" }
-
66
connection
-
end
-
-
1
def checkout_new_resolver(resolver_type, options)
-
32
resolver = if resolver_type.multi?
-
32
Resolver::Multi.new(resolver_type, options)
-
else
-
resolver_type.new(options)
-
end
-
32
resolver.log(level: 2) { "created resolver##{resolver.object_id} in pool##{object_id}" }
-
32
resolver
-
end
-
-
# drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),
-
# the first available connection from the pool will be dropped.
-
1
def drop_connection(connection = nil)
-
96
if connection
-
4
@connections.delete(connection)
-
else
-
92
connection = @connections.shift
-
-
92
return unless connection
-
end
-
-
32
@connections_counter -= 1
-
32
@origin_conds.delete(connection.origin) if (@origin_counters[connection.origin.to_s] -= 1).zero?
-
-
32
connection
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Punycode
-
1
module_function
-
-
begin
-
1
require "idnx"
-
-
1
def encode_hostname(hostname)
-
Idnx.to_punycode(hostname)
-
end
-
rescue LoadError
-
def encode_hostname(hostname)
-
warn "#{hostname} cannot be converted to punycode. Install the " \
-
"\"idnx\" gem: https://github.com/HoneyryderChuck/idnx"
-
-
hostname
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "delegate"
-
1
require "forwardable"
-
-
1
module HTTPX
-
# Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
-
# as well as maintaining the state machine which manages streaming the request onto the wire.
-
1
class Request
-
1
extend Forwardable
-
1
include Loggable
-
1
include Callbacks
-
-
1
using URIExtensions
-
-
1
ALLOWED_URI_SCHEMES = %w[https http].freeze
-
-
# the upcased string HTTP verb for this request.
-
1
attr_reader :verb
-
-
# the absolute URI object for this request.
-
1
attr_reader :uri
-
-
# an HTTPX::Headers object containing the request HTTP headers.
-
1
attr_reader :headers
-
-
# an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
-
1
attr_reader :body
-
-
# a symbol describing which frame is currently being flushed.
-
1
attr_reader :state
-
-
# an HTTPX::Options object containing request options.
-
1
attr_reader :options
-
-
# the corresponding HTTPX::Response object, when there is one.
-
1
attr_reader :response
-
-
# Exception raised during enumerable body writes.
-
1
attr_reader :drain_error
-
-
# when this request is sent via HTTP/2, it'll use this hash of options to set the priority of the
-
# respective HTTP/2 frame.
-
1
attr_reader :http2_stream_options
-
-
# The IP address from the peer server.
-
1
attr_accessor :peer_address
-
-
# the connection the request is currently being sent to (none if before or after transaction)
-
1
attr_writer :connection
-
-
# callback triggered when a response (which may not be the final response) was assigned to the request.
-
1
attr_writer :on_response_arrived
-
-
1
attr_writer :persistent
-
-
1
attr_reader :active_timeouts
-
-
# will be +true+ when request body has been completely flushed.
-
1
def_delegator :@body, :empty?
-
-
# closes the body
-
1
def_delegator :@body, :close
-
-
# initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
-
# an absolute or relative +uri+ (either as String or URI::HTTP object), the
-
# request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
-
#
-
# Besides any of the options documented in HTTPX::Options (which would override or merge with what
-
# +options+ sets), it accepts also the following:
-
#
-
# :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
-
# :body :: to be encoded in the request body payload. can be a String, an IO object (i.e. a File), or an Enumerable.
-
# :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
-
# :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
-
# :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
-
# :http2_stream_options :: hash of options to be used to set the HTTP/2 priority by sending an initial PRIORITY frame.
-
#
-
# :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
-
1
def initialize(verb, uri, options, params = EMPTY_HASH)
-
71
@verb = verb.to_s.upcase
-
71
@uri = Utils.to_uri(uri)
-
-
71
@headers = options.headers.dup
-
71
merge_headers(params.delete(:headers)) if params.key?(:headers)
-
-
71
@query_params = params.delete(:params) if params.key?(:params)
-
-
71
@http2_stream_options = params.key?(:http2_stream_options) ? params.delete(:http2_stream_options) : EMPTY_HASH
-
-
71
@body = options.request_body_class.new(@headers, options, **params)
-
-
71
@options = @body.options
-
-
71
if @uri.relative? || @uri.host.nil?
-
origin = @options.origin
-
raise(Error, "invalid URI: #{@uri}") unless origin
-
-
base_path = @options.base_path
-
-
@uri = origin.merge("#{base_path}#{@uri}")
-
end
-
-
71
raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
-
-
71
@state = :idle
-
@connection = @response =
-
@drainer = @peer_address =
-
71
@informational_status = @on_response_arrived = nil
-
71
@ping = @started = false
-
71
@persistent = @options.persistent
-
71
@active_timeouts = []
-
end
-
-
# dupped initialization
-
1
def initialize_dup(orig)
-
super
-
@uri = orig.instance_variable_get(:@uri).dup
-
@headers = orig.instance_variable_get(:@headers).dup
-
@body = orig.instance_variable_get(:@body).dup
-
end
-
-
1
def complete!(response = @response)
-
34
emit(:complete, response)
-
34
reset_timers(true)
-
end
-
-
# whether request has been buffered with a ping
-
1
def ping?
-
4
@ping
-
end
-
-
# marks the request as having been buffered with a ping
-
1
def ping!
-
@ping = true
-
end
-
-
# the read timeout defined for this request.
-
1
def read_timeout
-
34
@options.timeout[:read_timeout]
-
end
-
-
# the write timeout defined for this request.
-
1
def write_timeout
-
34
@options.timeout[:write_timeout]
-
end
-
-
# the request timeout defined for this request.
-
1
def request_timeout
-
34
@options.timeout[:request_timeout]
-
end
-
-
# the total request timeout defined for this request.
-
1
def total_request_timeout
-
31
@options.timeout[:total_request_timeout]
-
end
-
-
1
def persistent?
-
29
@persistent
-
end
-
-
# if the request contains trailer headers
-
1
def trailers?
-
2
defined?(@trailers)
-
end
-
-
# returns an instance of HTTPX::Headers containing the trailer headers
-
1
def trailers
-
@trailers ||= @options.headers_class.new
-
end
-
-
# returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
-
1
def interests
-
234
return :r if @state == :done || @state == :expect
-
-
30
:w
-
end
-
-
1
def can_buffer?
-
100
@state != :done
-
end
-
-
1
def started?
-
34
@started
-
end
-
-
# merges +h+ into the instance of HTTPX::Headers of the request.
-
1
def merge_headers(h)
-
10
@headers = @headers.merge(h)
-
10
return unless @headers.key?("range")
-
-
@headers.delete("accept-encoding")
-
end
-
-
# the URI scheme of the request +uri+.
-
1
def scheme
-
5
@uri.scheme
-
end
-
-
# sets the +response+ on this request.
-
1
def response=(response)
-
74
return unless response
-
-
74
case response
-
when Response
-
67
if response.status < 200
-
# deal with informational responses
-
-
if response.status == 100 && @headers.key?("expect")
-
@informational_status = response.status
-
return
-
end
-
-
# 103 Early Hints advertises resources in document to browsers.
-
# not very relevant for an HTTP client, discard.
-
return if response.status >= 103
-
-
end
-
when ErrorResponse
-
7
response.error.connection = nil if response.error.respond_to?(:connection=)
-
end
-
-
74
@response = response
-
-
74
emit(:response_started, response)
-
end
-
-
# returnns the URI path of the request +uri+.
-
1
def path
-
51
path = uri.path.dup
-
51
path = +"" if path.nil?
-
51
path << "/" if path.empty?
-
51
path << "?#{query}" unless query.empty?
-
51
path
-
end
-
-
# returs the URI authority of the request.
-
#
-
# session.build_request("GET", "https://google.com/query").authority #=> "google.com"
-
# session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
-
1
def authority
-
34
@uri.authority
-
end
-
-
# returs the URI origin of the request.
-
#
-
# session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
-
# session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
-
1
def origin
-
5
@uri.origin
-
end
-
-
# returs the URI query string of the request (when available).
-
#
-
# session.build_request("GET", "https://search.com").query #=> ""
-
# session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
-
# session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
-
# session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
-
1
def query
-
103
return @query if defined?(@query)
-
-
69
query = []
-
69
if (q = @query_params) && !q.empty?
-
4
query << Transcoder::Form.encode(q)
-
end
-
69
query << @uri.query if @uri.query
-
69
@query = query.join("&")
-
end
-
-
# consumes and returns the next available chunk of request body that can be sent
-
1
def drain_body
-
12
return if @body.nil?
-
-
12
@drainer ||= @body.each
-
12
@drainer.next.dup
-
rescue StopIteration
-
5
nil
-
rescue StandardError => e
-
# in case an error occurs while emitting body chunks
-
@drain_error = e
-
nil
-
end
-
-
# simplecov:disable
-
1
def inspect
-
1
"#<#{self.class}:#{object_id} " \
-
"#{@verb} " \
-
"#{uri} " \
-
"@headers=#{@headers} " \
-
"@body=#{@body}>"
-
end
-
# simplecov:enable
-
-
# moves on to the +nextstate+ of the request state machine (when all preconditions are met)
-
1
def transition(nextstate)
-
298
case nextstate
-
when :idle
-
6
@body.rewind
-
6
@ping = false
-
6
@response = @drainer = nil
-
-
# request may be sent to a different connection and will be
-
# reassigned a new set of timers.
-
6
reset_timers(false)
-
when :headers
-
74
return unless @state == :idle
-
-
72
@started = true
-
when :body
-
74
return unless @state == :headers ||
-
@state == :expect
-
-
72
if @headers.key?("expect")
-
if @informational_status && @informational_status == 100
-
# check for 100 Continue response, and deallocate the var
-
# if @informational_status == 100
-
# @response = nil
-
# end
-
else
-
return if @state == :expect # do not re-set it
-
-
nextstate = :expect
-
end
-
end
-
when :trailers
-
72
return unless @state == :body
-
when :done
-
72
return if @state == :expect
-
-
end
-
294
log(level: 3) { "#{@state} -> #{nextstate}" }
-
294
@state = nextstate
-
294
emit(@state, self)
-
nil
-
end
-
-
# whether the request supports the 100-continue handshake and already processed the 100 response.
-
1
def expects?
-
33
@headers["expect"] == "100-continue" && @informational_status == 100 && !@response
-
end
-
-
1
def set_timeout_callback(event, &callback)
-
170
clb = once(event, &callback)
-
-
# reset timeout callbacks when requests get rerouted to a different connection
-
170
once(:idle) do
-
15
callbacks(event).delete(clb)
-
end
-
end
-
-
1
def handle_error(error)
-
if (connection = @connection)
-
connection.on_error(error, self)
-
else
-
response = ErrorResponse.new(self, error)
-
self.response = response
-
emit_response(response)
-
end
-
end
-
-
1
def emit_response(response)
-
74
emit(:response, response)
-
-
74
return unless @on_response_arrived
-
-
36
@on_response_arrived.call
-
end
-
-
1
private
-
-
1
def reset_timers(reset_total_request_timers)
-
40
timers = @active_timeouts
-
-
81
until (timer = timers.shift).nil?
-
1
next if !reset_total_request_timers && timer.label == :total_request_timeout
-
-
# cancel active timers.
-
1
timer.cancel
-
end
-
end
-
end
-
end
-
-
1
require_relative "request/body"
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
# Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
-
1
class Request::Body < SimpleDelegator
-
1
class << self
-
1
def new(_, options, body: nil, **params)
-
71
if body.is_a?(self)
-
# request derives its options from body
-
body.options = options.merge(params)
-
return body
-
end
-
-
71
super
-
end
-
end
-
-
1
attr_accessor :options
-
-
# inits the instance with the request +headers+, +options+ and +params+, which contain the payload definition.
-
# it wraps the given body with the appropriate encoder on initialization.
-
#
-
# ..., json: { foo: "bar" }) #=> json encoder
-
# ..., form: { foo: "bar" }) #=> form urlencoded encoder
-
# ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
-
# ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
-
# ..., form: { body: "bla") }) #=> raw data encoder
-
1
def initialize(h, options, **params)
-
71
@headers = h
-
71
@body = self.class.initialize_body(params)
-
71
@options = options.merge(params)
-
-
71
if @body
-
9
if @options.compress_request_body && @headers.key?("content-encoding")
-
-
@headers.get("content-encoding").each do |encoding|
-
@body = self.class.initialize_deflater_body(@body, encoding)
-
end
-
end
-
-
9
@headers["content-type"] ||= @body.content_type
-
9
@headers["content-length"] = @body.bytesize unless unbounded_body?
-
end
-
-
71
super(@body)
-
end
-
-
# consumes and yields the request payload in chunks.
-
1
def each(&block)
-
10
return enum_for(__method__) unless block
-
5
return if @body.nil?
-
-
5
body = stream(@body)
-
5
if body.respond_to?(:read)
-
5
while (chunk = body.read(16_384))
-
3
block.call(chunk)
-
end
-
# TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
-
# IO.copy_stream(body, ProcIO.new(block))
-
4
elsif body.respond_to?(:each)
-
body.each(&block)
-
else
-
4
block[body.to_s]
-
end
-
end
-
-
1
def close
-
@body.close if @body.respond_to?(:close)
-
end
-
-
# if the +@body+ is rewindable, it rewinnds it.
-
1
def rewind
-
6
return if empty?
-
-
@body.rewind if @body.respond_to?(:rewind)
-
end
-
-
# return +true+ if the +body+ has been fully drained (or does nnot exist).
-
1
def empty?
-
57
return true if @body.nil?
-
11
return false if chunked?
-
-
11
@body.bytesize.zero?
-
end
-
-
# returns the +@body+ payload size in bytes.
-
1
def bytesize
-
25
return 0 if @body.nil?
-
-
@body.bytesize
-
end
-
-
# sets the body to yield using chunked trannsfer encoding format.
-
1
def stream(body)
-
5
return body unless chunked?
-
-
Transcoder::Chunker.encode(body.enum_for(:each))
-
end
-
-
# returns whether the body yields infinitely.
-
1
def unbounded_body?
-
11
return @unbounded_body if defined?(@unbounded_body)
-
-
9
@unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
-
end
-
-
# returns whether the chunked transfer encoding header is set.
-
1
def chunked?
-
54
@headers["transfer-encoding"] == "chunked"
-
end
-
-
# sets the chunked transfer encoding header.
-
1
def chunk!
-
@headers.add("transfer-encoding", "chunked")
-
end
-
-
# simplecov:disable
-
1
def inspect
-
"#<#{self.class}:#{object_id} " \
-
"#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
-
end
-
# simplecov:enable
-
-
1
class << self
-
1
def initialize_body(params)
-
71
if (body = params.delete(:body))
-
# @type var body: bodyIO
-
3
Transcoder::Body.encode(body)
-
68
elsif (form = params.delete(:form))
-
6
if Transcoder::Multipart.multipart?(form)
-
# @type var form: Transcoder::Multipart::multipart_input
-
3
Transcoder::Multipart.encode(form)
-
else
-
# @type var form: Transcoder::urlencoded_input
-
3
Transcoder::Form.encode(form)
-
end
-
62
elsif (json = params.delete(:json))
-
# @type var body: _ToJson
-
Transcoder::JSON.encode(json)
-
end
-
end
-
-
# returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
-
1
def initialize_deflater_body(body, encoding)
-
case encoding
-
when "gzip"
-
Transcoder::GZIP.encode(body)
-
when "deflate"
-
Transcoder::Deflate.encode(body)
-
when "identity"
-
body
-
else
-
body
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "socket"
-
1
require "resolv"
-
-
1
module HTTPX
-
1
module Resolver
-
1
extend self
-
-
1
RESOLVE_TIMEOUT = [2, 3].freeze
-
1
require "httpx/resolver/entry"
-
1
require "httpx/resolver/cache"
-
1
require "httpx/resolver/resolver"
-
1
require "httpx/resolver/system"
-
1
require "httpx/resolver/native"
-
1
require "httpx/resolver/https"
-
1
require "httpx/resolver/multi"
-
-
1
@identifier_mutex = Thread::Mutex.new
-
1
@identifier = 1
-
-
1
def supported_ip_families
-
33
if Utils.in_ractor?
-
Ractor.store_if_absent(:httpx_supported_ip_families) { find_supported_ip_families }
-
else
-
33
@supported_ip_families ||= find_supported_ip_families
-
end
-
end
-
-
1
def generate_id
-
10
if Utils.in_ractor?
-
identifier = Ractor.store_if_absent(:httpx_resolver_identifier) { -1 }
-
Ractor.current[:httpx_resolver_identifier] = (identifier + 1) & 0xFFFF
-
else
-
20
id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
-
end
-
end
-
-
1
def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
-
10
Resolv::DNS::Message.new(message_id).tap do |query|
-
10
query.rd = 1
-
10
query.add_question(hostname, type)
-
end.encode
-
end
-
-
1
def decode_dns_answer(payload)
-
begin
-
10
message = Resolv::DNS::Message.decode(payload)
-
rescue Resolv::DNS::DecodeError => e
-
return :decode_error, e
-
end
-
-
# no domain was found
-
10
return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
-
-
1
return :message_truncated if message.tc == 1
-
-
1
if message.rcode != Resolv::DNS::RCode::NoError
-
case message.rcode
-
when Resolv::DNS::RCode::ServFail
-
return :retriable_error, message.rcode
-
else
-
return :dns_error, message.rcode
-
end
-
end
-
-
1
addresses = []
-
-
1
now = Utils.now
-
1
message.each_answer do |question, _, value|
-
1
case value
-
when Resolv::DNS::Resource::IN::CNAME
-
addresses << {
-
"name" => question.to_s,
-
"TTL" => (now + value.ttl),
-
"alias" => value.name.to_s,
-
}
-
when Resolv::DNS::Resource::IN::A,
-
Resolv::DNS::Resource::IN::AAAA
-
1
addresses << {
-
"name" => question.to_s,
-
1
"TTL" => (now + value.ttl),
-
"data" => value.address.to_s,
-
}
-
end
-
end
-
-
1
[:ok, addresses]
-
end
-
-
1
private
-
-
1
def id_synchronize(&block)
-
10
@identifier_mutex.synchronize(&block)
-
end
-
-
1
def find_supported_ip_families
-
1
list = Socket.ip_address_list
-
-
begin
-
4
if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
-
[Socket::AF_INET6, Socket::AF_INET]
-
else
-
1
[Socket::AF_INET]
-
end
-
rescue NotImplementedError
-
[Socket::AF_INET]
-
end.freeze
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "httpx/resolver/cache/base"
-
1
require "httpx/resolver/cache/memory"
-
-
1
module HTTPX::Resolver
-
# The internal resolvers cache adapters are defined under this namespace.
-
#
-
# Adapters must comply with the Resolver Cache Adapter API and implement the following methods:
-
#
-
# * #resolve: (String hostname) -> Array[HTTPX::Entry]? => resolves hostname to a list of cached IPs (if found in cache or system)
-
# * #get: (String hostname) -> Array[HTTPX::Entry]? => resolves hostname to a list of cached IPs (if found in cache)
-
# * #set: (String hostname, Integer ip_family, Array[dns_result]) -> void => stores the set of results in the cache indexes for
-
# the hostname and the IP family
-
# * #evict: (String hostname, _ToS ip) -> void => evicts the ip for the hostname from the cache (usually done when no longer reachable)
-
1
module Cache
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "resolv"
-
-
1
module HTTPX
-
1
module Resolver::Cache
-
# Base class of the Resolver Cache adapter implementations.
-
#
-
# While resolver caches are not required to inherit from this class, it nevertheless provides
-
# common useful functions for desired functionality, such as singleton object ractor-safe access,
-
# or a default #resolve implementation which deals with IPs and the system hosts file.
-
#
-
1
class Base
-
1
MAX_CACHE_SIZE = 512
-
1
CACHE_MUTEX = Thread::Mutex.new
-
1
HOSTS = Resolv::Hosts.new
-
1
@cache = nil
-
-
1
class << self
-
1
attr_reader :hosts_resolver
-
-
# returns the singleton instance to be used within the current ractor.
-
1
def cache(label)
-
51
return Ractor.store_if_absent(:"httpx_resolver_cache_#{label}") { new } if Utils.in_ractor?
-
-
51
@cache ||= CACHE_MUTEX.synchronize do
-
1
@cache || new
-
end
-
end
-
end
-
-
# resolves +hostname+ into an instance of HTTPX::Resolver::Entry if +hostname+ is an IP,
-
# or can be found in the cache, or can be found in the system hosts file.
-
1
def resolve(hostname)
-
32
ip_resolve(hostname) || get(hostname) || hosts_resolve(hostname)
-
end
-
-
1
private
-
-
# tries to convert +hostname+ into an IPAddr, returns <tt>nil</tt> otherwise.
-
1
def ip_resolve(hostname)
-
32
[Resolver::Entry.new(hostname)]
-
rescue ArgumentError
-
end
-
-
# matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
-
# found, or there is no hosts file.
-
1
def hosts_resolve(hostname)
-
4
ips = if Utils.in_ractor?
-
Ractor.store_if_absent(:httpx_hosts_resolver) { Resolv::Hosts.new }
-
else
-
4
HOSTS
-
end.getaddresses(hostname)
-
-
4
return if ips.empty?
-
-
ips.map { |ip| Resolver::Entry.new(ip) }
-
rescue IOError
-
end
-
-
# not to be used directly!
-
1
def _get(hostname, lookups, hostnames, ttl)
-
32
return unless lookups.key?(hostname)
-
-
28
entries = lookups[hostname]
-
-
28
return unless entries
-
-
28
entries.delete_if do |address|
-
28
address["TTL"] < ttl
-
end
-
-
28
if entries.empty?
-
lookups.delete(hostname)
-
hostnames.delete(hostname)
-
end
-
-
28
ips = entries.flat_map do |address|
-
28
if (als = address["alias"])
-
_get(als, lookups, hostnames, ttl)
-
else
-
28
Resolver::Entry.new(address["data"], address["TTL"])
-
end
-
end.compact
-
-
28
ips unless ips.empty?
-
end
-
-
1
def _set(hostname, family, entries, lookups, hostnames)
-
# lru cleanup
-
2
while lookups.size >= MAX_CACHE_SIZE
-
hs = hostnames.shift
-
lookups.delete(hs)
-
end
-
2
hostnames << hostname
-
-
2
lookups[hostname] ||= [] # when there's no default proc
-
-
2
case family
-
when Socket::AF_INET6
-
lookups[hostname].concat(entries)
-
when Socket::AF_INET
-
2
lookups[hostname].unshift(*entries)
-
end
-
2
entries.each do |entry|
-
2
name = entry["name"]
-
2
next unless name != hostname
-
-
1
_set(name, family, [entry], lookups, hostnames)
-
end
-
end
-
-
1
def _evict(hostname, ip, lookups, hostnames)
-
return unless lookups.key?(hostname)
-
-
entries = lookups[hostname]
-
-
return unless entries
-
-
entries.delete_if { |entry| entry["data"] == ip }
-
-
return unless entries.empty?
-
-
lookups.delete(hostname)
-
hostnames.delete(hostname)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Resolver::Cache
-
# Implementation of a thread-safe in-memory LRU resolver cache.
-
1
class Memory < Base
-
1
def initialize
-
1
super
-
1
@hostnames = []
-
3
@lookups = Hash.new { |h, k| h[k] = [] }
-
1
@lookup_mutex = Thread::Mutex.new
-
end
-
-
1
def get(hostname)
-
32
now = Utils.now
-
32
synchronize do |lookups, hostnames|
-
32
_get(hostname, lookups, hostnames, now)
-
end
-
end
-
-
1
def set(hostname, family, entries)
-
1
synchronize do |lookups, hostnames|
-
1
_set(hostname, family, entries, lookups, hostnames)
-
end
-
end
-
-
1
def evict(hostname, ip)
-
ip = ip.to_s
-
-
synchronize do |lookups, hostnames|
-
_evict(hostname, ip, lookups, hostnames)
-
end
-
end
-
-
1
private
-
-
1
def synchronize
-
66
@lookup_mutex.synchronize { yield(@lookups, @hostnames) }
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "ipaddr"
-
-
1
module HTTPX
-
1
module Resolver
-
1
class Entry < SimpleDelegator
-
1
attr_reader :address
-
-
1
def self.convert(address)
-
new(address, rescue_on_convert: true)
-
end
-
-
1
def initialize(address, expires_in = Float::INFINITY, rescue_on_convert: false)
-
61
@expires_in = expires_in
-
61
@address = address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
-
29
super(@address)
-
rescue IPAddr::InvalidAddressError
-
32
raise unless rescue_on_convert
-
-
@address = address.to_s
-
super(@address)
-
end
-
-
1
def expired?
-
2
@expires_in < Utils.now
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "resolv"
-
1
require "uri"
-
1
require "forwardable"
-
1
require "httpx/base64"
-
-
1
module HTTPX
-
# Implementation of a DoH name resolver (https://www.youtube.com/watch?v=unMXvnY2FNM).
-
# It wraps an HTTPX::Connection object which integrates with the main session in the
-
# same manner as other performed HTTP requests.
-
#
-
1
class Resolver::HTTPS < Resolver::Resolver
-
1
extend Forwardable
-
-
1
using URIExtensions
-
-
1
module DNSExtensions
-
1
refine Resolv::DNS do
-
1
def generate_candidates(name)
-
@config.generate_candidates(name)
-
end
-
end
-
end
-
1
using DNSExtensions
-
-
1
NAMESERVER = "https://1.1.1.1/dns-query"
-
-
DEFAULTS = {
-
1
uri: NAMESERVER,
-
use_get: false,
-
}.freeze
-
-
1
def_delegators :@resolver_connection, :connecting?, :to_io, :call, :close,
-
:closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout
-
-
1
def initialize(_, options)
-
super
-
@resolver_options = DEFAULTS.merge(@options.resolver_options)
-
@queries = {}
-
@requests = {}
-
@_timeouts = Array(@resolver_options[:timeouts])
-
@timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-
@uri = URI(@resolver_options[:uri])
-
@name = @uri_addresses = nil
-
@resolver = Resolv::DNS.new
-
@resolver.timeouts = @_timeouts.empty? ? Resolver::RESOLVE_TIMEOUT : @_timeouts
-
@resolver.lazy_initialize
-
end
-
-
1
def state
-
@resolver_connection ? @resolver_connection.state : :idle
-
end
-
-
1
def <<(connection)
-
return if @uri.origin == connection.peer.to_s
-
-
@uri_addresses ||= @options.resolver_cache.resolve(@uri.host) || @resolver.getaddresses(@uri.host)
-
-
if @uri_addresses.empty?
-
ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
-
ex.set_backtrace(caller)
-
connection.force_close
-
throw(:resolve_error, ex)
-
end
-
-
resolve(connection)
-
end
-
-
1
def resolver_connection
-
# TODO: leaks connection object into the pool
-
@resolver_connection ||=
-
@current_session.find_connection(
-
@uri,
-
@current_selector,
-
@options.merge(resolver_class: :system, ssl: { alpn_protocols: %w[h2] })
-
).tap do |conn|
-
emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
-
conn.on(:force_closed, &method(:force_close))
-
end
-
end
-
-
1
private
-
-
1
def resolve(connection = nil, hostname = nil)
-
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
connection ||= @connections.first
-
-
return unless connection
-
-
hostname ||= @queries.key(connection)
-
-
if hostname.nil?
-
hostname = connection.peer.host
-
log do
-
"resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
-
end if connection.peer.non_ascii_hostname
-
-
hostname = @resolver.generate_candidates(hostname).each do |name|
-
@queries[name.to_s] = connection
-
end.first.to_s
-
else
-
@queries[hostname] = connection
-
end
-
-
@name = hostname
-
-
log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
-
-
send_request(hostname, connection)
-
end
-
-
1
def send_request(hostname, connection)
-
request = build_request(hostname)
-
request.on(:response, &method(:on_response).curry(2)[request])
-
request.on(:promise, &method(:on_promise))
-
@requests[request] = hostname
-
resolver_connection.send(request)
-
@connections << connection
-
rescue ResolveError, Resolv::DNS::EncodeError => e
-
reset_hostname(hostname)
-
throw(:resolve_error, e) if connection.pending.empty?
-
emit_resolve_error(connection, connection.peer.host, e)
-
close_or_resolve
-
end
-
-
1
def on_response(request, response)
-
if (e = response.error)
-
hostname = @requests.delete(request)
-
connection = reset_hostname(hostname)
-
emit_resolve_error(connection, connection.peer.host, e)
-
close_or_resolve
-
else
-
# @type var response: HTTPX::Response
-
if response.status.between?(300, 399) && response.headers.key?("location")
-
hostname = @requests[request]
-
connection = @queries[hostname]
-
location_uri = URI(response.headers["location"])
-
location_uri = response.uri.merge(location_uri) if location_uri.relative?
-
-
# we assume that the DNS server URI changed permanently and move on
-
@uri = location_uri
-
send_request(hostname, connection)
-
return
-
end
-
-
parse(request, response)
-
end
-
ensure
-
@requests.delete(request)
-
end
-
-
1
def on_promise(_, stream)
-
log(level: 2) { "#{stream.id}: refusing stream!" }
-
stream.refuse
-
end
-
-
1
def parse(request, response)
-
hostname = @name
-
-
@name = nil
-
-
code, result = decode_response_body(response)
-
-
case code
-
when :ok
-
parse_addresses(result, request)
-
when :no_domain_found
-
# Indicates no such domain was found.
-
-
host = @requests.delete(request)
-
connection = reset_hostname(host, reset_candidates: false)
-
-
unless @queries.value?(connection)
-
emit_resolve_error(connection)
-
close_or_resolve
-
return
-
end
-
-
resolve
-
when :retriable_error
-
timeouts = @timeouts[hostname]
-
-
unless timeouts.empty?
-
log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
-
-
connection = @queries[hostname]
-
-
resolve(connection, hostname)
-
return
-
end
-
-
host = @requests.delete(request)
-
connection = reset_hostname(host)
-
-
emit_resolve_error(connection)
-
close_or_resolve
-
when :dns_error
-
host = @requests.delete(request)
-
connection = reset_hostname(host)
-
-
emit_resolve_error(connection)
-
close_or_resolve
-
when :decode_error
-
host = @requests.delete(request)
-
connection = reset_hostname(host)
-
emit_resolve_error(connection, connection.peer.host, result)
-
close_or_resolve
-
end
-
end
-
-
1
def parse_addresses(answers, request)
-
if answers.empty?
-
# no address found, eliminate candidates
-
host = @requests.delete(request)
-
connection = reset_hostname(host)
-
emit_resolve_error(connection)
-
close_or_resolve
-
return
-
-
else
-
answers = answers.group_by { |answer| answer["name"] }
-
answers.each do |hostname, addresses|
-
addresses = addresses.flat_map do |address|
-
if address.key?("alias")
-
alias_address = answers[address["alias"]]
-
if alias_address.nil?
-
reset_hostname(address["name"])
-
if early_resolve(connection, hostname: address["alias"])
-
@connections.delete(connection)
-
else
-
resolve(connection, address["alias"])
-
return # rubocop:disable Lint/NonLocalExitFromIterator
-
end
-
else
-
alias_address
-
end
-
else
-
address
-
end
-
end.compact
-
next if addresses.empty?
-
-
hostname.delete_suffix!(".") if hostname.end_with?(".")
-
connection = reset_hostname(hostname, reset_candidates: false)
-
next unless connection # probably a retried query for which there's an answer
-
-
@connections.delete(connection)
-
-
# eliminate other candidates
-
@queries.delete_if { |_, conn| connection == conn }
-
-
@options.resolver_cache.set(hostname, @family, addresses) if @resolver_options[:cache]
-
catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) }) }
-
end
-
end
-
close_or_resolve(true)
-
end
-
-
1
def build_request(hostname)
-
uri = @uri.dup
-
rklass = @options.request_class
-
payload = Resolver.encode_dns_query(hostname, type: @record_type)
-
timeouts = @timeouts[hostname]
-
request_timeout = timeouts.shift
-
options = @options.merge(timeout: { request_timeout: request_timeout })
-
-
if @resolver_options[:use_get]
-
params = URI.decode_www_form(uri.query.to_s)
-
params << ["type", FAMILY_TYPES[@record_type]]
-
params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
-
uri.query = URI.encode_www_form(params)
-
request = rklass.new("GET", uri, options)
-
else
-
request = rklass.new("POST", uri, options, body: [payload])
-
request.headers["content-type"] = "application/dns-message"
-
end
-
request.headers["accept"] = "application/dns-message"
-
request
-
end
-
-
1
def decode_response_body(response)
-
case response.headers["content-type"]
-
when "application/dns-udpwireformat",
-
"application/dns-message"
-
Resolver.decode_dns_answer(response.to_s)
-
else
-
raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
-
end
-
end
-
-
1
def reset_hostname(hostname, reset_candidates: true)
-
@timeouts.delete(hostname)
-
connection = @queries.delete(hostname)
-
-
return connection unless connection && reset_candidates
-
-
# eliminate other candidates
-
candidates = @queries.select { |_, conn| connection == conn }.keys
-
@queries.delete_if { |h, _| candidates.include?(h) }
-
# reset timeouts
-
@timeouts.delete_if { |h, _| candidates.include?(h) }
-
-
connection
-
end
-
-
1
def close_or_resolve(should_deactivate = false)
-
# drop already closed connections
-
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
if (@connections - @queries.values).empty?
-
# the same resolver connection may be serving different https resolvers (AAAA and A).
-
return if inflight?
-
-
if should_deactivate
-
deactivate
-
else
-
disconnect
-
end
-
else
-
resolve
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "forwardable"
-
1
require "resolv"
-
-
1
module HTTPX
-
1
class Resolver::Multi
-
1
attr_reader :resolvers, :options
-
-
1
def initialize(resolver_type, options)
-
32
@current_selector = @current_session = nil
-
32
@options = options
-
32
@resolver_options = @options.resolver_options
-
-
32
ip_families = options.ip_families || Resolver.supported_ip_families
-
-
32
@resolvers = ip_families.map do |ip_family|
-
32
resolver = resolver_type.new(ip_family, options)
-
32
resolver.multi = self
-
32
resolver
-
end
-
end
-
-
1
def state
-
@resolvers.map(&:state).uniq.join(",")
-
end
-
-
1
def current_selector=(s)
-
32
@current_selector = s
-
64
@resolvers.each { |r| r.current_selector = s }
-
end
-
-
1
def current_session=(s)
-
32
@current_session = s
-
64
@resolvers.each { |r| r.current_session = s }
-
end
-
-
1
def log(*args, **kwargs, &blk)
-
128
@resolvers.each { |r| r.log(*args, **kwargs, &blk) }
-
end
-
-
1
def closed?
-
32
@resolvers.all?(&:closed?)
-
end
-
-
1
def early_resolve(connection)
-
32
hostname = connection.peer.host
-
32
addresses = @resolver_options[:cache] && (connection.addresses || nolookup_resolve(hostname, connection.options))
-
32
return false unless addresses
-
-
28
ip_families = connection.options.ip_families
-
-
28
resolved = false
-
28
addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
-
28
next unless ip_families.nil? || ip_families.include?(family)
-
-
# try to match the resolver by family. However, there are cases where that's not possible, as when
-
# the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
-
56
resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
-
-
28
next unless resolver # this should ever happen
-
-
# it does not matter which resolver it is, as early-resolve code is shared.
-
28
resolver.emit_addresses(connection, family, addrs, true)
-
-
28
resolved = true
-
end
-
-
28
resolved
-
end
-
-
1
def lazy_resolve(connection)
-
4
@resolvers.each do |resolver|
-
4
resolver.lazy_resolve(connection)
-
end
-
end
-
-
1
private
-
-
1
def nolookup_resolve(hostname, options)
-
32
options.resolver_cache.resolve(hostname)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "forwardable"
-
1
require "resolv"
-
-
1
module HTTPX
-
# Implements a pure ruby name resolver, which abides by the Selectable API.
-
# It delegates DNS payload encoding/decoding to the +resolv+ stlid gem.
-
#
-
1
class Resolver::Native < Resolver::Resolver
-
1
extend Forwardable
-
-
1
using URIExtensions
-
-
DEFAULTS = {
-
1
nameserver: nil,
-
**Resolv::DNS::Config.default_config_hash,
-
packet_size: 512,
-
timeouts: Resolver::RESOLVE_TIMEOUT,
-
}.freeze
-
-
1
DNS_PORT = 53
-
-
1
def_delegator :@connections, :empty?
-
-
1
attr_reader :state
-
-
1
def initialize(family, options)
-
32
super
-
32
@ns_index = 0
-
32
@resolver_options = DEFAULTS.merge(@options.resolver_options)
-
32
@socket_type = @resolver_options.fetch(:socket_type, :udp)
-
32
@nameserver = if (nameserver = @resolver_options[:nameserver])
-
32
nameserver = nameserver[family] if nameserver.is_a?(Hash)
-
32
Array(nameserver)
-
end
-
32
@ndots = @resolver_options.fetch(:ndots, 1)
-
96
@search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
-
32
@_timeouts = Array(@resolver_options[:timeouts])
-
42
@timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-
32
@name = nil
-
32
@queries = {}
-
32
@read_buffer = "".b
-
32
@write_buffer = Buffer.new(@resolver_options[:packet_size])
-
32
@state = :idle
-
32
@timer = nil
-
end
-
-
1
def close
-
4
transition(:closed)
-
end
-
-
1
def force_close(*)
-
@timer.cancel if @timer
-
@timer = @name = nil
-
@queries.clear
-
@timeouts.clear
-
close
-
super
-
ensure
-
terminate
-
end
-
-
1
def terminate
-
disconnect
-
end
-
-
1
def closed?
-
36
@state == :idle || @state == :closed
-
end
-
-
1
def to_io
-
10
@io.to_io
-
end
-
-
1
def call
-
14
case @state
-
when :idle, :closed
-
4
return if @connections.empty?
-
-
4
transition(:idle) if @state == :closed
-
4
transition(:open)
-
-
4
consume if @state == :open
-
when :open
-
10
consume
-
end
-
end
-
-
1
def interests
-
10
case @state
-
when :idle
-
transition(:open)
-
when :closed
-
transition(:idle)
-
transition(:open)
-
end
-
-
10
calculate_interests
-
end
-
-
1
def <<(connection)
-
4
if @nameserver.nil?
-
ex = ResolveError.new("No available nameserver")
-
ex.set_backtrace(caller)
-
connection.force_close
-
throw(:resolve_error, ex)
-
else
-
4
@connections << connection
-
4
resolve
-
end
-
end
-
-
1
def timeout
-
10
return unless @name
-
-
10
@start_timeout = Utils.now
-
-
10
timeouts = @timeouts[@name]
-
-
10
return if timeouts.empty?
-
-
10
log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
-
-
10
timeouts.first
-
end
-
-
1
def handle_socket_timeout(interval); end
-
-
1
def handle_error(error)
-
if error.respond_to?(:connection) &&
-
error.respond_to?(:host)
-
reset_hostname(error.host, connection: error.connection)
-
else
-
@queries.each do |host, connection|
-
reset_hostname(host, connection: connection)
-
end
-
end
-
-
super
-
end
-
-
1
private
-
-
1
def calculate_interests
-
66
if @queries.empty?
-
2
return @io.interests if (@socket_type == :tcp) && (@state == :idle)
-
-
2
return
-
end
-
-
64
return :r if @write_buffer.empty?
-
-
14
:w
-
end
-
-
1
def consume
-
14
loop do
-
24
dread if calculate_interests == :r
-
-
24
break unless calculate_interests == :w
-
-
10
dwrite
-
-
10
break unless calculate_interests == :r
-
end
-
rescue Errno::EHOSTUNREACH => e
-
@ns_index += 1
-
nameserver = @nameserver
-
if nameserver && @ns_index < nameserver.size
-
log { "resolver #{FAMILY_TYPES[@record_type]}: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
-
transition(:idle)
-
@timeouts.clear
-
retry
-
else
-
handle_error(e)
-
disconnect
-
end
-
rescue NativeResolveError => e
-
handle_error(e)
-
close_or_resolve
-
retry unless closed?
-
end
-
-
1
def schedule_retry
-
10
h = @name
-
-
10
return unless h
-
-
10
connection = @queries[h]
-
-
10
timeouts = @timeouts[h]
-
10
timeout = timeouts.shift
-
-
10
@timer = @current_selector.after(timeout) do
-
next unless @connections.include?(connection)
-
-
@timer = @name = nil
-
-
do_retry(h, connection, timeout)
-
end
-
end
-
-
1
def do_retry(h, connection, interval)
-
timeouts = @timeouts[h]
-
-
if !timeouts.empty?
-
log { "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..." }
-
# must downgrade to tcp AND retry on same host as last
-
downgrade_socket
-
resolve(connection, h)
-
elsif @ns_index + 1 < @nameserver.size
-
# try on the next nameserver
-
@ns_index += 1
-
log do
-
"resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
-
end
-
transition(:idle)
-
@timeouts.clear
-
resolve(connection, h)
-
else
-
reset_hostname(h, reset_candidates: false)
-
-
unless @queries.empty?
-
resolve(connection)
-
return
-
end
-
-
@connections.delete(connection)
-
-
host = connection.peer.host
-
-
# This loop_time passed to the exception is bogus. Ideally we would pass the total
-
# resolve timeout, including from the previous retries.
-
ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
-
ex.set_backtrace(ex ? ex.backtrace : caller)
-
emit_resolve_error(connection, host, ex)
-
-
close_or_resolve
-
end
-
end
-
-
1
def dread(wsize = @resolver_options[:packet_size])
-
20
loop do
-
20
wsize = @large_packet.capacity if @large_packet
-
-
20
siz = @io.read(wsize, @read_buffer)
-
-
20
unless siz
-
ex = EOFError.new("descriptor closed")
-
ex.set_backtrace(caller)
-
raise ex
-
end
-
-
20
return unless siz.positive?
-
-
10
if @socket_type == :tcp
-
# packet may be incomplete, need to keep draining from the socket
-
if @large_packet
-
# large packet buffer already exists, continue pumping
-
@large_packet << @read_buffer
-
-
next unless @large_packet.full?
-
-
parse(@large_packet.to_s)
-
@large_packet = nil
-
# downgrade to udp again
-
downgrade_socket
-
return
-
else
-
size = @read_buffer[0, 2].unpack1("n")
-
buffer = @read_buffer.byteslice(2..-1)
-
-
if size > @read_buffer.bytesize
-
# only do buffer logic if it's worth it, and the whole packet isn't here already
-
@large_packet = Buffer.new(size)
-
@large_packet << buffer
-
-
next
-
else
-
parse(buffer)
-
end
-
end
-
else # udp
-
10
parse(@read_buffer)
-
end
-
-
10
return if @state == :closed || !@write_buffer.empty?
-
end
-
end
-
-
1
def dwrite
-
10
loop do
-
20
return if @write_buffer.empty?
-
-
10
siz = @io.write(@write_buffer)
-
-
10
unless siz
-
ex = EOFError.new("descriptor closed")
-
ex.set_backtrace(caller)
-
raise ex
-
end
-
-
10
return unless siz.positive?
-
-
10
schedule_retry if @write_buffer.empty?
-
-
10
return if @state == :closed
-
end
-
end
-
-
1
def parse(buffer)
-
10
code, result = Resolver.decode_dns_answer(buffer)
-
-
10
case code
-
when :ok
-
1
reset_query
-
1
parse_addresses(result)
-
when :no_domain_found
-
9
reset_query
-
# Indicates no such domain was found.
-
9
hostname, connection = @queries.first
-
9
reset_hostname(hostname, reset_candidates: false)
-
-
15
other_candidate, _ = @queries.find { |_, conn| conn == connection }
-
-
9
if other_candidate
-
6
resolve(connection, other_candidate)
-
else
-
3
@connections.delete(connection)
-
3
ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
-
3
ex.set_backtrace(ex ? ex.backtrace : caller)
-
3
emit_resolve_error(connection, connection.peer.host, ex)
-
3
close_or_resolve
-
end
-
when :message_truncated
-
reset_query
-
# TODO: what to do if it's already tcp??
-
return if @socket_type == :tcp
-
-
@socket_type = :tcp
-
-
hostname, _ = @queries.first
-
reset_hostname(hostname)
-
transition(:closed)
-
when :retriable_error
-
if @name && @timer
-
log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
-
return
-
end
-
# retry now!
-
# connection = @queries[@name].shift
-
# @timer.fire
-
reset_query
-
hostname, connection = @queries.first
-
reset_hostname(hostname)
-
@connections.delete(connection)
-
ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
-
raise ex
-
when :dns_error
-
reset_query
-
hostname, connection = @queries.first
-
reset_hostname(hostname)
-
@connections.delete(connection)
-
ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
-
raise ex
-
when :decode_error
-
reset_query
-
hostname, connection = @queries.first
-
reset_hostname(hostname)
-
@connections.delete(connection)
-
ex = NativeResolveError.new(connection, connection.peer.host, result.message)
-
ex.set_backtrace(result.backtrace)
-
raise ex
-
end
-
end
-
-
1
def parse_addresses(addresses)
-
1
if addresses.empty?
-
# no address found, eliminate candidates
-
hostname, connection = @queries.first
-
reset_hostname(hostname)
-
@connections.delete(connection)
-
raise NativeResolveError.new(connection, connection.peer.host)
-
else
-
1
address = addresses.first
-
1
name = address["name"]
-
-
1
connection = @queries.delete(name)
-
-
1
unless connection
-
1
orig_name = name
-
# absolute name
-
1
name_labels = Resolv::DNS::Name.create(name).to_a
-
1
name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
-
-
# probably a retried query for which there's an answer
-
1
unless name
-
@timeouts.delete(orig_name)
-
return
-
end
-
-
1
address["name"] = name
-
1
connection = @queries.delete(name)
-
end
-
-
2
alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
-
-
1
if addresses.empty? && !alias_addresses.empty? # CNAME
-
hostname_alias = alias_addresses.first["alias"]
-
# clean up intermediate queries
-
@timeouts.delete(name) unless connection.peer.host == name
-
-
if early_resolve(connection, hostname: hostname_alias)
-
@connections.delete(connection)
-
else
-
if @socket_type == :tcp
-
# must downgrade to udp if tcp
-
@socket_type = @resolver_options.fetch(:socket_type, :udp)
-
transition(:idle)
-
transition(:open)
-
end
-
log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
-
resolve(connection, hostname_alias)
-
return
-
end
-
else
-
1
reset_hostname(name, connection: connection)
-
1
@timeouts.delete(connection.peer.host)
-
1
@connections.delete(connection)
-
1
@options.resolver_cache.set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
-
1
catch(:coalesced) do
-
2
emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) })
-
end
-
end
-
end
-
1
close_or_resolve
-
end
-
-
1
def resolve(connection = nil, hostname = nil)
-
10
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
14
connection ||= @connections.find { |c| !@queries.value?(c) }
-
-
10
raise Error, "no URI to resolve" unless connection
-
-
# do not buffer query if previous is still in the buffer or awaiting reply/retry
-
10
return unless @write_buffer.empty? && @timer.nil?
-
-
10
hostname ||= @queries.key(connection)
-
-
10
if hostname.nil?
-
4
hostname = connection.peer.host
-
4
if connection.peer.non_ascii_hostname
-
log { "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" }
-
end
-
-
4
hostname = generate_candidates(hostname).each do |name|
-
12
@queries[name] = connection
-
end.first
-
else
-
6
@queries[hostname] = connection
-
end
-
-
10
@name = hostname
-
-
10
log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
-
begin
-
10
@write_buffer << encode_dns_query(hostname)
-
rescue Resolv::DNS::EncodeError => e
-
reset_hostname(hostname, connection: connection)
-
@connections.delete(connection)
-
emit_resolve_error(connection, hostname, e)
-
close_or_resolve
-
end
-
end
-
-
1
def encode_dns_query(hostname)
-
10
message_id = Resolver.generate_id
-
10
msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
-
10
msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
-
10
msg
-
end
-
-
1
def generate_candidates(name)
-
4
return [name] if name.end_with?(".")
-
-
4
name_parts = name.scan(/[^.]+/)
-
12
candidates = @search.map { |domain| [*name_parts, *domain].join(".") }
-
4
fname = "#{name}."
-
4
if @ndots <= name_parts.size - 1
-
4
candidates.unshift(fname)
-
else
-
candidates << fname
-
end
-
4
candidates
-
end
-
-
1
def build_socket
-
4
ip, port = @nameserver[@ns_index]
-
4
port ||= DNS_PORT
-
-
4
case @socket_type
-
when :udp
-
4
log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
-
4
UDP.new(ip, port, @options)
-
when :tcp
-
log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
-
origin = URI("tcp://#{ip}:#{port}")
-
TCP.new(origin, [Resolver::Entry.new(ip)], @options)
-
end
-
end
-
-
1
def downgrade_socket
-
return unless @socket_type == :tcp
-
-
@socket_type = @resolver_options.fetch(:socket_type, :udp)
-
transition(:idle)
-
transition(:open)
-
end
-
-
# moves the resolver state machine to the +nextstate+ state (if all conditions are met)-
-
1
def transition(nextstate)
-
8
case nextstate
-
when :idle
-
if (io = @io)
-
@io = nil
-
io.close
-
-
# @fiber-switch-guard
-
return if @io
-
end
-
when :open
-
4
return unless @state == :idle
-
-
4
@io ||= build_socket
-
-
4
@io.connect
-
4
return unless @io.connected?
-
-
4
resolve if @queries.empty? && !@connections.empty?
-
when :closed
-
4
return if @state == :closed
-
-
4
if (io = @io)
-
4
@io = nil
-
4
io.close
-
-
# @fiber-switch-guard
-
4
return if @io
-
end
-
-
4
@start_timeout = nil
-
4
@write_buffer.clear
-
4
@read_buffer.clear
-
end
-
8
log(level: 3) { "#{@state} -> #{nextstate}" }
-
8
@state = nextstate
-
rescue Errno::ECONNREFUSED,
-
Errno::EADDRNOTAVAIL,
-
Errno::EHOSTUNREACH,
-
SocketError,
-
IOError,
-
ConnectTimeoutError => e
-
# these errors may happen during TCP handshake
-
# treat them as resolve errors.
-
on_error(e)
-
end
-
-
1
def reset_query
-
10
@timer.cancel
-
-
10
@timer = @name = nil
-
end
-
-
1
def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
-
10
@timeouts.delete(hostname)
-
-
10
return unless connection && reset_candidates
-
-
# eliminate other candidates
-
3
candidates = @queries.select { |_, conn| connection == conn }.keys
-
3
@queries.delete_if { |h, _| candidates.include?(h) }
-
# reset timeouts
-
1
@timeouts.delete_if { |h, _| candidates.include?(h) }
-
end
-
-
1
def close_or_resolve
-
# drop already closed connections
-
4
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
4
if (@connections - @queries.values).empty?
-
4
disconnect
-
else
-
resolve
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "resolv"
-
-
1
module HTTPX
-
# Base class for all internal internet name resolvers. It handles basic blocks
-
# from the Selectable API.
-
#
-
1
class Resolver::Resolver
-
1
include Loggable
-
-
1
using ArrayExtensions::Intersect
-
-
RECORD_TYPES = {
-
1
Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
-
Socket::AF_INET => Resolv::DNS::Resource::IN::A,
-
}.freeze
-
-
FAMILY_TYPES = {
-
1
Resolv::DNS::Resource::IN::AAAA => "AAAA",
-
Resolv::DNS::Resource::IN::A => "A",
-
}.freeze
-
-
1
class << self
-
1
def multi?
-
32
true
-
end
-
end
-
-
1
attr_reader :family, :options
-
-
1
attr_writer :current_selector, :current_session
-
-
1
attr_accessor :multi
-
-
1
def initialize(family, options)
-
32
@family = family
-
32
@record_type = RECORD_TYPES[family]
-
32
@options = options
-
32
@connections = []
-
end
-
-
1
def each_connection(&block)
-
1
enum_for(__method__) unless block
-
-
1
return unless @connections
-
-
1
@connections.each(&block)
-
end
-
-
1
def close; end
-
-
1
alias_method :terminate, :close
-
-
1
def initial_call
-
4
call
-
end
-
-
1
def force_close(*args)
-
while (connection = @connections.shift)
-
connection.force_close(*args)
-
end
-
end
-
-
1
def closed?
-
true
-
end
-
-
1
def empty?
-
true
-
end
-
-
1
def inflight?
-
false
-
end
-
-
1
def emit_addresses(connection, family, addresses, early_resolve = false)
-
58
addresses.map! { |address| address.is_a?(Resolver::Entry) ? address : Resolver::Entry.new(address) }
-
-
# double emission check, but allow early resolution to work
-
29
conn_addrs = connection.addresses
-
29
return if !early_resolve && conn_addrs && !conn_addrs.empty? && !addresses.intersect?(conn_addrs)
-
-
29
log do
-
"resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
-
"answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
-
end
-
-
# do not apply resolution delay for non-dns name resolution
-
29
if !early_resolve &&
-
# just in case...
-
@current_selector &&
-
# resolution delay only applies to IPv4
-
family == Socket::AF_INET &&
-
# connection already has addresses and initiated/ended handshake
-
!connection.io &&
-
# no need to delay if not supporting dual stack / multi-homed IP
-
1
(connection.options.ip_families || Resolver.supported_ip_families).size > 1 &&
-
# connection URL host is already the IP (early resolve included perhaps?)
-
addresses.first.to_s != connection.peer.host.to_s
-
log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
-
-
@current_selector.after(0.05) do
-
# double emission check
-
unless connection.addresses && addresses.intersect?(connection.addresses)
-
emit_resolved_connection(connection, addresses, early_resolve)
-
end
-
end
-
else
-
29
emit_resolved_connection(connection, addresses, early_resolve)
-
end
-
end
-
-
1
def handle_error(error)
-
if error.respond_to?(:connection) &&
-
error.respond_to?(:host)
-
@connections.delete(error.connection)
-
emit_resolve_error(error.connection, error.host, error)
-
else
-
while (connection = @connections.shift)
-
emit_resolve_error(connection, connection.peer.host, error)
-
end
-
end
-
end
-
-
1
def on_io_error(e)
-
on_error(e)
-
force_close(true)
-
end
-
-
1
def on_error(error)
-
handle_error(error)
-
disconnect
-
end
-
-
1
def early_resolve(connection, hostname: connection.peer.host) # rubocop:disable Naming/PredicateMethod
-
addresses = @resolver_options[:cache] && (connection.addresses || @options.resolver_cache.resolve(hostname))
-
-
return false unless addresses
-
-
addresses = addresses.select { |addr| addr.family == @family }
-
-
return false if addresses.empty?
-
-
emit_addresses(connection, @family, addresses, true)
-
-
true
-
end
-
-
1
def lazy_resolve(connection)
-
4
return unless @current_session && @current_selector
-
-
4
conn_to_resolve = @current_session.try_clone_connection(connection, @current_selector, @family)
-
4
self << conn_to_resolve
-
-
4
return if empty?
-
-
# both the resolver and the connection it's resolving must be pinned to the session
-
4
@current_session.pin(conn_to_resolve, @current_selector)
-
4
@current_session.select_resolver(self, @current_selector)
-
end
-
-
1
private
-
-
1
def emit_resolved_connection(connection, addresses, early_resolve)
-
begin
-
29
connection.addresses = addresses
-
-
29
return if connection.state == :closed
-
-
29
resolve_connection(connection)
-
rescue StandardError => e
-
if early_resolve
-
connection.force_close
-
throw(:resolve_error, e)
-
else
-
emit_connection_error(connection, e)
-
end
-
end
-
end
-
-
1
def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
-
3
emit_connection_error(connection, resolve_error(hostname, ex))
-
end
-
-
1
def resolve_error(hostname, ex = nil)
-
3
return ex if ex.is_a?(ResolveError) || ex.is_a?(ResolveTimeoutError)
-
-
message = ex ? ex.message : "Can't resolve #{hostname}"
-
error = ResolveError.new(message)
-
error.set_backtrace(ex ? ex.backtrace : caller)
-
error
-
end
-
-
1
def resolve_connection(connection)
-
29
@current_session.__send__(:on_resolver_connection, connection, @current_selector)
-
end
-
-
1
def emit_connection_error(connection, error)
-
3
return connection.handle_connect_error(error) if connection.connecting?
-
-
connection.on_error(error)
-
end
-
-
1
def disconnect
-
4
close
-
-
4
return unless closed?
-
-
4
@current_session.deselect_resolver(self, @current_selector)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "resolv"
-
-
1
module HTTPX
-
# Implementation of a synchronous name resolver which relies on the system resolver,
-
# which is lib'c getaddrinfo function (abstracted in ruby via Addrinfo.getaddrinfo).
-
#
-
# Its main advantage is relying on the reference implementation for name resolution
-
# across most/all OSs which deploy ruby (it's what TCPSocket also uses), its main
-
# disadvantage is the inability to set timeouts / check socket for readiness events,
-
# hence why it relies on using the Timeout module, which poses a lot of problems for
-
# the selector loop, specially when network is unstable.
-
#
-
1
class Resolver::System < Resolver::Resolver
-
1
using URIExtensions
-
-
1
RESOLV_ERRORS = [Resolv::ResolvError,
-
Resolv::DNS::Requester::RequestError,
-
Resolv::DNS::EncodeError,
-
Resolv::DNS::DecodeError].freeze
-
-
1
DONE = 1
-
1
ERROR = 2
-
-
1
class AddrinfoTimeoutError < StandardError
-
end
-
-
1
class << self
-
1
def multi?
-
false
-
end
-
end
-
-
1
attr_reader :state
-
-
1
def initialize(options)
-
super(0, options)
-
@resolver_options = @options.resolver_options
-
resolv_options = @resolver_options.dup
-
timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
-
@_timeouts = Array(timeouts)
-
@timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
-
resolv_options.delete(:cache)
-
@queries = []
-
@ips = []
-
@pipe_mutex = Thread::Mutex.new
-
@state = :idle
-
end
-
-
1
def resolvers
-
return enum_for(__method__) unless block_given?
-
-
yield self
-
end
-
-
1
def multi
-
self
-
end
-
-
1
def empty?
-
@connections.empty?
-
end
-
-
1
def close
-
transition(:closed)
-
end
-
-
1
def force_close(*)
-
close
-
@queries.clear
-
@timeouts.clear
-
@ips.clear
-
super
-
end
-
-
1
def closed?
-
@state == :closed
-
end
-
-
1
def to_io
-
@pipe_read.to_io
-
end
-
-
1
def call
-
case @state
-
when :open
-
consume
-
end
-
nil
-
end
-
-
1
def interests
-
return if @queries.empty?
-
-
:r
-
end
-
-
1
def timeout
-
_, connection = @queries.first
-
-
return unless connection
-
-
timeouts = @timeouts[connection.peer.host]
-
-
return if timeouts.empty?
-
-
log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
-
-
timeouts.first
-
end
-
-
1
def lazy_resolve(connection)
-
@connections << connection
-
resolve
-
-
return if empty?
-
-
@current_session.select_resolver(self, @current_selector)
-
end
-
-
1
def early_resolve(_, **) # rubocop:disable Naming/PredicateMethod
-
false
-
end
-
-
1
def handle_socket_timeout(interval)
-
error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
-
error.set_backtrace(caller)
-
@queries.each do |_, connection| # rubocop:disable Style/HashEachMethods
-
emit_resolve_error(connection, connection.peer.host, error) if @connections.delete(connection)
-
end
-
-
while (connection = @connections.shift)
-
emit_resolve_error(connection, connection.peer.host, error)
-
end
-
-
close_or_resolve
-
end
-
-
1
private
-
-
1
def transition(nextstate)
-
case nextstate
-
when :idle
-
@timeouts.clear
-
when :open
-
return unless @state == :idle
-
-
@pipe_read, @pipe_write = IO.pipe
-
when :closed
-
return unless @state == :open
-
-
@pipe_write.close
-
@pipe_read.close
-
end
-
@state = nextstate
-
end
-
-
1
def consume
-
return if @connections.empty?
-
-
event = @pipe_read.read_nonblock(1, exception: false)
-
-
return if event == :wait_readable
-
-
raise ResolveError, "socket pipe closed unexpectedly" if event.nil?
-
-
case event.unpack1("C")
-
when DONE
-
*pair, addrs = @pipe_mutex.synchronize { @ips.pop }
-
if pair
-
@queries.delete(pair)
-
family, connection = pair
-
@connections.delete(connection)
-
-
catch(:coalesced) { emit_addresses(connection, family, addrs) }
-
end
-
when ERROR
-
*pair, error = @pipe_mutex.synchronize { @ips.pop }
-
if pair && error
-
@queries.delete(pair)
-
_, connection = pair
-
@connections.delete(connection)
-
-
emit_resolve_error(connection, connection.peer.host, error)
-
end
-
end
-
-
return disconnect if @connections.empty?
-
-
resolve
-
rescue StandardError => e
-
on_error(e)
-
end
-
-
1
def resolve(connection = nil, hostname = nil)
-
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
connection ||= @connections.first
-
-
raise Error, "no URI to resolve" unless connection
-
-
return unless @queries.empty?
-
-
hostname ||= connection.peer.host
-
scheme = connection.origin.scheme
-
log do
-
"resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
-
end if connection.peer.non_ascii_hostname
-
-
transition(:open)
-
-
ip_families = connection.options.ip_families || Resolver.supported_ip_families
-
-
ip_families.each do |family|
-
@queries << [family, connection]
-
end
-
async_resolve(connection, hostname, scheme)
-
consume
-
end
-
-
1
def async_resolve(connection, hostname, scheme)
-
families = connection.options.ip_families || Resolver.supported_ip_families
-
log { "resolver: query for #{hostname}" }
-
timeouts = @timeouts[connection.peer.host]
-
resolve_timeout = timeouts.first
-
-
Thread.start do
-
Thread.current.report_on_exception = false
-
begin
-
addrs = if resolve_timeout
-
-
Timeout.timeout(resolve_timeout, AddrinfoTimeoutError) do
-
__addrinfo_resolve(hostname, scheme)
-
end
-
else
-
__addrinfo_resolve(hostname, scheme)
-
end
-
addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
-
families.each do |family|
-
addresses = addrs[family]
-
next unless addresses
-
-
addresses.map!(&:ip_address)
-
addresses.uniq!
-
@pipe_mutex.synchronize do
-
@ips.unshift([family, connection, addresses])
-
@pipe_write.putc(DONE) unless @pipe_write.closed?
-
end
-
end
-
rescue StandardError => e
-
if e.is_a?(AddrinfoTimeoutError)
-
timeouts.shift
-
retry unless timeouts.empty?
-
e = ResolveTimeoutError.new(resolve_timeout, e.message)
-
e.set_backtrace(e.backtrace)
-
end
-
@pipe_mutex.synchronize do
-
families.each do |family|
-
@ips.unshift([family, connection, e])
-
@pipe_write.putc(ERROR) unless @pipe_write.closed?
-
end
-
end
-
end
-
end
-
Thread.pass
-
end
-
-
1
def close_or_resolve
-
# drop already closed connections
-
@connections.shift until @connections.empty? || @connections.first.state != :closed
-
-
if (@connections - @queries.map(&:last)).empty?
-
disconnect
-
else
-
resolve
-
end
-
end
-
-
1
def __addrinfo_resolve(host, scheme)
-
Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "objspace"
-
1
require "stringio"
-
1
require "tempfile"
-
1
require "fileutils"
-
1
require "forwardable"
-
-
1
module HTTPX
-
# Defines a HTTP response is handled internally, with a few properties exposed as attributes.
-
#
-
# It delegates the following methods to the corresponding HTTPX::Request:
-
#
-
# * HTTPX::Request#uri
-
# * HTTPX::Request#peer_address
-
#
-
# It implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads.
-
#
-
# It implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
-
# (of payload chunks).
-
#
-
1
class Response
-
1
extend Forwardable
-
1
include Callbacks
-
-
# the HTTP response status code
-
1
attr_reader :status
-
-
# an HTTPX::Headers object containing the response HTTP headers.
-
1
attr_reader :headers
-
-
# a HTTPX::Response::Body object wrapping the response body. The following methods are delegated to it:
-
#
-
# * HTTPX::Response::Body#to_s
-
# * HTTPX::Response::Body#to_str
-
# * HTTPX::Response::Body#read
-
# * HTTPX::Response::Body#copy_to
-
# * HTTPX::Response::Body#close
-
1
attr_reader :body
-
-
# The HTTP protocol version used to fetch the response.
-
1
attr_reader :version
-
-
# returns the response body buffered in a string.
-
1
def_delegator :@body, :to_s
-
-
1
def_delegator :@body, :to_str
-
-
# implements the IO reader +#read+ interface.
-
1
def_delegator :@body, :read
-
-
# copies the response body to a different location.
-
1
def_delegator :@body, :copy_to
-
-
# the corresponding request uri.
-
1
def_delegator :@request, :uri
-
-
# the IP address of the peer server.
-
1
def_delegator :@request, :peer_address
-
-
# inits the instance with the corresponding +request+ to this response, an the
-
# response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
-
1
def initialize(request, status, version, headers)
-
67
@request = request
-
67
@options = request.options
-
67
@version = version
-
67
@status = Integer(status)
-
67
@headers = @options.headers_class.new(headers)
-
67
@body = @options.response_body_class.new(self, @options)
-
67
@finished = complete?
-
67
@content_type = @content_length = nil
-
end
-
-
# dupped initialization
-
1
def initialize_dup(orig)
-
super
-
# if a response gets dupped, the body handle must also get dupped to prevent
-
# two responses from using the same file handle to read.
-
@body = orig.body.dup
-
end
-
-
# closes the respective +@request+ and +@body+.
-
1
def close
-
@request.close
-
@body.close
-
end
-
-
# merges headers defined in +h+ into the response headers.
-
1
def merge_headers(h)
-
@headers = @headers.merge(h)
-
@content_type = @content_length = nil
-
end
-
-
# writes +data+ chunk into the response body.
-
1
def <<(data)
-
60
@body.write(data)
-
end
-
-
# returns the HTTPX::ContentType for the response, as per what's declared in the content-type header.
-
#
-
# response.content_type #=> #<HTTPX::ContentType:xxx @header_value="text/plain">
-
# response.content_type.mime_type #=> "text/plain"
-
1
def content_type
-
67
@content_type ||= ContentType.new(@headers["content-type"])
-
end
-
-
# returns the response content length as advertised in the HTTP Content-Length header value.
-
1
def content_length
-
33
return @content_length if defined?(@content_length)
-
-
@content_length = @headers["content-length"]&.to_i
-
end
-
-
# returns whether the response has been fully fetched.
-
1
def finished?
-
95
@finished
-
end
-
-
# marks the response as finished, freezes the headers.
-
1
def finish!
-
67
@finished = true
-
67
@headers.freeze
-
67
@request.connection = nil
-
end
-
-
# returns whether the response contains body payload.
-
1
def bodyless?
-
67
@request.verb == "HEAD" ||
-
@status < 200 || # informational response
-
@status == 204 ||
-
@status == 205 ||
-
@status == 304 || begin
-
67
content_length = @headers["content-length"]
-
67
return false if content_length.nil?
-
-
33
content_length == "0"
-
end
-
end
-
-
1
def complete?
-
67
bodyless? || (@request.verb == "CONNECT" && @status == 200)
-
end
-
-
# simplecov:disable
-
1
def inspect
-
"#<#{self.class}:#{object_id} " \
-
"HTTP/#{version} " \
-
"@status=#{@status} " \
-
"@headers=#{@headers} " \
-
"@body=#{@body.bytesize}>"
-
end
-
# simplecov:enable
-
-
# returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
-
# status code, or nothing.
-
#
-
# ok_response.error #=> nil
-
# not_found_response.error #=> HTTPX::HTTPError instance, status 404
-
1
def error
-
3
return if @status < 400
-
-
HTTPError.new(self)
-
end
-
-
# it raises the exception returned by +error+, or itself otherwise.
-
#
-
# ok_response.raise_for_status #=> ok_response
-
# not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
-
1
def raise_for_status
-
3
return self unless (err = error)
-
-
raise err
-
end
-
-
# decodes the response payload into a ruby object **if** the payload is valid json.
-
#
-
# response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
-
# response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
-
1
def json(*args)
-
decode(Transcoder::JSON, *args)
-
end
-
-
# decodes the response payload into a ruby object **if** the payload is valid
-
# "application/x-www-urlencoded" or "multipart/form-data".
-
1
def form
-
decode(Transcoder::Form)
-
end
-
-
1
def xml
-
# TODO: remove at next major version.
-
warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
-
"Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
-
require "httpx/plugins/xml"
-
decode(Plugins::XML::Transcoder)
-
end
-
-
1
private
-
-
# decodes the response payload using the given +transcoder+, which implements the decoding logic.
-
#
-
# +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
-
# which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
-
1
def decode(transcoder, *args)
-
# TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
-
-
decoder = transcoder.decode(self)
-
-
raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
-
-
@body.rewind
-
-
decoder.call(self, *args)
-
end
-
end
-
-
# Helper class which decodes the HTTP "content-type" header.
-
1
class ContentType
-
1
MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
-
1
CHARSET_RE = /;\s*charset=([^;]+)/i.freeze
-
-
1
def initialize(header_value)
-
67
@header_value = header_value
-
67
@mime_type = @charset = nil
-
67
@initialized = false
-
end
-
-
# returns the mime type declared in the header.
-
#
-
# ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
-
1
def mime_type
-
return @mime_type if @initialized
-
-
load
-
-
@mime_type
-
end
-
-
# returns the charset declared in the header.
-
#
-
# ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
-
# ContentType.new("text/plain").charset #=> nil
-
1
def charset
-
67
return @charset if @initialized
-
-
67
load
-
-
67
@charset
-
end
-
-
1
private
-
-
1
def load
-
67
m = @header_value.to_s[MIME_TYPE_RE, 1]
-
67
m && @mime_type = m.strip.downcase
-
-
67
c = @header_value.to_s[CHARSET_RE, 1]
-
67
c && @charset = c.strip.delete('"')
-
-
67
@initialized = true
-
end
-
end
-
-
# Wraps an error which has happened while processing an HTTP Request. It has partial
-
# public API parity with HTTPX::Response, so users should rely on it to infer whether
-
# the returned response is one or the other.
-
#
-
# response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
-
# response.raise_for_status #=> raises if it wraps an error
-
1
class ErrorResponse
-
1
extend Forwardable
-
-
# the corresponding HTTPX::Request instance.
-
1
attr_reader :request
-
-
# the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
-
1
attr_reader :response
-
-
# the wrapped exception.
-
1
attr_reader :error
-
-
# the request uri
-
1
def_delegator :@request, :uri
-
-
# the IP address of the peer server.
-
1
def_delegator :@request, :peer_address
-
-
1
def initialize(request, error)
-
7
@request = request
-
7
@response = request.response if request.response.is_a?(Response)
-
7
@error = error
-
7
@options = request.options
-
7
@request.log_exception(@error)
-
7
finish!
-
end
-
-
# returns the exception full message.
-
1
def to_s
-
@error.full_message(highlight: false)
-
end
-
-
# closes the error resources.
-
1
def close
-
@response.close if @response
-
end
-
-
# always true for error responses.
-
1
def finished?
-
7
true
-
end
-
-
1
def finish!
-
11
@request.connection = nil
-
end
-
-
# raises the wrapped exception.
-
1
def raise_for_status
-
1
raise @error
-
end
-
-
# buffers lost chunks to error response
-
1
def <<(data)
-
return unless @response
-
-
@response << data
-
end
-
end
-
end
-
-
1
require_relative "response/body"
-
1
require_relative "response/buffer"
-
1
require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
# Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
-
# (for buffering the response payload), the IO reader protocol (for consuming the response payload),
-
# and can be iterated over (via #each, which yields the payload in chunks).
-
1
class Response::Body
-
# the payload encoding (i.e. "utf-8", "ASCII-8BIT")
-
1
attr_reader :encoding
-
-
# Array of encodings contained in the response "content-encoding" header.
-
1
attr_reader :encodings
-
-
1
attr_reader :buffer
-
1
protected :buffer
-
-
# initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
-
1
def initialize(response, options)
-
67
@response = response
-
67
@headers = response.headers
-
67
@options = options
-
67
@window_size = options.window_size
-
67
@max_response_body_size = options.max_response_body_size
-
67
@encodings = []
-
67
@length = 0
-
67
@buffer = @reader = nil
-
67
@state = :idle
-
-
# initialize response encoding
-
67
@encoding = if (enc = response.content_type.charset)
-
begin
-
16
Encoding.find(enc)
-
rescue ArgumentError
-
Encoding::BINARY
-
end
-
else
-
51
Encoding::BINARY
-
end
-
-
67
initialize_inflaters
-
end
-
-
1
def initialize_dup(other)
-
super
-
-
@buffer = other.instance_variable_get(:@buffer).dup
-
end
-
-
1
def closed?
-
@state == :closed
-
end
-
-
# write the response payload +chunk+ into the buffer. Inflates the chunk when required
-
# and supported.
-
1
def write(chunk)
-
56
return if @state == :closed
-
-
56
return 0 if chunk.empty?
-
-
29
chunk = decode_chunk(chunk)
-
-
29
raise Error, "maximum response body size exceeded" if @max_response_body_size < @length
-
-
29
transition(:open)
-
29
@buffer.write(chunk)
-
-
29
@response.emit(:chunk_received, chunk)
-
29
chunk.bytesize
-
end
-
-
# reads a chunk from the payload (implementation of the IO reader protocol).
-
1
def read(*args)
-
return unless @buffer
-
-
unless @reader
-
rewind
-
@reader = @buffer
-
end
-
-
@reader.read(*args)
-
end
-
-
# size of the decoded response payload. May differ from "content-length" header if
-
# response was encoded over-the-wire.
-
1
def bytesize
-
@length
-
end
-
-
# yields the payload in chunks.
-
1
def each
-
return enum_for(__method__) unless block_given?
-
-
begin
-
if @buffer
-
rewind
-
while (chunk = @buffer.read(@window_size))
-
yield(chunk.force_encoding(@encoding))
-
end
-
end
-
ensure
-
close
-
end
-
end
-
-
# returns the declared filename in the "contennt-disposition" header, when present.
-
1
def filename
-
return unless @headers.key?("content-disposition")
-
-
Utils.get_filename(@headers["content-disposition"])
-
end
-
-
# returns the full response payload as a string.
-
1
def to_s
-
31
return "".b unless @buffer
-
-
18
@buffer.to_s
-
end
-
-
1
alias_method :to_str, :to_s
-
-
# whether the payload is empty.
-
1
def empty?
-
1
@length.zero?
-
end
-
-
# copies the payload to +dest+.
-
#
-
# body.copy_to("path/to/file")
-
# body.copy_to(Pathname.new("path/to/file"))
-
# body.copy_to(File.new("path/to/file"))
-
1
def copy_to(dest)
-
return unless @buffer
-
-
rewind
-
-
if dest.respond_to?(:path) && @buffer.respond_to?(:path)
-
FileUtils.mv(@buffer.path, dest.path)
-
else
-
IO.copy_stream(@buffer, dest)
-
end
-
ensure
-
close
-
end
-
-
# closes/cleans the buffer, resets everything
-
1
def close
-
2
if @buffer
-
@buffer.close
-
@buffer = nil
-
end
-
2
@length = 0
-
2
transition(:closed)
-
end
-
-
1
def ==(other)
-
super || case other
-
when Response::Body
-
@buffer == other.buffer
-
else
-
@buffer = other
-
end
-
end
-
-
# simplecov:disable
-
1
def inspect
-
"#<#{self.class}:#{object_id} " \
-
"@state=#{@state} " \
-
"@length=#{@length}>"
-
end
-
# simplecov:enable
-
-
# rewinds the response payload buffer.
-
1
def rewind
-
return unless @buffer
-
-
# in case there's some reading going on
-
@reader = nil
-
-
@buffer.rewind
-
end
-
-
1
private
-
-
# prepares inflaters for the advertised encodings in "content-encoding" header.
-
1
def initialize_inflaters
-
67
@inflaters = nil
-
-
67
return unless @headers.key?("content-encoding")
-
-
1
return unless @options.decompress_response_body
-
-
1
@inflaters = @headers.get("content-encoding").filter_map do |encoding|
-
1
next if encoding == "identity"
-
-
1
inflater = self.class.initialize_inflater_by_encoding(encoding, @response)
-
-
# do not uncompress if there is no decoder available. In fact, we can't reliably
-
# continue decompressing beyond that, so ignore.
-
1
break unless inflater
-
-
1
@encodings << encoding
-
1
inflater
-
end
-
end
-
-
# passes the +chunk+ through all inflaters to decode it.
-
1
def decode_chunk(chunk)
-
@inflaters.reverse_each do |inflater|
-
chunk = inflater.call(chunk)
-
33
end if @inflaters
-
-
33
@length += chunk.bytesize
-
-
33
chunk
-
end
-
-
# tries transitioning the body STM to the +nextstate+.
-
1
def transition(nextstate)
-
31
case nextstate
-
when :open
-
29
return unless @state == :idle
-
-
23
@buffer = Response::Buffer.new(
-
threshold_size: @options.body_threshold_size,
-
bytesize: @length,
-
encoding: @encoding
-
)
-
when :closed
-
2
return if @state == :closed
-
end
-
-
25
@state = nextstate
-
end
-
-
1
class << self
-
1
def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
-
1
case encoding
-
when "gzip"
-
1
Transcoder::GZIP.decode(response, **kwargs)
-
when "deflate"
-
Transcoder::Deflate.decode(response, **kwargs)
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "delegate"
-
1
require "stringio"
-
1
require "tempfile"
-
-
1
module HTTPX
-
# wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
-
1
class Response::Buffer < SimpleDelegator
-
1
attr_reader :buffer
-
1
protected :buffer
-
-
# initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
-
# the initial +bytesize+, and the +encoding+.
-
1
def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
-
23
@threshold_size = threshold_size
-
23
@bytesize = bytesize
-
23
@encoding = encoding
-
23
@buffer = StringIO.new("".b)
-
23
super(@buffer)
-
end
-
-
1
def initialize_dup(other)
-
super
-
-
# create new descriptor in READ-ONLY mode
-
@buffer =
-
case other.buffer
-
when StringIO
-
StringIO.new(other.buffer.string, mode: File::RDONLY)
-
else
-
other.buffer.class.new(other.buffer.path, encoding: Encoding::BINARY, mode: File::RDONLY).tap do |temp|
-
FileUtils.copy_file(other.buffer.path, temp.path)
-
end
-
end
-
end
-
-
# size in bytes of the buffered content.
-
1
def size
-
@bytesize
-
end
-
-
# writes the +chunk+ into the buffer.
-
1
def write(chunk)
-
29
@bytesize += chunk.bytesize
-
29
try_upgrade_buffer
-
29
@buffer.write(chunk)
-
end
-
-
# returns the buffered content as a string.
-
1
def to_s
-
18
case @buffer
-
when StringIO
-
begin
-
18
@buffer.string.force_encoding(@encoding)
-
rescue ArgumentError
-
@buffer.string
-
end
-
when Tempfile
-
rewind
-
content = @buffer.read
-
begin
-
content.force_encoding(@encoding)
-
rescue ArgumentError # ex: unknown encoding name - utf
-
content
-
end
-
end
-
end
-
-
# closes the buffer.
-
1
def close
-
@buffer.close
-
@buffer.unlink if @buffer.respond_to?(:unlink)
-
end
-
-
1
def ==(other)
-
super || begin
-
return false unless other.is_a?(Response::Buffer)
-
-
buffer_pos = @buffer.pos
-
other_pos = other.buffer.pos
-
@buffer.rewind
-
other.buffer.rewind
-
begin
-
FileUtils.compare_stream(@buffer, other.buffer)
-
ensure
-
@buffer.pos = buffer_pos
-
other.buffer.pos = other_pos
-
end
-
end
-
end
-
-
1
private
-
-
# initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
-
# has been reached.
-
1
def try_upgrade_buffer
-
29
return unless @bytesize > @threshold_size
-
-
return if @buffer.is_a?(Tempfile)
-
-
aux = @buffer
-
-
@buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
-
-
if aux
-
aux.rewind
-
IO.copy_stream(aux, @buffer)
-
aux.close
-
end
-
-
__setobj__(@buffer)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "io/wait"
-
-
1
module HTTPX
-
#
-
# Implements the selector loop, where it registers and monitors "Selectable" objects.
-
#
-
# A Selectable object is an object which can calculate the **interests** (<tt>:r</tt>, <tt>:w</tt> or <tt>:rw</tt>,
-
# respectively "read", "write" or "read-write") it wants to monitor for, and returns (via <tt>to_io</tt> method) an
-
# IO object which can be passed to functions such as IO.select . More exhaustively, a Selectable **must** implement
-
# the following methods:
-
#
-
# state :: returns the state as a Symbol, must return <tt>:closed</tt> when disposed of resources.
-
# to_io :: returns the IO object.
-
# call :: gets called when the IO is ready.
-
# interests :: returns the current interests to monitor for, as described above.
-
# timeout :: returns nil or an integer, representing how long to wait for interests.
-
# handle_socket_timeout(Numeric) :: called when waiting for interest times out.
-
#
-
1
class Selector
-
1
extend Forwardable
-
-
1
READABLE = %i[rw r].freeze
-
1
WRITABLE = %i[rw w].freeze
-
-
1
private_constant :READABLE
-
1
private_constant :WRITABLE
-
-
1
def_delegator :@timers, :after
-
-
1
def_delegator :@selectables, :each
-
-
1
def initialize
-
70
@timers = Timers.new
-
70
@selectables = []
-
70
@is_timer_interval = false
-
end
-
-
1
def empty?
-
79
@selectables.empty? && @timers.empty?
-
end
-
-
# first time the registered selectables are added, there's probably work to do.
-
1
def initial_call
-
63
@selectables.each(&:initial_call)
-
end
-
-
1
def next_tick
-
79
catch(:jump_tick) do
-
79
timeout = next_timeout
-
79
if timeout && timeout.negative?
-
@timers.fire
-
throw(:jump_tick)
-
end
-
-
begin
-
79
select(timeout) do |c|
-
79
c.log(level: 2) { "[#{c.state}] selected from selector##{object_id} #{" after #{timeout} secs" unless timeout.nil?}..." }
-
-
79
c.call
-
end
-
-
79
@timers.fire
-
rescue TimeoutError => e
-
@timers.fire(e)
-
end
-
end
-
end
-
-
1
def terminate
-
# array may change during iteration
-
73
selectables = @selectables.reject(&:inflight?)
-
-
73
selectables.delete_if do |sel|
-
38
sel.terminate
-
38
sel.state == :closed
-
end
-
-
73
until selectables.empty?
-
next_tick
-
-
selectables &= @selectables
-
end
-
end
-
-
1
def find_resolver(options)
-
32
res = @selectables.find do |c|
-
1
c.is_a?(Resolver::Resolver) &&
-
options.resolver_options_match?(c.options)
-
end
-
-
32
res.multi if res
-
end
-
-
1
def each_connection(&block)
-
302
return enum_for(__method__) unless block
-
-
151
@selectables.each do |c|
-
13
case c
-
when Resolver::Resolver
-
1
c.each_connection(&block)
-
when Connection
-
12
yield c
-
end
-
end
-
end
-
-
1
def find_connection(request_uri, options)
-
75
each_connection.find do |connection|
-
9
connection.match?(request_uri, options)
-
end
-
end
-
-
1
def find_mergeable_connection(connection)
-
67
each_connection.find do |ch|
-
3
ch != connection && ch.mergeable?(connection)
-
end
-
end
-
-
# deregisters +io+ from selectables.
-
1
def deregister(io)
-
76
@selectables.delete(io)
-
end
-
-
# register +io+.
-
1
def register(io)
-
73
return if @selectables.include?(io)
-
-
73
@selectables << io
-
end
-
-
1
private
-
-
1
def select(interval, &block)
-
# do not cause an infinite loop here.
-
#
-
# this may happen if timeout calculation actually triggered an error which causes
-
# the connections to be reaped (such as the total timeout error) before #select
-
# gets called.
-
79
if @selectables.empty?
-
begin
-
sleep(interval)
-
rescue IOError
-
# @fiber-switch-guard
-
# in a fiber scheduler scenario, IOs may be closed by the scheduler and raised in a separate fiber
-
# on wakeup, which includes a sleep call.
-
end if interval
-
return
-
end
-
-
# @type var r: (selectable | Array[selectable])?
-
# @type var w: (selectable | Array[selectable])?
-
79
r, w = nil
-
-
79
@selectables.delete_if do |io|
-
81
interests = io.interests
-
-
81
is_closed = io.state == :closed
-
-
81
if is_closed
-
# the process by which io was closed may have already triggered the on_close callback,
-
# which already deregistered the io. this check prevents it from deleting the wrong io,
-
# because of https://bugs.ruby-lang.org/issues/22021 .
-
next(@selectables.include?(io))
-
end
-
-
81
if interests
-
79
io.log(level: 2) do
-
"[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
-
end
-
-
79
if READABLE.include?(interests)
-
51
r = r.nil? ? io : (Array(r) << io)
-
end
-
-
79
if WRITABLE.include?(interests)
-
28
w = w.nil? ? io : (Array(w) << io)
-
end
-
end
-
-
81
is_closed
-
end
-
-
79
case r
-
when Array
-
w = Array(w) unless w.nil?
-
-
select_many(r, w, interval, &block)
-
when nil
-
28
case w
-
when Array
-
select_many(r, w, interval, &block)
-
when nil
-
return unless interval && @selectables.any?
-
-
# no selectables
-
# TODO: replace with sleep?
-
select_many(r, w, interval, &block)
-
else
-
28
select_one(w, :w, interval, &block)
-
end
-
-
else
-
51
case w
-
when Array
-
select_many(Array(r), w, interval, &block)
-
when nil
-
51
select_one(r, :r, interval, &block)
-
else
-
if r == w
-
select_one(r, :rw, interval, &block)
-
else
-
select_many(Array(r), Array(w), interval, &block)
-
end
-
end
-
end
-
end
-
-
1
def select_many(r, w, interval, &block)
-
begin
-
readers, writers = ::IO.select(r, w, nil, interval)
-
rescue IOError => e
-
(Array(r) + Array(w)).each do |sel|
-
# TODO: is there a way to cheaply find the IO associated with the error?
-
sel.on_io_error(e)
-
end
-
rescue StandardError => e
-
(Array(r) + Array(w)).each do |sel|
-
sel.on_error(e)
-
end
-
-
return
-
rescue Exception => e # rubocop:disable Lint/RescueException
-
(Array(r) + Array(w)).each do |sel|
-
sel.force_close(true)
-
end
-
-
raise e
-
end
-
-
if readers.nil? && writers.nil? && interval
-
[*r, *w].each { |io| io.handle_socket_timeout(interval) }
-
return
-
end
-
-
if writers
-
readers.each do |io|
-
yield io
-
-
# so that we don't yield 2 times
-
writers.delete(io)
-
end if readers
-
-
writers.each(&block)
-
else
-
readers.each(&block) if readers
-
end
-
end
-
-
1
def select_one(io, interests, interval)
-
begin
-
result =
-
79
case interests
-
51
when :r then io.to_io.wait_readable(interval)
-
28
when :w then io.to_io.wait_writable(interval)
-
when :rw then rw_wait(io, interval)
-
end
-
rescue IOError => e
-
io.on_io_error(e)
-
-
return
-
rescue StandardError => e
-
io.on_error(e)
-
-
return
-
rescue Exception => e # rubocop:disable Lint/RescueException
-
io.force_close(true)
-
-
raise e
-
end
-
-
79
unless result || interval.nil?
-
io.handle_socket_timeout(interval) unless @is_timer_interval
-
return
-
end
-
-
79
yield io
-
end
-
-
1
def next_timeout
-
79
@is_timer_interval = false
-
-
79
timer_interval = @timers.wait_interval
-
-
79
connection_interval = @selectables.filter_map(&:timeout).min
-
-
79
return connection_interval unless timer_interval
-
-
43
if connection_interval.nil? || timer_interval <= connection_interval
-
43
@is_timer_interval = true
-
-
43
return timer_interval
-
end
-
-
connection_interval
-
end
-
-
1
if RUBY_ENGINE == "jruby"
-
def rw_wait(io, interval)
-
io.to_io.wait(interval, :read_write)
-
end
-
1
elsif IO.const_defined?(:READABLE)
-
1
def rw_wait(io, interval)
-
io.to_io.wait(IO::READABLE | IO::WRITABLE, interval)
-
end
-
else
-
def rw_wait(io, interval)
-
if interval
-
io.to_io.wait(interval, :read_write)
-
else
-
io.to_io.wait(:read_write)
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
# Class implementing the APIs being used publicly.
-
#
-
# HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
-
# HTTPX.plugin(..).get(..) #=> creating an intermediate HTTPX::Session with plugin, then sending the GET request
-
1
class Session
-
1
include Loggable
-
1
include Chainable
-
-
# initializes the session with a set of +options+, which will be shared by all
-
# requests sent from it.
-
#
-
# When pass a block, it'll yield itself to it, then closes after the block is evaluated.
-
1
def initialize(options = EMPTY_HASH, &blk)
-
159
@options = self.class.default_options.merge(options)
-
159
@persistent = @options.persistent
-
159
@pool = @options.pool_class.new(@options.pool_options)
-
159
@wrapped = false
-
159
@closing = false
-
159
INSTANCES[self] = self if @persistent && @options.close_on_fork && INSTANCES
-
159
wrap(&blk) if blk
-
end
-
-
# Yields itself the block, then closes it after the block is evaluated.
-
#
-
# session.wrap do |http|
-
# http.get("https://wikipedia.com")
-
# end # wikipedia connection closes here
-
1
def wrap
-
prev_wrapped = @wrapped
-
@wrapped = true
-
was_initialized = false
-
current_selector = get_current_selector do
-
selector = Selector.new
-
-
set_current_selector(selector)
-
-
was_initialized = true
-
-
selector
-
end
-
begin
-
yield self
-
ensure
-
unless prev_wrapped
-
if @persistent
-
deactivate(current_selector)
-
else
-
close(current_selector)
-
end
-
end
-
@wrapped = prev_wrapped
-
set_current_selector(nil) if was_initialized
-
end
-
end
-
-
# closes all the active connections from the session.
-
#
-
# when called directly without specifying +selector+, all available connections
-
# will be picked up from the connection pool and closed. Connections in use
-
# by other sessions, or same session in a different thread, will not be reaped.
-
1
def close(selector = Selector.new)
-
# throw resolvers away from the pool
-
64
@pool.reset_resolvers
-
-
# preparing to throw away connections
-
156
while (connection = @pool.pop_connection)
-
28
next if connection.state == :closed
-
-
select_connection(connection, selector)
-
end
-
-
64
selector_close(selector)
-
end
-
-
# performs one, or multple requests; it accepts:
-
#
-
# 1. one or multiple HTTPX::Request objects;
-
# 2. an HTTP verb, then a sequence of URIs or URI/options tuples;
-
# 3. one or multiple HTTP verb / uri / (optional) options tuples;
-
#
-
# when present, the set of +options+ kwargs is applied to all of the
-
# sent requests.
-
#
-
# respectively returns a single HTTPX::Response response, or all of them in an Array, in the same order.
-
#
-
# resp1 = session.request(req1)
-
# resp1, resp2 = session.request(req1, req2)
-
# resp1 = session.request("GET", "https://server.org/a")
-
# resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"])
-
# resp1, resp2 = session.request(["GET", "https://server.org/a"], ["GET", "https://server.org/b"])
-
# resp1 = session.request("POST", "https://server.org/a", form: { "foo" => "bar" })
-
# resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
-
# resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
-
#
-
1
def request(*args, **params)
-
64
raise ArgumentError, "must perform at least one request" if args.empty?
-
-
64
requests = args.first.is_a?(Request) ? args : build_requests(*args, params)
-
64
responses = send_requests(*requests)
-
63
return responses.first if responses.size == 1
-
-
6
responses
-
end
-
-
# returns a HTTP::Request instance built from the HTTP +verb+, the request +uri+, and
-
# the optional set of request-specific +options+. This request **must** be sent through
-
# the same session it was built from.
-
#
-
# req = session.build_request("GET", "https://server.com")
-
# resp = session.request(req)
-
1
def build_request(verb, uri, params = EMPTY_HASH, options = @options)
-
71
rklass = options.request_class
-
71
request = rklass.new(verb, uri, options, params)
-
71
request.persistent = @persistent
-
71
set_request_callbacks(request)
-
71
request
-
end
-
-
1
def select_connection(connection, selector)
-
73
pin(connection, selector)
-
73
connection.log(level: 2) do
-
"registering into selector##{selector.object_id}"
-
end
-
73
selector.register(connection)
-
end
-
-
1
def pin(conn_or_resolver, selector)
-
141
conn_or_resolver.current_session = self
-
141
conn_or_resolver.current_selector = selector
-
end
-
-
1
alias_method :select_resolver, :select_connection
-
-
1
def deselect_connection(connection, selector, cloned = false)
-
72
connection.log(level: 2) do
-
"deregistering connection##{connection.object_id}(#{connection.state}) from selector##{selector.object_id}"
-
end
-
72
selector.deregister(connection)
-
-
# do not check-in connections only created for Happy Eyeballs
-
72
return if cloned
-
-
72
return if @closing && connection.state == :closed && !connection.used?
-
-
38
connection.log(level: 2) { "check-in connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
-
38
@pool.checkin_connection(connection)
-
end
-
-
1
def deselect_resolver(resolver, selector)
-
4
resolver.log(level: 2) do
-
"deregistering resolver##{resolver.object_id}(#{resolver.state}) from selector##{selector.object_id}"
-
end
-
4
selector.deregister(resolver)
-
-
4
return if @closing && resolver.closed?
-
-
4
resolver.log(level: 2) { "check-in resolver##{resolver.object_id}(#{resolver.state}) in pool##{@pool.object_id}" }
-
4
@pool.checkin_resolver(resolver)
-
end
-
-
1
def try_clone_connection(connection, selector, family)
-
4
connection.family ||= family
-
-
4
return connection if connection.family == family
-
-
new_connection = connection.class.new(connection.origin, connection.options)
-
-
new_connection.family = family
-
-
connection.sibling = new_connection
-
-
do_init_connection(new_connection, selector)
-
new_connection
-
end
-
-
# returns the HTTPX::Connection through which the +request+ should be sent through.
-
1
def find_connection(request_uri, selector, options)
-
75
if (connection = selector.find_connection(request_uri, options))
-
7
connection.idling if connection.state == :closed
-
7
log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
-
7
return connection
-
end
-
-
68
connection = @pool.checkout_connection(request_uri, options)
-
-
68
log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
-
-
68
case connection.state
-
when :idle
-
66
do_init_connection(connection, selector)
-
when :open
-
# external io
-
select_connection(connection, selector)
-
when :closing, :closed
-
2
connection.idling
-
2
if connection.addresses?
-
2
select_connection(connection, selector)
-
else
-
# if addresses expired, resolve again
-
resolve_connection(connection, selector)
-
end
-
else
-
pin(connection, selector)
-
end
-
-
68
connection
-
end
-
-
1
private
-
-
1
def selector_close(selector)
-
begin
-
73
@closing = true
-
73
selector.terminate
-
ensure
-
73
@closing = false
-
end
-
end
-
-
# tries deactivating connections in the +selector+, deregistering the ones that have been deactivated.
-
1
def deactivate(selector)
-
9
selector.each_connection.to_a.each(&:deactivate)
-
end
-
-
# callback executed when an HTTP/2 promise frame has been received.
-
1
def on_promise(_, stream)
-
log(level: 2) { "#{stream.id}: refusing stream!" }
-
stream.refuse
-
end
-
-
# returns the corresponding HTTP::Response to the given +request+ if it has been received.
-
1
def fetch_response(request, _selector, _options)
-
110
response = request.response
-
-
110
return unless response && response.finished?
-
-
74
log(level: 2) { "response##{response.object_id} fetched" }
-
-
74
response
-
end
-
-
# sends the +request+ to the corresponding HTTPX::Connection
-
1
def send_request(request, selector, options = request.options)
-
error = begin
-
75
catch(:resolve_error) do
-
75
log(level: 2) { "finding connection for request##{request.object_id}..." }
-
75
connection = find_connection(request.uri, selector, options)
-
75
connection.send(request)
-
end
-
rescue StandardError => e
-
e
-
end
-
74
return unless error && error.is_a?(Exception)
-
-
raise error unless error.is_a?(Error)
-
-
response = ErrorResponse.new(request, error)
-
request.response = response
-
request.emit_response(response)
-
end
-
-
# returns a set of HTTPX::Request objects built from the given +args+ and +options+.
-
1
def build_requests(*args, params)
-
55
requests = if args.size == 1
-
1
reqs = args.first
-
1
reqs.map do |verb, uri, ps = EMPTY_HASH|
-
2
request_params = params
-
2
request_params = request_params.merge(ps) unless ps.empty?
-
2
build_request(verb, uri, request_params)
-
end
-
else
-
54
verb, uris = args
-
54
if uris.respond_to?(:each)
-
54
uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
-
59
request_params = params
-
59
request_params = request_params.merge(ps) unless ps.empty?
-
59
build_request(verb, uri, request_params)
-
end
-
else
-
[build_request(verb, uris, params)]
-
end
-
end
-
55
raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
-
-
55
requests
-
end
-
-
1
def set_request_callbacks(request)
-
71
request.on(:promise, &method(:on_promise))
-
end
-
-
1
def do_init_connection(connection, selector)
-
66
resolve_connection(connection, selector) unless connection.family
-
end
-
-
# sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
-
1
def send_requests(*requests)
-
125
selector = get_current_selector { Selector.new }
-
begin
-
64
receive_requests(requests, selector)
-
ensure
-
64
unless @wrapped
-
64
if @persistent
-
9
deactivate(selector)
-
else
-
55
close(selector)
-
end
-
end
-
end
-
end
-
-
# returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
-
1
def receive_requests(requests, selector)
-
64
pending_idxs = [] #: Array[Integer]
-
64
pending = 0
-
-
64
waiting = false
-
-
64
requests.each do |request|
-
70
send_request(request, selector)
-
end
-
-
# do work first
-
63
selector.initial_call
-
-
63
responses = requests.each_with_index.map do |request, idx|
-
69
fetch_response(request, selector, request.options).tap do |response|
-
69
if response.nil?
-
34
pending += 1
-
34
request.on_response_arrived = lambda do
-
36
pending_idxs << idx if waiting
-
end
-
end
-
end
-
end
-
-
63
until pending.zero? || selector.empty?
-
# loop on selector until at least one response has been received.
-
79
waiting = true
-
158
catch(:coalesced) { selector.next_tick }
-
79
waiting = false
-
-
194
while (idx = pending_idxs.shift)
-
36
request = requests[idx]
-
-
36
response = fetch_response(request, selector, request.options)
-
-
# stop on first pending response. this avoids traversing pending idxs all the way
-
# (which is more expensive in the beginning, when the array is larger and N) while
-
# making the next loop cheaper (because we're dropping).
-
36
next unless response
-
-
34
request.complete!(response)
-
34
responses[idx] = response
-
34
request.on_response_arrived = nil
-
34
pending -= 1
-
end
-
end
-
-
63
raise Error, "something went wrong, responses not found and requests not resent" unless pending.zero?
-
-
63
responses
-
end
-
-
1
def resolve_connection(connection, selector)
-
70
if connection.addresses? || connection.open?
-
#
-
# there are two cases in which we want to activate initialization of
-
# connection immediately:
-
#
-
# 1. when the connection already has addresses, i.e. it doesn't need to
-
# resolve a name (not the same as name being an IP, yet)
-
# 2. when the connection is initialized with an external already open IO.
-
#
-
38
on_resolver_connection(connection, selector)
-
38
return
-
end
-
-
32
resolver = find_resolver_for(connection, selector)
-
-
32
pin(connection, selector)
-
32
if early_resolve(resolver, connection)
-
28
@pool.checkin_resolver(resolver)
-
else
-
4
resolver.lazy_resolve(connection)
-
end
-
end
-
-
1
def early_resolve(resolver, connection)
-
32
resolver.early_resolve(connection)
-
end
-
-
1
def on_resolver_connection(connection, selector)
-
67
from_pool = false
-
67
found_connection = selector.find_mergeable_connection(connection) || begin
-
67
from_pool = true
-
67
connection.log(level: 2) do
-
"try finding a mergeable connection in pool##{@pool.object_id}"
-
end
-
67
@pool.checkout_mergeable_connection(connection)
-
end
-
-
67
return select_connection(connection, selector) unless found_connection
-
-
connection.log(level: 2) do
-
"try coalescing from #{from_pool ? "pool##{@pool.object_id}" : "selector##{selector.object_id}"} " \
-
"(connection##{found_connection.object_id}[#{found_connection.origin}])"
-
end
-
-
coalesce_connections(found_connection, connection, selector, from_pool)
-
end
-
-
1
def find_resolver_for(connection, selector)
-
32
if (resolver = selector.find_resolver(connection.options))
-
resolver.log(level: 2) { "found resolver##{resolver.object_id}(#{resolver.state}) in selector##{selector.object_id}" }
-
return resolver
-
end
-
-
32
resolver = @pool.checkout_resolver(connection.options)
-
32
resolver.log(level: 2) { "found resolver##{resolver.object_id}(#{resolver.state}) in pool##{@pool.object_id}" }
-
32
pin(resolver, selector)
-
-
32
resolver
-
end
-
-
# coalesces +conn2+ into +conn1+. if +conn1+ was loaded from the connection pool
-
# (it is known via +from_pool+), then it adds its to the +selector+.
-
1
def coalesce_connections(conn1, conn2, selector, from_pool)
-
unless conn1.coalescable?(conn2)
-
conn2.log(level: 2) { "not coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
-
select_connection(conn2, selector)
-
if from_pool
-
conn1.log(level: 2) { "check-in connection##{conn1.object_id}(#{conn1.state}) in pool##{@pool.object_id}" }
-
@pool.checkin_connection(conn1)
-
end
-
return
-
end
-
-
conn2.log(level: 2) { "coalescing with connection##{conn1.object_id}[#{conn1.origin}])" }
-
select_connection(conn1, selector) if from_pool
-
conn2.coalesce!(conn1)
-
conn2.disconnect
-
end
-
-
1
def get_current_selector
-
64
selector_store[self] || (yield if block_given?)
-
end
-
-
1
def set_current_selector(selector)
-
12
if selector
-
12
selector_store[self] = selector
-
else
-
selector_store.delete(self)
-
end
-
end
-
-
1
def selector_store
-
76
th_current = Thread.current
-
-
76
thread_selector_store(th_current) || begin
-
4
{}.compare_by_identity.tap do |store|
-
4
th_current.thread_variable_set(:httpx_persistent_selector_store, store)
-
end
-
end
-
end
-
-
1
def thread_selector_store(th)
-
148
th.thread_variable_get(:httpx_persistent_selector_store)
-
end
-
-
1
Options.freeze
-
1
@default_options = Options.new
-
1
@default_options.freeze
-
1
@plugins = []
-
-
1
class << self
-
1
attr_reader :default_options
-
-
1
def inherited(klass)
-
106
super
-
106
klass.instance_variable_set(:@default_options, @default_options)
-
106
klass.instance_variable_set(:@plugins, @plugins.dup)
-
106
klass.instance_variable_set(:@callbacks, @callbacks.dup)
-
end
-
-
# returns a new HTTPX::Session instance, with the plugin pointed by +pl+ loaded.
-
#
-
# session_with_retries = session.plugin(:retries)
-
# session_with_custom = session.plugin(CustomPlugin)
-
#
-
1
def plugin(pl, options = nil, &block)
-
125
label = pl
-
125
pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
-
125
raise ArgumentError, "Invalid plugin type: #{pl.class.inspect}" unless pl.is_a?(Module)
-
-
125
if !@plugins.include?(pl)
-
85
@plugins << pl
-
85
pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
-
-
85
@default_options = @default_options.dup
-
-
85
include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
-
85
extend(pl::ClassMethods) if defined?(pl::ClassMethods)
-
-
85
opts = @default_options
-
85
opts.extend_with_plugin_classes(pl)
-
-
85
if defined?(pl::OptionsMethods)
-
# when a class gets dup'ed, the #initialize_dup callbacks isn't triggered.
-
# moreover, and because #method_added does not get triggered on mixin include,
-
# the callback is also forcefully manually called here.
-
16
opts.options_class.instance_variable_set(:@options_names, opts.options_class.options_names.dup)
-
16
(pl::OptionsMethods.instance_methods + pl::OptionsMethods.private_instance_methods - Object.instance_methods).each do |meth|
-
69
opts.options_class.method_added(meth)
-
end
-
16
@default_options = opts.options_class.new(opts)
-
end
-
-
85
@default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
-
85
@default_options = @default_options.merge(options) if options
-
-
85
if pl.respond_to?(:subplugins)
-
12
pl.subplugins.transform_keys(&Plugins.method(:load_plugin)).each do |main_pl, sub_pl|
-
# in case the main plugin has already been loaded, then apply subplugin functionality
-
# immediately
-
24
next unless @plugins.include?(main_pl)
-
-
plugin(sub_pl, options, &block)
-
end
-
end
-
-
85
pl.configure(self, &block) if pl.respond_to?(:configure)
-
-
85
if label.is_a?(Symbol)
-
# in case an already-loaded plugin complements functionality of
-
# the plugin currently being loaded, loaded it now
-
37
@plugins.each do |registered_pl|
-
96
next if registered_pl == pl
-
-
59
next unless registered_pl.respond_to?(:subplugins)
-
-
18
sub_pl = registered_pl.subplugins[label]
-
-
18
next unless sub_pl
-
-
plugin(sub_pl, options, &block)
-
end
-
end
-
-
85
@default_options.freeze
-
85
set_temporary_name("#{superclass}/#{pl}") if respond_to?(:set_temporary_name) # ruby 3.4 only
-
40
elsif options
-
# this can happen when two plugins are loaded, an one of them calls the other under the hood,
-
# albeit changing some default.
-
@default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
-
@default_options = @default_options.merge(options) if options
-
-
@default_options.freeze
-
end
-
-
125
self
-
end
-
end
-
-
# setup of the support for close_on_fork sessions.
-
# adapted from https://github.com/mperham/connection_pool/blob/main/lib/connection_pool.rb#L48
-
1
if Process.respond_to?(:fork)
-
1
INSTANCES = ObjectSpace::WeakMap.new
-
1
private_constant :INSTANCES
-
-
1
def self.after_fork
-
INSTANCES.each_value(&:close)
-
nil
-
end
-
-
1
if ::Process.respond_to?(:_fork)
-
1
module ForkTracker
-
1
def _fork
-
pid = super
-
Session.after_fork if pid.zero?
-
pid
-
end
-
end
-
1
Process.singleton_class.prepend(ForkTracker)
-
end
-
else
-
INSTANCES = nil
-
private_constant :INSTANCES
-
-
def self.after_fork
-
# noop
-
end
-
end
-
end
-
-
# session may be overridden by certain adapters.
-
1
S = Session
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
unless ENV.keys.grep(/\Ahttps?_proxy\z/i).empty?
-
proxy_session = plugin(:proxy)
-
remove_const(:Session)
-
const_set(:Session, proxy_session.class)
-
-
# redefine the default options static var, which needs to
-
# refresh options_class
-
options = proxy_session.class.default_options.to_hash
-
original_verbosity = $VERBOSE
-
$VERBOSE = nil
-
new_options_class = proxy_session.class.default_options.options_class.dup
-
const_set(:Options, new_options_class)
-
options[:options_class] = Class.new(new_options_class).freeze
-
options.freeze
-
Options.send(:const_set, :DEFAULT_OPTIONS, options)
-
Session.instance_variable_set(:@default_options, Options.new(options))
-
$VERBOSE = original_verbosity
-
end
-
-
# simplecov:disable
-
1
if Session.default_options.debug_level > 2
-
proxy_session = plugin(:internal_telemetry)
-
remove_const(:Session)
-
const_set(:Session, proxy_session.class)
-
end
-
# simplecov:enable
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
class Timers
-
1
def initialize
-
70
@intervals = []
-
end
-
-
1
def empty?
-
@intervals.empty?
-
end
-
-
1
def after(interval_in_secs, cb = nil, &blk)
-
78
callback = cb || blk
-
-
78
raise Error, "timer must have a callback" unless callback
-
-
# I'm assuming here that most requests will have the same
-
# request timeout, as in most cases they share common set of
-
# options. A user setting different request timeouts for 100s of
-
# requests will already have a hard time dealing with that.
-
120
unless (interval = @intervals.bsearch { |t| t.interval == interval_in_secs })
-
36
interval = Interval.new(interval_in_secs)
-
36
@intervals << interval
-
36
@intervals.sort!
-
end
-
-
78
interval << callback
-
-
78
@next_interval_at = nil
-
-
78
Timer.new(interval, callback)
-
end
-
-
1
def wait_interval
-
79
return if @intervals.empty?
-
-
43
first_interval = @intervals.first
-
-
43
drop_elapsed!(0) if first_interval.elapsed?(0)
-
-
43
@next_interval_at = Utils.now
-
-
43
first_interval.interval
-
end
-
-
1
def fire(error = nil)
-
79
raise error if error && error.timeout != @intervals.first
-
79
return if @intervals.empty? || !@next_interval_at
-
-
37
elapsed_time = Utils.elapsed_time(@next_interval_at)
-
-
37
drop_elapsed!(elapsed_time)
-
-
37
@next_interval_at = nil if @intervals.empty?
-
end
-
-
1
private
-
-
1
def drop_elapsed!(elapsed_time)
-
74
@intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
-
end
-
-
1
class Timer
-
# simpler helper which allows classification
-
1
attr_accessor :label
-
-
1
def initialize(interval, callback)
-
78
@interval = interval
-
78
@callback = callback
-
end
-
-
1
def cancel
-
111
@interval.delete(@callback)
-
end
-
end
-
-
1
class Interval
-
1
include Comparable
-
-
1
attr_reader :interval
-
-
1
def initialize(interval)
-
36
@interval = interval
-
36
@callbacks = []
-
end
-
-
1
def <=>(other)
-
@interval <=> other.interval
-
end
-
-
1
def ==(other)
-
return @interval == other if other.is_a?(Numeric)
-
-
@interval == other.to_f # rubocop:disable Lint/FloatComparison
-
end
-
-
1
def to_f
-
Float(@interval)
-
end
-
-
1
def <<(callback)
-
78
@callbacks << callback
-
end
-
-
1
def delete(callback)
-
111
@callbacks.delete(callback)
-
end
-
-
1
def no_callbacks?
-
@callbacks.empty?
-
end
-
-
1
def elapsed?(elapsed = 0)
-
43
(@interval - elapsed) <= 0 || @callbacks.empty?
-
end
-
-
1
def elapse(elapsed)
-
# same as elapsing
-
37
return 0 if @callbacks.empty?
-
-
1
@interval -= elapsed
-
-
1
if @interval <= 0
-
cb = @callbacks.dup
-
cb.each(&:call)
-
end
-
-
1
@interval
-
end
-
end
-
1
private_constant :Interval
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Transcoder
-
1
module_function
-
-
1
def normalize_keys(key, value, transcoder = self, &block)
-
9
if value.respond_to?(:to_ary)
-
1
if value.empty?
-
block.call("#{key}[]")
-
else
-
1
value.to_ary.each do |element|
-
2
transcoder.normalize_keys("#{key}[]", element, transcoder, &block)
-
end
-
end
-
8
elsif value.respond_to?(:to_hash)
-
value.to_hash.each do |child_key, child_value|
-
transcoder.normalize_keys("#{key}[#{child_key}]", child_value, transcoder, &block)
-
end
-
else
-
8
block.call(key.to_s, value)
-
end
-
end
-
-
# based on https://github.com/rack/rack/blob/d15dd728440710cfc35ed155d66a98dc2c07ae42/lib/rack/query_parser.rb#L82
-
1
def normalize_query(params, name, v, depth)
-
raise Error, "params depth surpasses what's supported" if depth <= 0
-
-
name =~ /\A[\[\]]*([^\[\]]+)\]*/
-
k = Regexp.last_match(1) || ""
-
after = Regexp.last_match ? Regexp.last_match.post_match : ""
-
-
if k.empty?
-
return Array(v) if !v.empty? && name == "[]"
-
-
return
-
end
-
-
case after
-
when ""
-
params[k] = v
-
when "["
-
params[name] = v
-
when "[]"
-
params[k] ||= []
-
raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
-
-
params[k] << v
-
when /^\[\]\[([^\[\]]+)\]$/, /^\[\](.+)$/
-
child_key = Regexp.last_match(1)
-
params[k] ||= []
-
raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
-
-
if params[k].last.is_a?(Hash) && !params_hash_has_key?(params[k].last, child_key)
-
normalize_query(params[k].last, child_key, v, depth - 1)
-
else
-
params[k] << normalize_query({}, child_key, v, depth - 1)
-
end
-
else
-
params[k] ||= {}
-
raise Error, "expected Hash (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Hash)
-
-
params[k] = normalize_query(params[k], after, v, depth - 1)
-
end
-
-
params
-
end
-
-
1
def params_hash_has_key?(hash, key)
-
return false if key.include?("[]")
-
-
key.split(/[\[\]]+/).inject(hash) do |h, part|
-
next h if part == ""
-
return false unless h.is_a?(Hash) && h.key?(part)
-
-
h[part]
-
end
-
-
true
-
end
-
end
-
end
-
-
1
require "httpx/transcoder/body"
-
1
require "httpx/transcoder/form"
-
1
require "httpx/transcoder/json"
-
1
require "httpx/transcoder/chunker"
-
1
require "httpx/transcoder/deflate"
-
1
require "httpx/transcoder/gzip"
-
# frozen_string_literal: true
-
-
1
require "delegate"
-
-
1
module HTTPX::Transcoder
-
1
module Body
-
1
class Error < HTTPX::Error; end
-
-
1
module_function
-
-
1
class Encoder < SimpleDelegator
-
1
def initialize(body)
-
3
body = body.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && body.is_a?(Pathname)
-
3
@body = body
-
3
super
-
end
-
-
1
def bytesize
-
8
if @body.respond_to?(:bytesize)
-
8
@body.bytesize
-
elsif @body.respond_to?(:to_ary)
-
@body.sum(&:bytesize)
-
elsif @body.respond_to?(:size)
-
@body.size || Float::INFINITY
-
elsif @body.respond_to?(:length)
-
@body.length || Float::INFINITY
-
elsif @body.respond_to?(:each)
-
Float::INFINITY
-
else
-
raise Error, "cannot determine size of body: #{@body.inspect}"
-
end
-
end
-
-
1
def content_type
-
2
"application/octet-stream"
-
end
-
end
-
-
1
def encode(body)
-
3
Encoder.new(body)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "forwardable"
-
-
1
module HTTPX::Transcoder
-
1
module Chunker
-
1
class Error < HTTPX::Error; end
-
-
1
CRLF = "\r\n".b
-
-
1
class Encoder
-
1
extend Forwardable
-
-
1
def initialize(body)
-
@raw = body
-
end
-
-
1
def each
-
return enum_for(__method__) unless block_given?
-
-
@raw.each do |chunk|
-
yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
-
end
-
yield "0#{CRLF}"
-
end
-
-
1
def respond_to_missing?(meth, *args)
-
@raw.respond_to?(meth, *args) || super
-
end
-
end
-
-
1
class Decoder
-
1
extend Forwardable
-
-
1
def_delegator :@buffer, :empty?
-
-
1
def_delegator :@buffer, :<<
-
-
1
def_delegator :@buffer, :clear
-
-
1
def initialize(buffer, trailers = false)
-
@buffer = buffer
-
@chunk_buffer = "".b
-
@finished = false
-
@state = :length
-
@trailers = trailers
-
end
-
-
1
def to_s
-
@buffer
-
end
-
-
1
def each
-
loop do
-
case @state
-
when :length
-
index = @buffer.index(CRLF)
-
return unless index && index.positive?
-
-
# Read hex-length
-
hexlen = @buffer.byteslice(0, index)
-
@buffer = @buffer.byteslice(index..-1) || "".b
-
hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
-
@chunk_length = hexlen.hex
-
# check if is last chunk
-
@finished = @chunk_length.zero?
-
nextstate(:crlf)
-
when :crlf
-
crlf_size = @finished && !@trailers ? 4 : 2
-
# consume CRLF
-
return if @buffer.bytesize < crlf_size
-
raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
-
-
@buffer = @buffer.byteslice(crlf_size..-1)
-
if @chunk_length.nil?
-
nextstate(:length)
-
else
-
return if @finished
-
-
nextstate(:data)
-
end
-
when :data
-
chunk = @buffer.byteslice(0, @chunk_length)
-
@buffer = @buffer.byteslice(@chunk_length..-1) || "".b
-
@chunk_buffer << chunk
-
@chunk_length -= chunk.bytesize
-
if @chunk_length.zero?
-
yield @chunk_buffer unless @chunk_buffer.empty?
-
@chunk_buffer.clear
-
@chunk_length = nil
-
nextstate(:crlf)
-
end
-
end
-
break if @buffer.empty?
-
end
-
end
-
-
1
def finished?
-
@finished
-
end
-
-
1
private
-
-
1
def nextstate(state)
-
@state = state
-
end
-
end
-
-
1
module_function
-
-
1
def encode(chunks)
-
Encoder.new(chunks)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "zlib"
-
1
require_relative "utils/deflater"
-
-
1
module HTTPX
-
1
module Transcoder
-
1
module Deflate
-
1
class Deflater < Transcoder::Deflater
-
1
def deflate(chunk)
-
@deflater ||= Zlib::Deflate.new
-
-
unless chunk.nil?
-
chunk = @deflater.deflate(chunk)
-
-
# deflate call may return nil, while still
-
# retaining the last chunk in the deflater.
-
return chunk unless chunk.empty?
-
end
-
-
return if @deflater.closed?
-
-
last = @deflater.finish
-
@deflater.close
-
-
last unless last.empty?
-
end
-
end
-
-
1
module_function
-
-
1
def encode(body)
-
Deflater.new(body)
-
end
-
-
1
def decode(response, bytesize: nil)
-
bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
-
GZIP::Inflater.new(bytesize)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "forwardable"
-
1
require "uri"
-
1
require_relative "multipart"
-
-
1
module HTTPX
-
1
module Transcoder
-
1
module Form
-
1
module_function
-
-
1
PARAM_DEPTH_LIMIT = 32
-
-
1
class Encoder
-
1
extend Forwardable
-
-
1
def_delegator :@raw, :to_s
-
-
1
def_delegator :@raw, :to_str
-
-
1
def_delegator :@raw, :bytesize
-
-
1
def_delegator :@raw, :==
-
-
1
def initialize(form)
-
7
@raw = form.each_with_object("".b) do |(key, val), buf|
-
7
HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
-
8
buf << "&" unless buf.empty?
-
8
buf << URI.encode_www_form_component(k)
-
8
buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
-
end
-
end
-
end
-
-
1
def content_type
-
3
"application/x-www-form-urlencoded"
-
end
-
end
-
-
1
module Decoder
-
1
module_function
-
-
1
def call(response, *)
-
URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
-
HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
-
end
-
end
-
end
-
-
1
def encode(form)
-
7
Encoder.new(form)
-
end
-
-
1
def decode(response)
-
content_type = response.content_type.mime_type
-
-
case content_type
-
when "application/x-www-form-urlencoded"
-
Decoder
-
when "multipart/form-data"
-
Multipart::Decoder.new(response)
-
else
-
raise Error, "invalid form mime type (#{content_type})"
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "zlib"
-
-
1
module HTTPX
-
1
module Transcoder
-
1
module GZIP
-
1
class Deflater < Transcoder::Deflater
-
1
def initialize(body)
-
@compressed_chunk = "".b
-
@deflater = nil
-
super
-
end
-
-
1
def deflate(chunk)
-
@deflater ||= Zlib::GzipWriter.new(self)
-
-
if chunk.nil?
-
unless @deflater.closed?
-
@deflater.flush
-
@deflater.close
-
compressed_chunk
-
end
-
else
-
@deflater.write(chunk)
-
compressed_chunk
-
end
-
end
-
-
1
private
-
-
1
def write(*chunks)
-
chunks.sum do |chunk|
-
chunk = chunk.to_s
-
@compressed_chunk << chunk
-
chunk.bytesize
-
end
-
end
-
-
1
def compressed_chunk
-
@compressed_chunk.dup
-
ensure
-
@compressed_chunk.clear
-
end
-
end
-
-
1
class Inflater
-
1
def initialize(bytesize)
-
1
@inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
-
1
@bytesize = bytesize
-
end
-
-
1
def call(chunk)
-
buffer = @inflater.inflate(chunk)
-
@bytesize -= chunk.bytesize
-
if @bytesize <= 0
-
buffer << @inflater.finish
-
@inflater.close
-
end
-
buffer
-
end
-
end
-
-
1
module_function
-
-
1
def encode(body)
-
Deflater.new(body)
-
end
-
-
1
def decode(response, bytesize: nil)
-
1
bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
-
1
Inflater.new(bytesize)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "forwardable"
-
-
1
module HTTPX::Transcoder
-
1
module JSON
-
1
module_function
-
-
1
JSON_REGEX = %r{
-
\b
-
application/
-
# optional vendor specific type
-
(?:
-
# token as per https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6
-
[!#$%&'*+\-.^_`|~0-9a-z]+
-
# literal plus sign
-
\+
-
)?
-
json
-
\b
-
}ix.freeze
-
-
1
class Encoder
-
1
extend Forwardable
-
-
1
def_delegator :@raw, :to_s
-
-
1
def_delegator :@raw, :bytesize
-
-
1
def_delegator :@raw, :==
-
-
1
def initialize(json)
-
@raw = JSON.json_dump(json)
-
@charset = @raw.encoding.name.downcase
-
end
-
-
1
def content_type
-
"application/json; charset=#{@charset}"
-
end
-
end
-
-
1
def encode(json)
-
Encoder.new(json)
-
end
-
-
1
def decode(response)
-
content_type = response.content_type.mime_type
-
-
raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
-
-
method(:json_load)
-
end
-
-
# rubocop:disable Style/SingleLineMethods
-
1
if defined?(MultiJson)
-
def json_load(*args); MultiJson.load(*args); end
-
def json_dump(*args); MultiJson.dump(*args); end
-
1
elsif defined?(Oj)
-
def json_load(response, *args); Oj.load(response.to_s, *args); end
-
def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
-
1
elsif defined?(Yajl)
-
def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
-
def json_dump(*args); Yajl::Encoder.encode(*args); end
-
else
-
1
require "json"
-
1
def json_load(*args); ::JSON.parse(*args); end
-
1
def json_dump(*args); ::JSON.generate(*args); end
-
end
-
# rubocop:enable Style/SingleLineMethods
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require_relative "multipart/encoder"
-
1
require_relative "multipart/decoder"
-
1
require_relative "multipart/part"
-
1
require_relative "multipart/mime_type_detector"
-
-
1
module HTTPX::Transcoder
-
1
module Multipart
-
1
module_function
-
-
1
def multipart?(form_data)
-
6
form_data.any? do |_, v|
-
6
multipart_value?(v) ||
-
3
(v.respond_to?(:to_ary) && v.to_ary.any? { |av| multipart_value?(av) }) ||
-
3
(v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| multipart_value?(e) })
-
end
-
end
-
-
1
def multipart_value?(value)
-
12
value.respond_to?(:read) ||
-
3
(value.is_a?(Hash) &&
-
value.key?(:body) &&
-
(value.key?(:filename) || value.key?(:content_type)))
-
end
-
-
1
def normalize_keys(key, value, transcoder = self, &block)
-
6
if multipart_value?(value)
-
6
block.call(key.to_s, value)
-
else
-
HTTPX::Transcoder.normalize_keys(key, value, transcoder, &block)
-
end
-
end
-
-
1
def encode(form_data)
-
3
Encoder.new(form_data)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "tempfile"
-
1
require "delegate"
-
-
1
module HTTPX
-
1
module Transcoder
-
1
module Multipart
-
1
class FilePart < SimpleDelegator
-
1
attr_reader :original_filename, :content_type
-
-
1
def initialize(filename, content_type)
-
@original_filename = filename
-
@content_type = content_type
-
@file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
-
super(@file)
-
end
-
end
-
-
1
class Decoder
-
1
include HTTPX::Utils
-
-
1
CRLF = "\r\n"
-
1
BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
-
1
MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
-
1
MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
-
1
MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
-
1
WINDOW_SIZE = 2 << 14
-
-
1
def initialize(response)
-
@boundary = begin
-
m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
-
raise Error, "no boundary declared in content-type header" unless m
-
-
m.strip
-
end
-
@buffer = "".b
-
@parts = {}
-
@intermediate_boundary = "--#{@boundary}"
-
@state = :idle
-
@current = nil
-
end
-
-
1
def call(response, *)
-
response.body.each do |chunk|
-
@buffer << chunk
-
-
parse
-
end
-
-
raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
-
-
@parts
-
end
-
-
1
private
-
-
1
def parse
-
case @state
-
when :idle
-
raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
-
-
@buffer = @buffer.byteslice((@intermediate_boundary.bytesize + 2)..-1)
-
-
@state = :part_header
-
when :part_header
-
idx = @buffer.index("#{CRLF}#{CRLF}")
-
-
# raise Error, "couldn't parse part headers" unless idx
-
return unless idx
-
-
# @type var head: String
-
head = @buffer.byteslice(0..(idx + 4 - 1))
-
-
@buffer = @buffer.byteslice(head.bytesize..-1)
-
-
content_type = head[MULTIPART_CONTENT_TYPE, 1] || "text/plain"
-
if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
-
name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
-
name.gsub!(/\\(.)/, "\\1")
-
name
-
else
-
name = head[MULTIPART_CONTENT_ID, 1]
-
end
-
-
filename = HTTPX::Utils.get_filename(head)
-
-
name = filename || +"#{content_type}[]" if name.nil? || name.empty?
-
-
@current = name
-
-
@parts[name] = if filename
-
FilePart.new(filename, content_type)
-
else
-
"".b
-
end
-
-
@state = :part_body
-
when :part_body
-
part = @parts[@current]
-
-
body_separator = if part.is_a?(FilePart)
-
"#{CRLF}#{CRLF}"
-
else
-
CRLF
-
end
-
idx = @buffer.index(body_separator)
-
-
if idx
-
payload = @buffer.byteslice(0..(idx - 1))
-
@buffer = @buffer.byteslice((idx + body_separator.bytesize)..-1)
-
part << payload
-
part.rewind if part.respond_to?(:rewind)
-
@state = :parse_boundary
-
else
-
part << @buffer
-
@buffer.clear
-
end
-
when :parse_boundary
-
raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
-
-
@buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
-
-
if @buffer == "--"
-
@buffer.clear
-
@state = :done
-
return
-
elsif @buffer.start_with?(CRLF)
-
@buffer = @buffer.byteslice(2..-1)
-
@state = :part_header
-
else
-
return
-
end
-
when :done
-
raise Error, "parsing should have been over by now"
-
end until @buffer.empty?
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Transcoder::Multipart
-
1
class Encoder
-
1
attr_reader :bytesize
-
-
1
def initialize(form)
-
3
@boundary = ("-" * 21) << SecureRandom.hex(21)
-
3
@part_index = 0
-
3
@buffer = "".b
-
-
3
@form = form
-
3
@bytesize = 0
-
3
@parts = to_parts(form)
-
end
-
-
1
def content_type
-
3
"multipart/form-data; boundary=#{@boundary}"
-
end
-
-
1
def to_s
-
3
read || ""
-
ensure
-
3
rewind
-
end
-
-
1
def read(length = nil, outbuf = nil)
-
7
data = String(outbuf).clear.force_encoding(Encoding::BINARY) if outbuf
-
7
data ||= "".b
-
-
7
read_chunks(data, length)
-
-
7
data unless length && data.empty?
-
end
-
-
1
def rewind
-
3
form = @form.each_with_object([]) do |(key, val), aux|
-
3
if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
-
# @type var val: File
-
3
val = val.reopen(val.path, File::RDONLY)
-
end
-
3
val.rewind if val.respond_to?(:rewind)
-
3
aux << [key, val]
-
end
-
3
@form = form
-
3
@bytesize = 0
-
3
@parts = to_parts(form)
-
3
@part_index = 0
-
end
-
-
1
private
-
-
1
def to_parts(form)
-
6
params = form.each_with_object([]) do |(key, val), aux|
-
6
Transcoder::Multipart.normalize_keys(key, val) do |k, v|
-
6
next if v.nil?
-
-
6
value, content_type, filename = Part.call(v)
-
-
6
header = header_part(k, content_type, filename)
-
6
@bytesize += header.size
-
6
aux << header
-
-
6
@bytesize += value.size
-
6
aux << value
-
-
6
delimiter = StringIO.new("\r\n")
-
6
@bytesize += delimiter.size
-
6
aux << delimiter
-
end
-
end
-
6
final_delimiter = StringIO.new("--#{@boundary}--\r\n")
-
6
@bytesize += final_delimiter.size
-
6
params << final_delimiter
-
-
6
params
-
end
-
-
1
def header_part(key, content_type, filename)
-
6
header = "--#{@boundary}\r\n".b
-
6
header << "Content-Disposition: form-data; name=#{key.inspect}".b
-
6
header << "; filename=#{filename.inspect}" if filename
-
6
header << "\r\nContent-Type: #{content_type}\r\n\r\n"
-
6
StringIO.new(header)
-
end
-
-
1
def read_chunks(buffer, length = nil)
-
7
while @part_index < @parts.size
-
33
chunk = read_from_part(length)
-
-
33
next unless chunk
-
-
17
buffer << chunk.force_encoding(Encoding::BINARY)
-
-
17
next unless length
-
-
6
length -= chunk.bytesize
-
-
6
break if length.zero?
-
end
-
end
-
-
# if there's a current part to read from, tries to read a chunk.
-
1
def read_from_part(max_length = nil)
-
33
part = @parts[@part_index]
-
-
33
chunk = part.read(max_length, @buffer)
-
-
33
return chunk if chunk && !chunk.empty?
-
-
16
part.close if part.respond_to?(:close)
-
-
16
@part_index += 1
-
-
nil
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Transcoder::Multipart
-
1
module MimeTypeDetector
-
1
module_function
-
-
1
DEFAULT_MIMETYPE = "application/octet-stream"
-
-
# inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
-
1
if defined?(FileMagic)
-
MAGIC_NUMBER = 256 * 1024
-
-
def call(file, _)
-
return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
-
-
mime = FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
-
filemagic.buffer(file.read(MAGIC_NUMBER))
-
end
-
-
file.rewind
-
-
mime
-
end
-
1
elsif defined?(Marcel)
-
def call(file, filename)
-
return nil if file.eof? # marcel returns "application/octet-stream" for empty files
-
-
Marcel::MimeType.for(file, name: filename)
-
end
-
-
1
elsif defined?(MimeMagic)
-
-
def call(file, _)
-
mime = MimeMagic.by_magic(file)
-
mime.type if mime
-
end
-
-
1
elsif system("which file", out: File::NULL)
-
1
require "open3"
-
-
1
def call(file, _)
-
6
return if file.eof? # file command returns "application/x-empty" for empty files
-
-
4
Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
-
begin
-
4
IO.copy_stream(file, stdin.binmode)
-
rescue Errno::EPIPE
-
end
-
4
file.rewind
-
4
stdin.close
-
-
4
status = thread.value
-
-
# call to file command failed
-
4
if status.nil? || !status.success?
-
$stderr.print(stderr.read)
-
else
-
-
4
output = stdout.read.strip
-
-
4
if output.include?("cannot open")
-
$stderr.print(output)
-
else
-
4
output
-
end
-
end
-
end
-
end
-
-
else
-
-
def call(_, _); end
-
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Transcoder::Multipart
-
1
module Part
-
1
module_function
-
-
1
def call(value)
-
# take out specialized objects of the way
-
6
if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
-
return value, value.content_type, value.filename
-
end
-
-
6
content_type = filename = nil
-
-
6
if value.is_a?(Hash)
-
content_type = value[:content_type]
-
filename = value[:filename]
-
value = value[:body]
-
end
-
-
6
value = value.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
-
-
6
if value.respond_to?(:path) && value.respond_to?(:read)
-
# either a File, a Tempfile, or something else which has to quack like a file
-
6
filename ||= File.basename(value.path)
-
6
content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
-
6
[value, content_type, filename]
-
else
-
[StringIO.new(value.to_s), content_type || "text/plain", filename]
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "stringio"
-
-
1
module HTTPX
-
1
module Transcoder
-
1
class BodyReader
-
1
def initialize(body)
-
@body = if body.respond_to?(:read)
-
body.rewind if body.respond_to?(:rewind)
-
body
-
elsif body.respond_to?(:each)
-
body.enum_for(:each)
-
else
-
StringIO.new(body.to_s)
-
end
-
end
-
-
1
def bytesize
-
return @body.bytesize if @body.respond_to?(:bytesize)
-
-
Float::INFINITY
-
end
-
-
1
def read(length = nil, outbuf = nil)
-
return @body.read(length, outbuf) if @body.respond_to?(:read)
-
-
begin
-
chunk = @body.next
-
if outbuf
-
outbuf.replace(chunk)
-
else
-
outbuf = chunk
-
end
-
outbuf unless length && outbuf.empty?
-
rescue StopIteration
-
end
-
end
-
-
1
def close
-
@body.close if @body.respond_to?(:close)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require_relative "body_reader"
-
-
1
module HTTPX
-
1
module Transcoder
-
1
class Deflater
-
1
attr_reader :content_type
-
-
1
def initialize(body)
-
@content_type = body.content_type
-
@body = BodyReader.new(body)
-
@closed = false
-
end
-
-
1
def bytesize
-
buffer_deflate!
-
-
@buffer.size
-
end
-
-
1
def read(length = nil, outbuf = nil)
-
return @buffer.read(length, outbuf) if @buffer
-
-
return if @closed
-
-
chunk = @body.read(length)
-
-
compressed_chunk = deflate(chunk)
-
-
return unless compressed_chunk
-
-
if outbuf
-
outbuf.replace(compressed_chunk)
-
else
-
compressed_chunk
-
end
-
end
-
-
1
def close
-
return if @closed
-
-
@buffer.close if @buffer
-
-
@body.close
-
-
@closed = true
-
end
-
-
1
def rewind
-
return unless @buffer
-
-
@buffer.rewind
-
end
-
-
1
private
-
-
# rubocop:disable Naming/MemoizedInstanceVariableName
-
1
def buffer_deflate!
-
return @buffer if defined?(@buffer)
-
-
buffer = Response::Buffer.new(
-
threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
-
)
-
IO.copy_stream(self, buffer)
-
-
buffer.rewind if buffer.respond_to?(:rewind)
-
-
@buffer = buffer
-
end
-
# rubocop:enable Naming/MemoizedInstanceVariableName
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module HTTPX
-
1
module Utils
-
1
using URIExtensions
-
-
1
TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
-
1
VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
-
1
FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
-
1
FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
-
-
1
module_function
-
-
1
def now
-
153
Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
end
-
-
1
def elapsed_time(monotonic_timestamp)
-
37
Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
-
end
-
-
# The value of this field can be either an HTTP-date or a number of
-
# seconds to delay after the response is received.
-
1
def parse_retry_after(retry_after)
-
# first: bet on it being an integer
-
Integer(retry_after)
-
rescue ArgumentError
-
# Then it's a datetime
-
time = Time.httpdate(retry_after)
-
time - Time.now
-
end
-
-
1
def get_filename(header, _prefix_regex = nil)
-
filename = nil
-
case header
-
when FILENAME_REGEX
-
filename = Regexp.last_match(1)
-
filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
-
when FILENAME_EXTENSION_REGEX
-
filename = Regexp.last_match(1)
-
encoding, _, filename = filename.split("'", 3)
-
end
-
-
return unless filename
-
-
filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
-
-
filename.scrub!
-
-
filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
-
-
filename.force_encoding ::Encoding.find(encoding) if encoding
-
-
filename
-
end
-
-
1
URIParser = URI::RFC2396_Parser.new
-
-
1
def to_uri(uri)
-
138
return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
-
-
uri = URI(URIParser.escape(uri))
-
-
non_ascii_hostname = URIParser.unescape(uri.host)
-
-
non_ascii_hostname.force_encoding(Encoding::UTF_8)
-
-
idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
-
-
uri.host = idna_hostname
-
uri.non_ascii_hostname = non_ascii_hostname
-
uri
-
end
-
-
1
if defined?(Ractor) &&
-
# no ractor support for 3.0
-
RUBY_VERSION >= "3.1.0"
-
-
1
def in_ractor?
-
98
Ractor.main != Ractor.current
-
end
-
else
-
def in_ractor?
-
false
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module ResponseHelpers
-
1
private
-
-
1
if RUBY_ENGINE == "jruby" || RUBY_ENGINE == "truffleruby" || defined?(RBS::Test)
-
def can_run_ractor_tests?
-
false
-
end
-
else
-
1
def can_run_ractor_tests?
-
defined?(Ractor) && Ractor.method_defined?(:value)
-
end
-
end
-
-
1
def verify_status(response, expect)
-
27
raise response.error if response.is_a?(HTTPX::ErrorResponse)
-
-
27
assert response.status == expect, "status assertion failed: #{response.status} (expected: #{expect})"
-
end
-
-
1
%w[header param].each do |meth|
-
2
class_eval <<-DEFINE, __FILE__, __LINE__ + 1
-
def verify_#{meth}(#{meth}s, key, expect) # def verify_header(headers, key, expect)
-
assert #{meth}s.key?(key), "#{meth}s don't contain the given key ('\#{key}', headers: \#{#{meth}s})" # assert headers.key?(key), "headers ...
-
value = #{meth}s[key] # value = headers[key]
-
if value.respond_to?(:start_with?) # if value.respond_to?(:start_with?)
-
assert value.start_with?(expect), "#{meth} assertion failed: \#{key}=\#{value} (expected: \#{expect}})" # assert value.start_with?(expect), "headers assertion failed: ...
-
else # else
-
assert value == expect, "#{meth} assertion failed: \#{key}=\#{value.inspect} (expected: \#{expect.to_s})" # assert value == expect, "headers assertion failed: ...
-
end # end
-
end # end
-
-
def verify_no_#{meth}(#{meth}s, key) # def verify_no_header(headers, key)
-
assert !#{meth}s.key?(key), "#{meth}s contains the given key (" + key + ": \#{#{meth}s[key].inspect})" # assert !headers.key?(key), "headers contains ...
-
end # end
-
DEFINE
-
end
-
-
1
def verify_body_length(response, expect = response.headers["content-length"].to_i)
-
3
len = response.body.to_s.bytesize
-
3
assert len == expect, "length assertion failed: #{len} (expected: #{expect})"
-
end
-
-
1
def verify_execution_delta(expected, actual, delta = 0)
-
delta += 3 # because of jitter
-
-
delta += if RUBY_ENGINE == "truffleruby"
-
# truffleruby has a hard time complying reliably with this delta when running in parallel. Therefore,
-
# we give it a bit of leeway.
-
20
-
else
-
# delta checks become very innacurate under multi-thread mode, and elapsed time. we give it some leeway too.
-
3
-
end
-
-
assert_in_delta expected, actual, delta, "expected to have executed in #{expected} secs (actual: #{actual} secs)"
-
end
-
-
1
def data_base64(path)
-
"data:application/octet-stream;base64" \
-
",#{Base64.strict_encode64(File.read(path))}"
-
end
-
-
1
def verify_uploaded(body, type, expect)
-
assert body[type] == expect, "#{type} is unexpected: #{body[type]} (expected: #{expect})"
-
end
-
-
1
def verify_error_response(response, expectation = nil)
-
1
assert response.is_a?(HTTPX::ErrorResponse), "expected an error response (instead got: #{response.inspect})"
-
-
1
return unless expectation
-
-
1
case expectation
-
when Regexp
-
1
assert response.error.message =~ expectation,
-
"expected to match /#{expectation}/ in \"#{response.error.message}\""
-
when String
-
assert response.error.message.include?(expectation),
-
"expected \"#{response.error.message}\" to include \"#{expectation}\""
-
when Class
-
assert response.error.is_a?(expectation) || response.error.cause.is_a?(expectation),
-
"expected #{response.error} to be a #{expectation}"
-
else
-
raise "unexpected expectation (#{expectation})"
-
end
-
end
-
-
# test files
-
-
1
def verify_uploaded_image(body, key, mime_type, skip_verify_data: false)
-
1
assert body.key?("files"), "there were no files uploaded"
-
1
assert body["files"].key?(key), "there is no image in the file"
-
# checking mime-type is a bit leaky, as httpbin displays the base64-encoded data
-
1
return if skip_verify_data
-
-
1
assert body["files"][key].start_with?("data:#{mime_type}"), "data was wrongly encoded (#{body["files"][key][0..64]})"
-
end
-
-
1
def fixture
-
File.read(fixture_file_path, encoding: Encoding::BINARY)
-
end
-
-
1
def fixture_name
-
File.basename(fixture_file_path)
-
end
-
-
1
def fixture_file_name
-
2
"image.jpg"
-
end
-
-
1
def fixture_file_path
-
2
File.join("test", "support", "fixtures", fixture_file_name)
-
end
-
-
1
def start_test_servlet(servlet_class, *args, **kwargs)
-
server = servlet_class.new(*args, **kwargs)
-
th = Thread.new { server.start }
-
begin
-
yield server
-
ensure
-
if server.respond_to?(:shutdown)
-
server.shutdown
-
-
begin
-
Timeout.timeout(3) { th.join }
-
rescue Timeout::Error
-
th.kill
-
end
-
else
-
th.kill
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module ConnectTimeoutHelpers
-
# 9090 drops SYN packets for connect timeout tests, make sure there's a server binding there.
-
1
CONNECT_TIMEOUT_PORT_MUTEX = Thread::Mutex.new
-
1
CONNECT_TIMEOUT_PORT = ENV.fetch("CONNECT_TIMEOUT_PORT", 9090).to_i
-
-
1
def start_connect_timeout_tcp_server
-
CONNECT_TIMEOUT_PORT_MUTEX.synchronize do
-
i = 3
-
begin
-
server = TCPServer.new("127.0.0.1", CONNECT_TIMEOUT_PORT)
-
rescue Errno::EADDRINUSE
-
retry unless (i -= 1).zero?
-
-
raise
-
end
-
-
begin
-
yield "127.0.0.1:#{CONNECT_TIMEOUT_PORT}"
-
ensure
-
server.close
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module FaradayHelpers
-
1
private
-
-
# extra options to pass when building the adapter
-
1
def adapter_options
-
9
[]
-
end
-
-
1
def faraday_connection(options = {}, &optional_connection_config_blk)
-
9
return @faraday_connection if defined?(@faraday_connection)
-
-
9
builder_block = proc do |b|
-
9
b.request :url_encoded
-
9
b.adapter :httpx, *adapter_options, &optional_connection_config_blk
-
end
-
-
9
options[:ssl] ||= {}
-
9
options[:ssl][:ca_file] ||= ENV["SSL_FILE"]
-
-
9
server = options.delete(:server_uri) || URI("https://#{httpbin}")
-
-
9
@faraday_connection = Faraday::Connection.new(server.to_s, options, &builder_block).tap do |conn|
-
9
conn.headers["X-Faraday-Adapter"] = "httpx"
-
end
-
end
-
-
1
def request_headers(response)
-
6
if response.is_a?(Hash)
-
2
response[:request][:headers]
-
else
-
4
response.env.request_headers
-
end.transform_keys(&:downcase)
-
end
-
-
1
def verify_http_error_span(span, status, error)
-
2
assert span.get_tag("http.status_code") == status.to_s
-
-
2
if status >= 500 || Gem::Version.new(DatadogHelpers::DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
-
1
assert span.get_tag("error.type") == error
-
1
assert span.status == 1
-
else
-
1
assert span.status.zero?
-
end
-
end
-
-
1
def teardown
-
9
super
-
-
9
@faraday_connection.close if defined?(@faraday_connection)
-
end
-
end
-
# frozen_string_literal: true
-
-
begin
-
1
require "grpc"
-
1
require "logging"
-
-
# A test message
-
1
class EchoMsg
-
1
attr_reader :msg
-
-
1
def initialize(msg: "")
-
@msg = msg
-
end
-
-
1
def self.marshal(o)
-
o.msg
-
end
-
-
1
def self.unmarshal(msg)
-
EchoMsg.new(msg: msg)
-
end
-
end
-
-
# a test service that checks the cert of its peer
-
1
class TestService
-
1
include GRPC::GenericService
-
-
1
rpc :an_rpc, EchoMsg, EchoMsg
-
1
rpc :a_cancellable_rpc, EchoMsg, EchoMsg
-
1
rpc :a_client_streaming_rpc, stream(EchoMsg), EchoMsg
-
1
rpc :a_server_streaming_rpc, EchoMsg, stream(EchoMsg)
-
1
rpc :a_bidi_rpc, stream(EchoMsg), stream(EchoMsg)
-
-
1
def check_peer_cert(call)
-
# error_msg = "want:\n#{client_cert}\n\ngot:\n#{call.peer_cert}"
-
# fail(error_msg) unless call.peer_cert == client_cert
-
end
-
-
1
def an_rpc(req, call)
-
check_peer_cert(call)
-
req
-
end
-
-
1
def a_cancellable_rpc(_req, call)
-
check_peer_cert(call)
-
raise GRPC::Cancelled, "dump"
-
end
-
-
1
def a_client_streaming_rpc(call)
-
check_peer_cert(call)
-
call.each_remote_read.each { |r| GRPC.logger.info(r) }
-
EchoMsg.new(msg: "client stream pong")
-
end
-
-
1
def a_server_streaming_rpc(_, call)
-
check_peer_cert(call)
-
call.send_initial_metadata
-
[EchoMsg.new(msg: "server stream pong"), EchoMsg.new(msg: "server stream pong")]
-
end
-
-
1
def a_bidi_rpc(requests, call)
-
check_peer_cert(call)
-
requests.each { |r| GRPC.logger.info(r) }
-
call.send_initial_metadata
-
[EchoMsg.new(msg: "bidi pong"), EchoMsg.new(msg: "bidi pong")]
-
end
-
end
-
-
1
if ENV.key?("HTTPX_DEBUG")
-
log_level = ENV["HTTPX_DEBUG"].to_i
-
log_level = log_level > 1 ? :debug : :info
-
-
module GRPC
-
extend Logging.globally
-
end
-
Logging.logger.root.appenders = Logging.appenders.stdout
-
Logging.logger.root.level = log_level
-
Logging.logger["GRPC"].level = log_level
-
Logging.logger["GRPC::ActiveCall"].level = log_level
-
Logging.logger["GRPC::BidiCall"].level = log_level
-
end
-
-
1
module GRPCHelpers
-
1
include ::GRPC::Core::StatusCodes
-
1
include ::GRPC::Core::TimeConsts
-
1
include ::GRPC::Core::CallOps
-
-
1
private
-
-
1
def teardown
-
super
-
if @grpc_server
-
-
@grpc_server.shutdown_and_notify(from_relative_time(2))
-
@grpc_server.close
-
@grpc_server_th.join if @grpc_server_th
-
end
-
-
return unless @rpc_server && !@rpc_server.stopped?
-
-
@rpc_server.stop
-
@rpc_server_th.join
-
end
-
-
1
def grpc_plugin
-
grpc = HTTPX.plugin(:grpc)
-
-
grpc = grpc.with_channel_credentials(*channel_credentials_paths, hostname: "foo.test.google.fr") if origin.start_with?("https")
-
-
grpc
-
end
-
-
1
def run_rpc(service, server_args: {})
-
@rpc_server = ::GRPC::RpcServer.new(server_args: server_args.merge("grpc.so_reuseport" => 0))
-
-
cred = origin.start_with?("https") ? server_credentials : :this_port_is_insecure
-
-
server_port = @rpc_server.add_http2_port("localhost:0", cred)
-
@rpc_server.handle(service)
-
-
@rpc_server_th = Thread.new { @rpc_server.run }
-
@rpc_server.wait_till_running
-
-
server_port
-
end
-
-
1
def run_request_response(resp, status, marshal: nil, server_args: {}, server_initial_md: {}, server_trailing_md: {})
-
@grpc_server = ::GRPC::Core::Server.new(server_args.merge("grpc.so_reuseport" => 0))
-
-
cred = origin.start_with?("https") ? server_credentials : :this_port_is_insecure
-
-
server_port = @grpc_server.add_http2_port("localhost:0", cred)
-
-
@grpc_server_th = wakey_thread do |notifier|
-
c = expect_server_to_be_invoked(notifier, metadata_to_send: server_initial_md, marshal: marshal)
-
begin
-
yield c
-
ensure
-
c.remote_send(resp)
-
c.send_status(status, status == OK ? "OK" : "NOK", true, metadata: server_trailing_md)
-
c.send(:set_input_stream_done)
-
c.send(:set_output_stream_done)
-
end
-
end
-
-
server_port
-
end
-
-
1
def wakey_thread(&blk)
-
n = ::GRPC::Notifier.new
-
t = Thread.new do
-
begin
-
blk.call(n)
-
rescue GRPC::Core::CallError
-
end
-
end
-
t.abort_on_exception = true
-
n.wait
-
t
-
end
-
-
1
def expect_server_to_be_invoked(notifier, metadata_to_send: nil, marshal: nil)
-
@grpc_server.start
-
notifier.notify(nil)
-
recvd_rpc = @grpc_server.request_call
-
recvd_call = recvd_rpc.call
-
recvd_call.metadata = recvd_rpc.metadata
-
recvd_call.run_batch(SEND_INITIAL_METADATA => metadata_to_send)
-
::GRPC::ActiveCall.new(recvd_call, marshal, marshal, INFINITE_FUTURE, metadata_received: true)
-
end
-
-
1
def server_credentials
-
creds = ["ca.pem", "server1.key", "server1.pem"]
-
.map { |path| File.join(grpc_testdata_path, path) }
-
.map(&File.method(:read))
-
-
GRPC::Core::ServerCredentials.new(
-
creds[0],
-
[{ private_key: creds[1], cert_chain: creds[2] }],
-
true
-
) # force client auth
-
end
-
-
1
def channel_credentials_paths
-
["ca.pem", "client.key", "client.pem"]
-
.map { |path| File.join(grpc_testdata_path, path) }
-
# .map(&File.method(:read))
-
# GRPC::Core::ChannelCredentials.new(*creds)
-
end
-
-
1
def grpc_testdata_path
-
grpc_path = Gem::Specification.find_by_path("grpc").full_gem_path
-
File.join(grpc_path, "src", "ruby", "spec", "testdata")
-
end
-
end
-
rescue LoadError
-
module GRPCHelpers
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require_relative "assertion_helpers"
-
-
1
module HTTPHelpers
-
1
include ResponseHelpers
-
-
1
private
-
-
1
def build_uri(suffix = "/", uri_origin = origin)
-
30
"#{uri_origin}#{suffix || "/"}"
-
end
-
-
1
def json_body(response)
-
9
raise response.error if response.is_a?(HTTPX::ErrorResponse)
-
-
9
JSON.parse(response.body.to_s)
-
end
-
-
1
def httpbin
-
41
ENV.fetch("HTTPBIN_HOST", "nghttp2.org/httpbin")
-
end
-
-
1
def httpbin_no_proxy
-
URI(ENV.fetch("HTTPBIN_NO_PROXY_HOST", "#{scheme}httpbin.org"))
-
end
-
-
1
def origin(orig = httpbin)
-
3
"#{scheme}#{orig}"
-
end
-
-
1
def next_available_port
-
server = TCPServer.new("localhost", 0)
-
server.addr[1]
-
ensure
-
server.close
-
end
-
-
1
def tls?
-
scheme == "https://"
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module MinitestExtensions
-
1
module TimeoutForTest
-
# our own subclass so we never confused different timeouts
-
1
class TestTimeout < Timeout::Error
-
end
-
-
1
def run(*)
-
126
::Timeout.timeout(60 * 5, TestTimeout) { super }
-
end
-
end
-
-
1
module FirstFailedTestInThread
-
1
def self.prepended(*)
-
1
super
-
1
HTTPX::Connection.include ConnectionExtensions
-
end
-
-
1
def setup
-
63
super
-
63
extend(OnTheFly)
-
end
-
-
1
module ConnectionExtensions
-
1
def send(request)
-
request.instance_variable_set(:@connection, connection)
-
super
-
end
-
end
-
-
1
def run(*)
-
63
(Thread.current[:passed_tests] ||= []) << "#{self.class.name}##{name}"
-
63
super
-
ensure
-
63
if !skipped? && !Thread.current[:tests_already_failed] && !failures.empty?
-
Thread.current[:tests_already_failed] = true
-
puts "first test failed: #{Thread.current[:passed_tests].pop}\n"
-
puts "this thread also executed: #{Thread.current[:passed_tests].join(", ")}" unless Thread.current[:passed_tests].empty?
-
end
-
end
-
-
1
module OnTheFly
-
1
def verify_status(response, expect)
-
27
if response.is_a?(HTTPX::ErrorResponse) && response.error.message.include?("execution expired")
-
connection = response.request.instance_variable_get(:@connection)
-
puts connection.inspect
-
end
-
-
27
super
-
end
-
end
-
end
-
-
1
module TestName
-
1
def run(*)
-
print "#{self.class.name}##{name}: "
-
super
-
ensure
-
puts " "
-
end
-
end
-
end
-
-
1
Minitest::Test.prepend(MinitestExtensions::TimeoutForTest) unless ENV.key?("HTTPX_DEBUG")
-
1
Minitest::Test.prepend(MinitestExtensions::FirstFailedTestInThread)
-
# Minitest::Test.prepend(MinitestExtensions::TestName)
-
# frozen_string_literal: true
-
-
1
require "uri"
-
1
require "net/http"
-
1
require "oga"
-
-
1
module ProxyHelper
-
1
private
-
-
1
def socks4_proxy
-
Array(ENV["HTTPX_SOCKS4_PROXY"] || begin
-
socks_proxies_list.select { |_, _, version, https| version == "Socks4" && https }
-
.map { |ip, port, _, _| "socks4://#{ip}:#{port}" }
-
end)
-
end
-
-
1
def socks4a_proxy
-
Array(ENV["HTTPX_SOCKS4A_PROXY"] || begin
-
socks_proxies_list.select { |_, _, version, https| version == "Socks4" && https }
-
.map { |ip, port, _, _| "socks4a://#{ip}:#{port}" }
-
end)
-
end
-
-
1
def socks5_proxy
-
Array(ENV["HTTPX_SOCKS5_PROXY"] || begin
-
socks_proxies_list.select { |_, _, version, https| version == "Socks5" && https }
-
.map { |ip, port, _, _| "socks5://#{ip}:#{port}" }
-
end)
-
end
-
-
1
def http_proxy
-
Array(ENV["HTTPX_HTTP_PROXY"] || begin
-
http_proxies_list.map do |ip, port, _|
-
"http://#{ip}:#{port}"
-
end
-
end)
-
end
-
-
1
def http2_proxy
-
Array(ENV["HTTPX_HTTP2_PROXY"])
-
end
-
-
1
def https_proxy
-
Array(ENV["HTTPX_HTTPS_PROXY"] || begin
-
http_proxies_list.select { |_, _, https| https }.map do |ip, port, _|
-
"http://#{ip}:#{port}"
-
end
-
end)
-
end
-
-
1
def ssh_proxy
-
Array(ENV["HTTPX_SSH_PROXY"] || begin
-
http_proxies_list.select { |_, _, https| https }.map do |ip, port, _|
-
"ssh://#{ip}:#{port}"
-
end
-
end)
-
end
-
-
1
def http_proxies_list
-
proxies_list(parse_http_proxies)
-
.map do |line|
-
ip, port, _, _, _, _, https, _ = line.css("td").map(&:text)
-
[ip, port, https == "yes"]
-
end.select { |ip, port, _| ip && port } # rubocop:disable Style/MultilineBlockChain
-
end
-
-
1
def socks_proxies_list
-
proxies_list(parse_socks_proxies)
-
.map do |line|
-
ip, port, _, _, version, _, https, _ = line.css("td").map(&:text)
-
[ip, port, version, https == "Yes"]
-
end.select { |ip, port, _, _| ip && port } # rubocop:disable Style/MultilineBlockChain
-
end
-
-
1
def proxies_list(document)
-
row = document.enum_for(:each_node).find do |node|
-
next unless node.is_a?(Oga::XML::Element)
-
-
id = node.attribute("id")
-
next unless id
-
-
id.value == "list"
-
end
-
row ? row.css("tr") : []
-
end
-
-
1
def parse_http_proxies
-
@parse_http_proxies ||= Oga.parse_html(fetch_http_proxies)
-
end
-
-
1
def fetch_http_proxies
-
Net::HTTP.get_response(URI("https://free-proxy-list.net/ssl-proxy.html")).body
-
end
-
-
1
def parse_socks_proxies
-
@parse_socks_proxies ||= Oga.parse_html(fetch_socks_proxies)
-
end
-
-
1
def fetch_socks_proxies
-
Net::HTTP.get_response(URI("https://free-proxy-list.net/socks-proxy.html")).body
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module ProxyResponseDetector
-
1
module RequestMethods
-
1
attr_writer :proxied
-
-
1
def proxied?
-
@proxied
-
end
-
end
-
-
1
module ResponseMethods
-
1
def proxied?
-
@request.proxied?
-
end
-
end
-
-
1
module ConnectionMethods
-
1
def send(request)
-
return super unless @options.respond_to?(:proxy) && @options.proxy
-
-
request.proxied = true
-
-
super
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module ProxyRetry
-
1
def run(*)
-
63
return super unless name.include?("_proxy")
-
-
result = nil
-
3.times.each do |_i|
-
result = super
-
break if result.passed?
-
-
self.failures = []
-
self.assertions = 0
-
end
-
result
-
end
-
end
-
-
1
Minitest::Test.prepend(ProxyRetry) unless ENV.key?("HTTPX_DEBUG")
-
# frozen_string_literal: true
-
-
1
module RequestInspector
-
1
module InstanceMethods
-
1
attr_reader :calls, :total_requests, :total_responses
-
-
1
def initialize(*args)
-
super
-
# we're comparing against max-retries + 1, because the calls increment will happen
-
# also in the last call, where the request is not going to be retried.
-
@calls = -1
-
@total_requests = []
-
@total_responses = []
-
end
-
-
1
def reset
-
@calls = -1
-
@total_requests.clear
-
@total_responses.clear
-
end
-
-
1
private
-
-
1
def send_request(request, *)
-
@total_requests << request.dup
-
super
-
end
-
-
1
def fetch_response(*)
-
response = super
-
if response
-
@calls += 1
-
@total_responses << response
-
end
-
response
-
end
-
end
-
-
1
module ResponseMethods
-
1
attr_reader :request
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module AltSvc
-
1
def test_altsvc_get
-
altsvc_host = ENV["HTTPBIN_ALTSVC_HOST"]
-
altsvc_origin = origin(altsvc_host)
-
-
HTTPX.plugin(SessionWithPool).wrap do |http|
-
altsvc_uri = build_uri("/get", altsvc_origin)
-
res1, res2 = http.get(altsvc_uri, altsvc_uri)
-
verify_status(res1, 200)
-
verify_header(res1.headers, "alt-svc", "h2=\"nghttp2:443\"")
-
verify_status(res2, 200)
-
verify_header(res2.headers, "alt-svc", "h2=\"nghttp2:443\"")
-
res3 = http.get(altsvc_uri)
-
verify_status(res3, 200)
-
verify_no_header(res3.headers, "alt-svc")
-
# introspection time
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "resolv"
-
1
require "httpx/plugins/callbacks"
-
-
1
module Requests
-
1
using HTTPX::URIExtensions
-
1
module Callbacks
-
1
def test_callbacks_connection_opened
-
uri = URI(build_uri("/get"))
-
origin = ip = nil
-
opened = 0
-
-
response = HTTPX.plugin(SessionWithPool).plugin(:callbacks).on_connection_opened do |o, sock|
-
origin = o
-
ip = sock.to_io.remote_address.ip_address
-
opened += 1
-
end.get(uri)
-
verify_status(response, 200)
-
-
assert !origin.nil?
-
assert origin.to_s == uri.origin
-
assert !ip.nil?
-
assert opened == 1
-
-
assert Resolv.getaddresses(uri.host).include?(ip)
-
end
-
-
1
def test_callbacks_connection_closed
-
uri = URI(build_uri("/get"))
-
origin = nil
-
closed = 0
-
-
response = HTTPX.plugin(SessionWithPool).plugin(:callbacks).on_connection_closed do |o|
-
origin = o
-
closed += 1
-
end.get(uri)
-
verify_status(response, 200)
-
-
assert !origin.nil?
-
assert origin.to_s == uri.origin
-
assert closed == 1
-
end
-
-
1
def test_callbacks_request_error
-
uri = URI(build_uri("/get"))
-
error = nil
-
-
http = HTTPX.plugin(:callbacks).on_request_error { |_, err| error = err }
-
-
response = http.get(uri)
-
verify_status(response, 200)
-
-
assert error.nil?
-
-
unavailable_host = URI(origin("localhost"))
-
unavailable_host.port = next_available_port
-
response = http.get(unavailable_host.to_s)
-
verify_error_response(response, /Connection refused| not available/)
-
-
assert !error.nil?
-
assert error == response.error
-
end
-
-
1
def test_callbacks_request_error_allow_reraise_in_bock
-
URI(build_uri("/get"))
-
err_type = Class.new(StandardError)
-
-
ex = assert_raises(err_type) do
-
http = HTTPX.plugin(:callbacks).on_request_error do |_, err|
-
raise err_type, err.message
-
end
-
# unavailable_host = URI(origin("localhost"))
-
# unavailable_host.port = next_available_port
-
# http.get(unavailable_host.to_s)
-
http.get("http://unknownhost")
-
end
-
assert ex.message == "name or service not known"
-
end
-
-
1
def test_callbacks_request
-
uri = URI(build_uri("/post"))
-
started = completed = false
-
chunks = 0
-
-
http = HTTPX.plugin(:callbacks)
-
.on_request_started { |_| started = true }
-
.on_request_body_chunk { |_, _chunk| chunks += 1 }
-
.on_request_completed { |_| completed = true }
-
-
response = http.post(uri, body: "data")
-
verify_status(response, 200)
-
-
assert started
-
assert completed
-
assert chunks.positive?
-
end
-
-
1
def test_callbacks_response
-
uri = URI(build_uri("/get"))
-
started = completed = false
-
chunks = 0
-
-
http = HTTPX.plugin(:callbacks)
-
.on_response_started { |_, _| started = true }
-
.on_response_body_chunk { |_, _, _chunk| chunks += 1 }
-
.on_response_completed { |_, _| completed = true }
-
-
response = http.get(uri)
-
verify_status(response, 200)
-
-
assert started
-
assert completed
-
assert chunks.positive?
-
end
-
-
1
def test_callbacks_keeps_callbacks_when_building_new_sessions
-
http = HTTPX.plugin(:callbacks).on_request_started { puts "test" }
-
http.singleton_class.class_eval do
-
public :callbacks_for?
-
end
-
-
assert http.callbacks_for?(:request_started)
-
http = http.with(headers: { a: 1 })
-
assert http.callbacks_for?(:request_started)
-
end
-
-
1
%i[
-
connection_opened connection_closed
-
request_started request_completed
-
response_started response_body_chunk response_completed
-
].each do |callback|
-
7
define_method :"test_callbacks_bug_inside_#{callback}_callback" do
-
assert_raises(NameError) do
-
HTTPX.plugin(SessionWithPool).plugin(:callbacks).send(:"on_#{callback}") { i_dont_exist }.get(build_uri("/get"))
-
end
-
end
-
end
-
-
1
def test_callbacks_can_compose_with
-
http = HTTPX.plugin(:callbacks).with(persistent: true)
-
assert http.instance_variable_get(:@persistent)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module ChunkedGet
-
1
def test_http_chunked_get
-
uri = build_uri("/stream-bytes/30?chunk_size=5")
-
response = HTTPX.get(uri)
-
verify_status(response, 200)
-
verify_header(response.headers, "transfer-encoding", "chunked")
-
verify_body_length(response, 30)
-
end
-
-
1
def test_http_head_chunked_to_next_request
-
start_test_servlet(KeepAliveServer) do |server|
-
chunked_uri = "#{server.origin}/chunk"
-
-
HTTPX.with(persistent: true) do |http|
-
response = http.head(chunked_uri)
-
verify_status(response, 200)
-
verify_header(response.headers, "transfer-encoding", "chunked")
-
verify_body_length(response, 0)
-
-
response = http.get(chunked_uri)
-
verify_status(response, 200)
-
verify_header(response.headers, "transfer-encoding", "chunked")
-
body = json_body(response)
-
assert body["chunked"] == true
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Coalescing
-
1
def test_connection_coalescing
-
coalesced_origin = "https://#{ENV["HTTPBIN_COALESCING_HOST"]}"
-
HTTPX.plugin(SessionWithPool).wrap do |http|
-
response1 = http.get(origin)
-
verify_status(response1, 200)
-
response2 = http.get(coalesced_origin)
-
verify_status(response2, 200)
-
# introspection time
-
connections = http.connections
-
assert connections.size == 2
-
origins = connections.map(&:origins)
-
assert origins.any? { |orgs| orgs.sort == [origin, coalesced_origin].sort },
-
"connections for #{[origin, coalesced_origin]} didn't coalesce (expected connection with both origins (#{origins}))"
-
-
assert http.pool.connections_counter == 1, "coalesced connection should not have been accounted for in the pool"
-
-
unsafe_origin = URI(origin)
-
unsafe_origin.scheme = "http"
-
response3 = http.get(unsafe_origin)
-
verify_status(response3, 200)
-
-
# introspection time
-
connections = http.connections
-
assert connections.size == 3
-
origins = connections.map(&:origins)
-
refute origins.any?([origin]),
-
"connection coalesced inexpectedly (expected connection with both origins (#{origins}))"
-
end
-
end
-
-
1
def test_coalesce_should_not_leak_across_threads
-
# https://gitlab.com/os85/httpx/-/issues/365
-
uri = URI(build_uri("/get", "https://#{httpbin}"))
-
coalesced_uri = URI(build_uri("/get", "https://#{ENV["HTTPBIN_COALESCING_HOST"]}"))
-
q = Queue.new
-
-
http = HTTPX.plugin(SessionWithPool).plugin(:persistent)
-
-
registered_conns = Set[]
-
http.define_singleton_method(:select_connection) do |conn, selector|
-
registered_conns << [conn, selector]
-
super(conn, selector)
-
end
-
-
http.singleton_class.class_eval do
-
public(:get_current_selector)
-
end
-
-
th1 = Thread.start do
-
q.pop
-
res = http.get(coalesced_uri)
-
verify_status(res, 200)
-
http.get_current_selector
-
end
-
-
th2 = Thread.start do
-
res = http.get(uri)
-
verify_status(res, 200)
-
sel = http.get_current_selector
-
q << :done
-
sel
-
end
-
-
th2_selector = th2.value
-
th1_selector = th1.value
-
-
conns = http.connections.select(&:open?)
-
assert conns.size == 1
-
conn = conns.first
-
assert conn.current_session.nil?, "connection should have reset its session already"
-
assert conn.current_selector.nil?, "connection should have reset its selector already"
-
-
assert http.pool.connections_counter == 1, "connection"
-
assert http.pool.connections.include?(conn)
-
-
assert registered_conns.size == 2
-
-
assert registered_conns.include?([conn, th1_selector])
-
assert registered_conns.include?([conn, th2_selector])
-
ensure
-
http.close if defined?(http)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Compression
-
1
def test_compression_accepts
-
url = "https://github.com"
-
-
response = HTTPX.get(url)
-
skip if response == 429
-
verify_status(response, 200)
-
assert response.body.encodings == %w[gzip], "response should be sent with gzip encoding"
-
response.close
-
end
-
-
1
def test_compression_identity_post
-
uri = build_uri("/post")
-
response = HTTPX.with_headers("content-encoding" => "identity")
-
.post(uri, body: "a" * 8012)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/octet-stream")
-
compressed_data = body["data"]
-
assert compressed_data.bytesize == 8012, "body shouldn't have been compressed"
-
end
-
-
1
def test_compression_gzip
-
uri = build_uri("/gzip")
-
response = HTTPX.get(uri)
-
verify_status(response, 200)
-
assert response.headers["content-length"].to_i != response.body.bytesize
-
body = json_body(response)
-
assert body["gzipped"], "response should be gzipped"
-
end
-
-
1
def test_compression_gzip_do_not_decompress
-
uri = build_uri("/gzip")
-
response = HTTPX.get(uri, decompress_response_body: false)
-
verify_status(response, 200)
-
assert response.headers["content-length"].to_i == response.body.bytesize
-
end
-
-
1
def test_compression_gzip_post
-
uri = build_uri("/post")
-
response = HTTPX.with_headers("content-encoding" => "gzip")
-
.post(uri, body: "a" * 8012)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/octet-stream")
-
compressed_data = body["data"]
-
compressed_data = compressed_data.delete_prefix("data:application/octet-stream;base64,")
-
compressed_data = Base64.decode64(compressed_data)
-
assert compressed_data.bytesize < 8012, "body hasn't been compressed"
-
assert inflate_test_data(compressed_data) == "a" * 8012
-
end
-
-
1
def test_compression_gzip_post_already_compressed
-
uri = build_uri("/post")
-
gzip_body = Zlib.gzip("a" * 8012)
-
-
response = HTTPX.with(
-
compress_request_body: false,
-
headers: { "content-encoding" => "gzip" }
-
).post(uri, body: gzip_body)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/octet-stream")
-
compressed_data = body["data"]
-
compressed_data = compressed_data.delete_prefix("data:application/octet-stream;base64,")
-
compressed_data = Base64.decode64(compressed_data)
-
assert compressed_data.bytesize < 8012, "body hasn't been compressed"
-
assert inflate_test_data(compressed_data) == "a" * 8012
-
end
-
-
1
def test_compression_deflate
-
uri = build_uri("/deflate")
-
response = HTTPX.get(uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
assert body["deflated"], "response should be deflated"
-
end
-
-
1
def test_compression_deflate_post
-
uri = build_uri("/post")
-
response = HTTPX.with_headers("content-encoding" => "deflate")
-
.post(uri, body: "a" * 8012)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/octet-stream")
-
compressed_data = body["data"]
-
compressed_data = compressed_data.delete_prefix("data:application/octet-stream;base64,")
-
compressed_data = Base64.decode64(compressed_data)
-
assert compressed_data.bytesize < 8012, "body hasn't been compressed"
-
assert inflate_test_data(compressed_data) == "a" * 8012
-
end
-
-
# regression test
-
1
def test_compression_no_content_length
-
# run this only for http/1.1 mode, as this is a local test server
-
return unless origin.start_with?("http://")
-
-
start_test_servlet(NoContentLengthServer) do |server|
-
uri = build_uri("/", server.origin)
-
response = HTTPX.get(uri)
-
verify_status(response, 200)
-
body = response.body.to_s
-
assert body == "helloworld"
-
end
-
end
-
-
1
def test_compression_ignore_encoding_on_range
-
uri = build_uri("/get")
-
response = HTTPX.get(uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
assert body["headers"].key?("Accept-Encoding")
-
-
response = HTTPX.get(uri, headers: { "range" => "bytes=100-200" })
-
body = json_body(response)
-
assert !body["headers"].key?("Accept-Encoding")
-
end
-
-
1
private
-
-
1
def inflate_test_data(string)
-
zstream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
-
buf = zstream.inflate(string)
-
zstream.finish
-
zstream.close
-
buf
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Errors
-
1
def test_errors_invalid_uri
-
exc = assert_raises { HTTPX.get("/get") }
-
assert exc.message.include?("invalid URI: /get")
-
exc = assert_raises { HTTPX.get("http:/smth/get") }
-
assert exc.message.include?("invalid URI: http:/smth/get")
-
end
-
-
1
def test_errors_invalid_scheme
-
assert_raises(HTTPX::UnsupportedSchemeError) { HTTPX.get("foo://example.com") }
-
end
-
-
1
def test_errors_connection_refused
-
unavailable_host = URI(origin("localhost"))
-
unavailable_host.port = next_available_port
-
response = HTTPX.get(unavailable_host.to_s)
-
verify_error_response(response, /Connection refused| not available/)
-
end
-
-
1
def test_errors_log_error
-
log = StringIO.new
-
unavailable_host = URI(origin("localhost"))
-
unavailable_host.port = next_available_port
-
response = HTTPX.plugin(SessionWithPool).get(unavailable_host.to_s, debug: log, debug_level: 3)
-
output = log.string
-
assert output.include?(response.error.message)
-
end
-
-
1
def test_errors_host_unreachable
-
uri = URI(origin("localhost")).to_s
-
return unless uri.start_with?("http://")
-
-
response = HTTPX.get(uri, addresses: [EHOSTUNREACH_HOST] * 2)
-
verify_error_response(response, /No route to host/)
-
end
-
-
# TODO: reset this test once it's possible to test ETIMEDOUT again
-
# the new iptables crapped out on me
-
# def test_errors_host_etimedout
-
# uri = URI(origin("etimedout:#{ETIMEDOUT_PORT}")).to_s
-
# return unless uri.start_with?("http://")
-
-
# server = TCPServer.new("127.0.0.1", ETIMEDOUT_PORT)
-
# begin
-
# response = HTTPX.get(uri, addresses: %w[127.0.0.1] * 2)
-
# verify_error_response(response, Errno::ETIMEDOUT)
-
# ensure
-
# server.close
-
# end
-
# end
-
-
1
SocketErrorPlugin = Module.new do
-
1
self::ResolverNativeMethods = Module.new do
-
1
define_method :call do
-
end
-
-
1
define_method :to_io do
-
raise "socket error here"
-
end
-
end
-
end
-
-
1
SocketExceptionPlugin = Module.new do
-
1
self::SocketException = Class.new(Exception) # rubocop:disable Style/EmptyClassDefinition
-
1
self::ResolverNativeMethods = Module.new do
-
1
define_method :call do
-
end
-
-
1
define_method :to_io do
-
raise SocketExceptionPlugin::SocketException, "socket exception here"
-
end
-
end
-
1
self::ResolverHTTPSMethods = Module.new do
-
1
def resolver_connection
-
super.tap do |conn|
-
def conn.to_io
-
raise SocketExceptionPlugin::SocketException, "socket exception here"
-
end
-
end
-
end
-
end
-
1
self::ResolverSystemMethods = Module.new do
-
1
def __addrinfo_resolve(*)
-
sleep(0.1)
-
super
-
end
-
-
1
define_method :to_io do
-
raise SocketExceptionPlugin::SocketException, "socket exception here"
-
end
-
end
-
end
-
-
1
def test_errors_native_resolver_error_mid_dns_query_io_wait
-
uri = URI(build_uri("/get"))
-
HTTPX
-
.plugin(SessionWithPool)
-
.plugin(SocketErrorPlugin)
-
.with(resolver_class: :native, resolver_options: { cache: false }) do |http|
-
response = http.get(uri)
-
verify_error_response(response, /socket error here/)
-
-
pool = http.pool
-
assert pool.connections_counter.nonzero?
-
assert pool.connections_counter == pool.connections.size
-
assert(pool.connections.all? { |conn| conn.state == :closed })
-
-
assert http.resolvers.size == 1
-
resolver = http.resolvers.first
-
resolver = resolver.resolvers.first # because it's a multi
-
assert resolver.state == :closed
-
assert resolver.connections.empty?
-
end
-
end
-
-
{
-
1
single: [Socket::AF_INET],
-
multihomed: [Socket::AF_INET6, Socket::AF_INET],
-
}.each do |type, ip_families|
-
2
%i[native system https].each do |resolver_class|
-
6
define_method :"test_errors_#{type}_#{resolver_class}_resolver_exception_mid_dns_query_io_wait" do
-
uri = URI(build_uri("/get"))
-
HTTPX
-
.plugin(SessionWithPool)
-
.plugin(SocketExceptionPlugin)
-
.with(resolver_class: resolver_class, resolver_options: { cache: false }, ip_families: ip_families) do |http|
-
assert_raises(SocketExceptionPlugin::SocketException) do
-
http.get(uri)
-
end
-
-
# some state is going to be corrupted in the face of an Exception,
-
# the only thing we care about is whether all used sockets are closed.
-
-
connections = http.connections
-
assert connections.size >= 1
-
assert(connections.all? { |conn| conn.state == :closed })
-
-
# https resolver will also need to resolve its resolver connection
-
assert http.resolvers.size == (resolver_class == :https ? 2 : 1)
-
resolver = http.resolvers.first
-
resolver = resolver.resolvers.first # because it's a multi
-
assert resolver.state == :closed
-
end
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "time"
-
-
1
module Requests
-
1
using HTTPX::URIExtensions
-
-
1
module Get
-
1
def test_http_get
-
uri = build_uri("/get")
-
response = HTTPX.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
-
return unless can_run_ractor_tests?
-
-
response2 = Ractor.new(uri) do |uri|
-
HTTPX.get(uri)
-
end.value
-
-
verify_status(response2, 200)
-
verify_body_length(response2)
-
end
-
-
1
def test_http_get_option_origin
-
uri = URI(build_uri("/get"))
-
response = HTTPX.with(origin: uri.origin).get(uri.path)
-
verify_status(response, 200)
-
verify_body_length(response)
-
end
-
-
1
def test_http_get_option_origin_base_path
-
status_uri = URI(build_uri("/status"))
-
http = HTTPX.with(origin: status_uri.origin, base_path: status_uri.request_uri)
-
response = http.get("/200")
-
verify_status(response, 200)
-
assert response.uri.request_uri == "#{status_uri.request_uri}/200"
-
end
-
-
1
def test_http_get_request
-
uri = build_uri("/get")
-
response = HTTPX.request("GET", uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
end
-
-
1
def test_http_get_build_request
-
uri = build_uri("/get")
-
HTTPX.wrap do |http|
-
request = http.build_request("GET", uri)
-
response = http.request(request)
-
verify_status(response, 200)
-
verify_body_length(response)
-
end
-
end
-
-
1
def test_get_multiple_same_origin
-
uri = build_uri("/delay/2")
-
-
session = HTTPX.plugin(SessionWithPool)
-
-
response1, response2 = session.get(uri, uri)
-
-
verify_status(response1, 200)
-
verify_body_length(response1)
-
-
verify_status(response2, 200)
-
verify_body_length(response2)
-
-
assert session.resolvers.size == 1
-
end
-
-
1
def test_get_multiple_different_origin
-
session = HTTPX.plugin(SessionWithPool)
-
-
req1 = ["/delay/2", { origin: origin(httpbin) }]
-
req2 = ["/delay/2", { origin: httpbin_no_proxy }]
-
-
response1, response2 = session.get(req1, req2)
-
-
verify_status(response1, 200)
-
verify_body_length(response1)
-
-
verify_status(response2, 200)
-
verify_body_length(response2)
-
-
num_resolvers = session.resolvers.size
-
assert num_resolvers == 1, "should have only had 1 resolver, was #{num_resolvers}"
-
end
-
-
1
def test_get_multiple_no_concurrency
-
uri = build_uri("/delay/2")
-
response1, response2 = HTTPX.plugin(:persistent).get(uri, uri, max_concurrent_requests: 1)
-
-
verify_status(response1, 200)
-
verify_body_length(response1)
-
-
verify_status(response2, 200)
-
verify_body_length(response2)
-
-
assert response1.to_s == response2.to_s, "request should have been the same"
-
-
date1 = Time.parse(response1.headers["date"])
-
date2 = Time.parse(response2.headers["date"])
-
-
# I test for greater than 2 due to the concurrent test, which affect the times.
-
# However, most important is, it takes certainly more than 2 seconds.
-
time_it_took = (date2 - date1).abs
-
assert time_it_took >= 2, "time between requests took < 2 secs (actual: #{time_it_took} secs)"
-
end
-
-
1
def test_get_http_accept
-
uri = build_uri("/get")
-
response = HTTPX.accept("text/html").get(uri)
-
verify_status(response, 200)
-
request = response.instance_variable_get(:@request)
-
verify_header(request.headers, "accept", "text/html")
-
response.close
-
end
-
-
1
def test_get_idn
-
response = HTTPX.get("http://bücher.ch")
-
verify_status(response, 301)
-
verify_header(response.headers, "location", "https://www.buecher.de")
-
-
response.close
-
-
assert response.instance_variable_get(:@request).authority == "xn--bcher-kva.ch"
-
end
-
-
1
def test_get_non_ascii
-
response = HTTPX.get(build_uri("/get?q=ã"))
-
verify_status(response, 200)
-
response.close
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Head
-
1
def test_http_head
-
uri = build_uri("/get")
-
response = HTTPX.head(uri)
-
verify_status(response, 200)
-
verify_body_length(response, 0)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Headers
-
1
def test_http_headers
-
uri = build_uri("/headers")
-
response = HTTPX.get(uri)
-
body = json_body(response)
-
assert body.key?("headers"), "no headers"
-
assert body["headers"]["Accept"] == "*/*", "unexpected accept"
-
-
response = HTTPX.with_headers("accept" => "text/css").get(uri)
-
body = json_body(response)
-
verify_header(body["headers"], "Accept", "text/css")
-
end
-
-
1
def test_http_user_agent
-
uri = build_uri("/user-agent")
-
response = HTTPX.get(uri)
-
body = json_body(response)
-
verify_header(body, "user-agent", "httpx.rb/#{HTTPX::VERSION}")
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module IO
-
1
using HTTPX::URIExtensions
-
-
1
def test_http_io
-
io = origin_io
-
uri = build_uri("/get")
-
response = HTTPX.get(uri, io: io)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert !io.closed?, "io should have been left open"
-
ensure
-
io.close if io
-
end
-
-
1
def test_http_io_hash
-
io = origin_io
-
uri = build_uri("/get")
-
response = HTTPX.get(uri, io: { URI(origin).authority => io })
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert !io.closed?, "io should have been left open"
-
ensure
-
io.close if io
-
end
-
end
-
-
1
private
-
-
1
def origin_io
-
uri = URI(origin)
-
case uri.scheme
-
when "http"
-
TCPSocket.new(uri.host, uri.port)
-
when "https"
-
ctx = OpenSSL::SSL::SSLContext.new
-
ctx.alpn_protocols = %w[h2 http/1.1]
-
sock = OpenSSL::SSL::SSLSocket.new(TCPSocket.new(uri.host, uri.port), ctx)
-
sock.hostname = uri.host
-
sock.sync_close = true
-
sock.connect
-
sock
-
else
-
raise "#{uri.scheme}: unsupported scheme"
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "time"
-
-
1
module Requests
-
1
module Limits
-
1
def test_limits_max_response_body_size
-
uri = build_uri("/get")
-
response = HTTPX.get(uri, max_response_body_size: 200)
-
verify_error_response(response)
-
verify_error_response(response, /maximum response body size exceeded/)
-
-
chunked_uri = build_uri("/stream-bytes/30?chunk_size=5")
-
chunked_response = HTTPX.get(chunked_uri, max_response_body_size: 25)
-
verify_error_response(chunked_response)
-
verify_error_response(chunked_response, /maximum response body size exceeded/)
-
end
-
-
1
def test_limits_max_response_headers
-
frontend_headers = 7 # date, content-type, etc always there
-
frontend_headers += 1 if scheme == "http://" # connection header for http/1
-
uri = build_uri("/response-headers?h1=v1&h2=v2&h3=v3&v4=v4")
-
response = HTTPX.get(uri, max_response_headers: 4 + frontend_headers)
-
verify_status(response, 200)
-
-
response = HTTPX.get(uri, max_response_headers: 3 + frontend_headers)
-
verify_error_response(response)
-
verify_error_response(response, /maximum number of response headers exceeded/)
-
end
-
-
1
def test_limits_max_header_value_size
-
uri = build_uri("/response-headers?cookie=asdfasdfasdf")
-
response = HTTPX.get(uri, max_response_header_value_size: 200)
-
verify_status(response, 200)
-
-
response = HTTPX.get(uri, max_response_header_value_size: 10)
-
verify_error_response(response)
-
verify_error_response(response, /maximum header value size exceeded/)
-
-
uri2 = build_uri("/response-headers?cookie=asdf&cookie=asdf")
-
response = HTTPX.get(uri2, max_response_header_value_size: 4)
-
verify_error_response(response)
-
verify_error_response(response, /maximum header value size exceeded/)
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "http/form_data"
-
-
1
module Requests
-
1
module Multipart
-
1
%w[post put patch delete].each do |meth|
-
4
define_method :"test_multipart_urlencoded_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { "foo" => "bar" })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/x-www-form-urlencoded")
-
verify_uploaded(body, "form", "foo" => "bar")
-
end
-
-
4
define_method :"test_multipart_nested_urlencoded_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { "q" => { "a" => "z" }, "a" => %w[1 2] })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/x-www-form-urlencoded")
-
verify_uploaded(body, "form", "q[a]" => "z", "a[]" => %w[1 2])
-
end
-
-
4
define_method :"test_multipart_repeated_field_urlencoded_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: [%w[foo bar1], %w[foo bar2]])
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/x-www-form-urlencoded")
-
verify_uploaded(body, "form", "foo" => %w[bar1 bar2])
-
end
-
-
4
define_method :"test_multipart_hash_#{meth}" do
-
uri = build_uri("/#{meth}")
-
req_body = JSON.dump({ a: 1 }).freeze
-
response = HTTPX.send(meth, uri, form: { metadata: { content_type: "application/json", body: req_body } })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
assert JSON.parse(body["form"]["metadata"], symbolize_names: true) == { a: 1 }
-
-
return unless can_run_ractor_tests?
-
-
response2 = Ractor.new(meth, uri, req_body) do |meth, uri, req_body|
-
HTTPX.send(meth, uri, form: { metadata: { content_type: "application/json", body: req_body } })
-
end.value
-
-
verify_status(response2, 200)
-
body2 = json_body(response2)
-
verify_header(body2["headers"], "Content-Type", "multipart/form-data")
-
assert JSON.parse(body2["form"]["metadata"], symbolize_names: true) == { a: 1 }
-
end
-
-
4
define_method :"test_multipart_nested_hash_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { q: { metadata: { content_type: "application/json", body: JSON.dump({ a: 1 }) } } })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
assert JSON.parse(body["form"]["q[metadata]"], symbolize_names: true) == { a: 1 }
-
end
-
-
4
define_method :"test_multipart_nested_array_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { q: [{ content_type: "application/json", body: JSON.dump({ a: 1 }) }] })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
assert JSON.parse(body["form"]["q[]"], symbolize_names: true) == { a: 1 }
-
end
-
-
4
define_method :"test_multipart_file_#{meth}" do
-
uri = build_uri("/#{meth}")
-
image_path = fixture_file_path
-
response = HTTPX.send(meth, uri, form: { image: File.new(image_path) })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "image", "image/jpeg")
-
-
return unless can_run_ractor_tests?
-
-
response2 = Ractor.new(meth, uri, image_path) do |meth, uri, image_path|
-
HTTPX.send(meth, uri, form: { image: File.new(image_path) })
-
end.value
-
-
verify_status(response2, 200)
-
body2 = json_body(response2)
-
verify_header(body2["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body2, "image", "image/jpeg")
-
end
-
-
4
define_method :"test_multipart_file_repeated_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: [
-
%w[foo bar1],
-
["image1", File.new(fixture_file_path)],
-
%w[foo bar2],
-
["image2", File.new(fixture_file_path)],
-
])
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded(body, "form", "foo" => %w[bar1 bar2])
-
verify_uploaded_image(body, "image1", "image/jpeg")
-
verify_uploaded_image(body, "image2", "image/jpeg")
-
end
-
-
4
define_method :"test_multipart_nested_file_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { q: { image: File.new(fixture_file_path) } })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "q[image]", "image/jpeg")
-
end
-
-
4
define_method :"test_multipart_nested_ary_file_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { images: [File.new(fixture_file_path)] })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "images[]", "image/jpeg")
-
end
-
-
4
define_method :"test_multipart_filename_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { image: { filename: "selfie", body: File.new(fixture_file_path) } })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "image", "image/jpeg")
-
# TODO: find out how to check the filename given.
-
end
-
-
4
define_method :"test_multipart_nested_filename_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { q: { image: { filename: "selfie", body: File.new(fixture_file_path) } } })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "q[image]", "image/jpeg")
-
# TODO: find out how to check the filename given.
-
end
-
-
4
define_method :"test_multipart_subnested_filename_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { q: { image: File.new(fixture_file_path) } })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "q[image]", "image/jpeg")
-
end
-
-
4
define_method :"test_multipart_pathname_#{meth}" do
-
uri = build_uri("/#{meth}")
-
image_path = fixture_file_path
-
response = HTTPX.send(meth, uri, form: { image: Pathname.new(image_path) })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "image", "image/jpeg")
-
-
return unless can_run_ractor_tests?
-
-
response2 = Ractor.new(meth, uri, image_path) do |meth, uri, image_path|
-
HTTPX.send(meth, uri, form: { image: Pathname.new(image_path) })
-
end.value
-
-
verify_status(response2, 200)
-
body2 = json_body(response2)
-
verify_header(body2["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body2, "image", "image/jpeg")
-
end
-
-
4
define_method :"test_multipart_nested_pathname_#{meth}" do
-
uri = build_uri("/#{meth}")
-
file = Pathname.new(fixture_file_path)
-
response = HTTPX.send(meth, uri, form: { q: { image: file } })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "q[image]", "image/jpeg")
-
end
-
-
4
define_method :"test_multipart_http_formdata_#{meth}" do
-
uri = build_uri("/#{meth}")
-
file = HTTP::FormData::File.new(fixture_file_path, content_type: "image/jpeg")
-
response = HTTPX.send(meth, uri, form: { image: file })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "image", "image/jpeg")
-
end
-
-
4
define_method :"test_multipart_nested_http_formdata_#{meth}" do
-
uri = build_uri("/#{meth}")
-
file = HTTP::FormData::File.new(fixture_file_path, content_type: "image/jpeg")
-
response = HTTPX.send(meth, uri, form: { q: { image: file } })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
verify_uploaded_image(body, "q[image]", "image/jpeg")
-
end
-
-
4
define_method :"test_multipart_spoofed_file_#{meth}" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { image: {
-
content_type: "image/jpeg",
-
filename: "selfie",
-
body: "spoofpeg",
-
} })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "multipart/form-data")
-
# httpbin accepts the spoofed part, but it wipes our the content-type header
-
verify_uploaded_image(body, "image", "spoofpeg", skip_verify_data: true)
-
end
-
end
-
-
1
def test_multipart_response_decoder
-
form_response = HTTPX::Response.new(
-
HTTPX::Request.new("GET", "http://example.com", HTTPX::Options.new),
-
200,
-
"2.0",
-
{ "content-type" => "multipart/form-data; boundary=90" }
-
)
-
form_response << [
-
"--90\r\n",
-
"Content-Disposition: form-data; name=\"text\"\r\n\r\n",
-
"text default\r\n",
-
"--90\r\n",
-
"Content-Disposition: form-data; name=\"file1\"; filename=\"a.txt\"\r\n",
-
"Content-Type: text/plain\r\n\r\n",
-
"Content of a.txt.\r\n\r\n",
-
"--90\r\n",
-
"Content-Disposition: form-data; name=\"file2\"; filename=\"a.html\"\r\n",
-
"Content-Type: text/html\r\n\r\n",
-
"<!DOCTYPE html><title>Content of a.html.</title>\r\n\r\n",
-
"--90--",
-
].join
-
form = form_response.form
-
-
begin
-
assert form["text"] == "text default"
-
assert form["file1"].original_filename == "a.txt"
-
assert form["file1"].content_type == "text/plain"
-
assert form["file1"].read == "Content of a.txt."
-
-
assert form["file2"].original_filename == "a.html"
-
assert form["file2"].content_type == "text/html"
-
assert form["file2"].read == "<!DOCTYPE html><title>Content of a.html.</title>"
-
ensure
-
form["file1"].close
-
form["file1"].unlink
-
form["file2"].close
-
form["file2"].unlink
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Authentication
-
1
def test_plugin_auth
-
get_uri = build_uri("/get")
-
session = HTTPX.plugin(:auth)
-
-
response = session.authorization("TOKEN").get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "TOKEN")
-
end
-
-
1
def test_plugin_auth_with_block
-
get_uri = build_uri("/get")
-
session = HTTPX.plugin(:auth)
-
-
i = 0
-
response = session.authorization { "TOKEN#{i += 1}" }.get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "TOKEN1")
-
response = session.authorization { "TOKEN#{i += 1}" }.get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "TOKEN2")
-
end
-
-
1
def test_plugin_auth_reset_auth_value
-
get_uri = build_uri("/get")
-
session = HTTPX.plugin(:auth)
-
-
i = 0
-
authed = session.authorization { "TOKEN#{i += 1}" }
-
2.times do
-
# proves that token is reused
-
response = authed.get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "TOKEN1")
-
end
-
-
# proves that token is discarded
-
authed.reset_auth_header_value!
-
response = authed.get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "TOKEN2")
-
end
-
-
1
def test_plugin_auth_reset_auth_value_expires_at
-
get_uri = build_uri("/get")
-
session = HTTPX.plugin(:auth, auth_header_expires_at: ->(_req) { Time.now.to_i + 2 })
-
-
i = 0
-
authed = session.authorization { "TOKEN#{i += 1}" }
-
2.times do
-
# proves that token is reused
-
response = authed.get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "TOKEN1")
-
end
-
-
sleep 2
-
-
# proves that token is discarded
-
response = authed.get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "TOKEN2")
-
end
-
-
1
def test_plugin_auth_reset_auth_value_expires_in
-
get_uri = build_uri("/get")
-
session = HTTPX.plugin(:auth, auth_header_expires_in: 2)
-
-
i = 0
-
authed = session.authorization { "TOKEN#{i += 1}" }
-
2.times do
-
# proves that token is reused
-
response = authed.get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "TOKEN1")
-
end
-
-
sleep 2
-
-
# proves that token is discarded
-
response = authed.get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "TOKEN2")
-
end
-
-
1
def test_plugin_auth_generate_token_once_for_multi_request
-
get_uri = build_uri("/get")
-
authed = HTTPX.plugin(:auth)
-
i = 0
-
r1, r2 = authed.authorization { "TOKEN#{i += 1}" }.get(get_uri, get_uri)
-
verify_status(r1, 200)
-
body = json_body(r1)
-
verify_header(body["headers"], "Authorization", "TOKEN1")
-
-
verify_status(r2, 200)
-
body = json_body(r2)
-
verify_header(body["headers"], "Authorization", "TOKEN1")
-
end
-
-
1
def test_plugin_auth_regenerate_on_retry
-
i = 0
-
session = HTTPX.plugin(RequestInspector)
-
.plugin(:retries, max_retries: 1, retry_on: ->(res) { res.respond_to?(:status) && res.status == 400 })
-
.plugin(:auth, generate_auth_value_on_retry: ->(res) { res.respond_to?(:status) && res.status == 400 })
-
.with(timeout: { request_timeout: 3 })
-
.authorization { "TOKEN#{i += 1}" }
-
-
response = session.get(build_uri("/status/400"))
-
verify_status(response, 400)
-
assert session.calls == 1, "expected two errors to have been sent"
-
req1, req2 = session.total_requests
-
assert req1.headers["authorization"] == "TOKEN1"
-
assert req2.headers["authorization"] == "TOKEN2"
-
session.reset
-
-
# 401 errors are always retried with a fresh token, no matter the verb
-
response = session.get(build_uri("/status/401"))
-
verify_status(response, 401)
-
assert session.calls == 1, "expected two errors to have been sent"
-
req1, req2 = session.total_requests
-
assert req1.headers["authorization"] == "TOKEN2", "the last successful token should have been reused"
-
assert req2.headers["authorization"] == "TOKEN3"
-
session.reset
-
-
# on regular errors, it should try to reuse the same token
-
response = session.get(build_uri("/delay/10"))
-
verify_error_response(response, HTTPX::RequestTimeoutError)
-
assert session.calls == 1, "expected two errors to have been sent"
-
req1, req2 = session.total_requests
-
assert req1.headers["authorization"] == "TOKEN3", "the last successful token should have been reused"
-
assert req2.headers["authorization"] == "TOKEN3", "the previous token should have been reused"
-
end
-
-
1
def test_plugin_auth_multi_request_regenerate_on_retry
-
i = 0
-
session = HTTPX.plugin(RequestInspector)
-
.plugin(:retries, max_retries: 1, retry_on: ->(res) { res.respond_to?(:status) && res.status == 400 })
-
.plugin(:auth, generate_auth_value_on_retry: ->(res) { res.respond_to?(:status) && res.status == 400 })
-
.with(timeout: { request_timeout: 3 })
-
.authorization { "TOKEN#{i += 1}" }
-
-
uri = build_uri("/status/401")
-
-
responses = session.get(uri, uri, uri)
-
-
assert responses.size == 3
-
assert session.calls == 5, "expected two errors to have been sent per request"
-
-
responses.each do |response|
-
verify_status(response, 401)
-
request = response.request
-
assert request.headers["authorization"] == "TOKEN2", "the last successful token should have been reused"
-
end
-
end
-
-
# Bearer Auth
-
-
1
def test_plugin_bearer_auth
-
get_uri = build_uri("/get")
-
session = HTTPX.plugin(:auth)
-
response = session.bearer_auth("TOKEN").get(get_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Authorization", "Bearer TOKEN")
-
end
-
-
# Basic Auth
-
-
1
def test_plugin_basic_auth
-
no_auth_response = HTTPX.get(basic_auth_uri)
-
verify_status(no_auth_response, 401)
-
verify_header(no_auth_response.headers, "www-authenticate", "Basic realm=\"Fake Realm\"")
-
no_auth_response.close
-
-
session = HTTPX.plugin(:basic_auth)
-
response = session.basic_auth(user, pass).get(basic_auth_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body, "authenticated", true)
-
verify_header(body, "user", user)
-
-
invalid_response = session.basic_auth(user, "fake").get(basic_auth_uri)
-
verify_status(invalid_response, 401)
-
end
-
-
# Digest
-
-
1
def test_plugin_digest_auth
-
session = HTTPX.plugin(:digest_auth).with_headers("cookie" => "fake=fake_value")
-
response = session.digest_auth(user, pass).get(digest_auth_uri)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body, "authenticated", true)
-
verify_header(body, "user", user)
-
end
-
-
1
%w[SHA1 SHA2 SHA256 SHA384 SHA512 RMD160].each do |alg|
-
6
define_method :"test_plugin_digest_auth_#{alg}" do
-
session = HTTPX.plugin(:digest_auth).with_headers("cookie" => "fake=fake_value")
-
response = session.digest_auth(user, pass).get("#{digest_auth_uri}/#{alg}")
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body, "authenticated", true)
-
verify_header(body, "user", user)
-
end
-
end
-
-
1
%w[MD5 SHA1].each do |alg|
-
2
define_method :"test_plugin_digest_auth_#{alg}_sess" do
-
start_test_servlet(DigestServer, algorithm: "#{alg}-sess") do |server|
-
uri = "#{server.origin}/"
-
session = HTTPX.plugin(:digest_auth).with_headers("cookie" => "fake=fake_value")
-
response = session.digest_auth(user, server.get_passwd(user), hashed: true).get(uri)
-
verify_status(response, 200)
-
assert response.read == "yay"
-
end
-
end
-
end
-
-
1
def test_plugin_digest_auth_bypass
-
session = HTTPX.plugin(:digest_auth).with_headers("cookie" => "fake=fake_value")
-
response = session.get(digest_auth_uri)
-
verify_status(response, 401)
-
response = session.get(build_uri("/get"))
-
verify_status(response, 200)
-
response = session.digest_auth(user, pass).get(build_uri("/get"))
-
verify_status(response, 200)
-
end
-
-
1
def test_plugin_digest_auth_invalid_header
-
start_test_servlet(InvalidDigestServer) do |server|
-
uri = "#{server.origin}/"
-
session = HTTPX.plugin(:digest_auth)
-
response = session.digest_auth("user", "pass").get(uri)
-
verify_error_response(response, "unsupported digest header format")
-
end
-
end
-
-
# NTLM
-
-
1
if RUBY_VERSION < "3.1.0"
-
# TODO: enable again once ruby-openssl 3 supports legacy ciphers
-
def test_plugin_ntlm_auth
-
return if origin.start_with?("https")
-
-
start_test_servlet(NTLMServer) do |server|
-
uri = "#{server.origin}/"
-
HTTPX.plugin(SessionWithPool).plugin(:ntlm_auth).wrap do |http|
-
# skip unless NTLM
-
no_auth_response = http.get(uri)
-
verify_status(no_auth_response, 401)
-
no_auth_response.close
-
-
response = http.ntlm_auth("user", "password").get(uri)
-
verify_status(response, 200)
-
-
# bypass
-
response = http.get(build_uri("/get"))
-
verify_status(response, 200)
-
response = http.ntlm_auth("user", "password").get(build_uri("/get"))
-
verify_status(response, 200)
-
# invalid_response = http.ntlm_auth("user", "fake").get(uri)
-
# verify_status(invalid_response, 401)
-
end
-
end
-
end
-
end
-
-
# NTLMv2
-
-
1
def test_plugin_ntlm_v2_auth
-
return if origin.start_with?("https")
-
-
start_test_servlet(NTLMServer) do |server|
-
uri = "#{server.origin}/"
-
HTTPX.plugin(SessionWithPool).plugin(:ntlm_v2_auth).wrap do |http|
-
# authenticated request should succeed via NTLMv2 handshake
-
response = http.ntlm_auth("user", "password").get(uri)
-
verify_status(response, 200)
-
end
-
end
-
end
-
-
1
def test_plugin_ntlm_v2_auth_with_domain
-
return if origin.start_with?("https")
-
-
start_test_servlet(NTLMServer) do |server|
-
uri = "#{server.origin}/"
-
HTTPX.plugin(SessionWithPool).plugin(:ntlm_v2_auth).wrap do |http|
-
response = http.ntlm_auth("user", "password", "DOMAIN").get(uri)
-
verify_status(response, 200)
-
end
-
end
-
end
-
-
1
def test_plugin_ntlm_v2_auth_option_type_check
-
session = HTTPX.plugin(:ntlm_v2_auth)
-
assert_raises(TypeError) do
-
session.with(ntlm: "not an authenticator")
-
end
-
end
-
-
1
private
-
-
1
def basic_auth_uri
-
build_uri("/basic-auth/#{user}/#{pass}")
-
end
-
-
1
def digest_auth_uri(qop = "auth")
-
build_uri("/digest-auth/#{qop}/#{user}/#{pass}")
-
end
-
-
1
def user
-
"user"
-
end
-
-
1
def pass
-
"pass"
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "aws-sdk-s3"
-
-
1
module Requests
-
1
module Plugins
-
1
module AWSAuthentication
-
1
AWS_URI = ENV.fetch("AMZ_HOST", "aws:9090")
-
1
AWSS_URI = ENV.fetch("AMZS_HOST", "aws:9191")
-
-
1
def test_plugin_aws_authentication_put_object
-
begin
-
s3_client = Aws::S3::Client.new(
-
endpoint: amz_uri,
-
force_path_style: true,
-
ssl_verify_peer: false,
-
# http_wire_trace: true,
-
# logger: Logger.new(STDERR)
-
)
-
s3_client.create_bucket(bucket: "test", acl: "private")
-
rescue Aws::S3::Errors::BucketAlreadyExists,
-
Aws::S3::Errors::BucketAlreadyOwnedByYou
-
# because this test will run 2 times (http and https)
-
end
-
-
object = s3_client.put_object(bucket: "test", key: "testimage", body: "bucketz")
-
-
# now let's get it
-
# no_sig_response = HTTPX.get("http://#{AWS_URI}/test/testimage")
-
# verify_error_response(no_sig_response)
-
-
aws_req_headers = object.context.http_request.headers
-
-
response = aws_s3_session(unsigned_headers: %w[accept content-type content-length])
-
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE },
-
headers: {
-
"user-agent" => aws_req_headers["user-agent"],
-
# gotta fix localstack first
-
# "expect" => "100-continue",
-
"x-amz-date" => aws_req_headers["x-amz-date"],
-
"content-md5" => OpenSSL::Digest.base64digest("MD5", "bucketz"),
-
})
-
.put("#{amz_uri}/test/testimage", body: "bucketz")
-
verify_status(response, 200)
-
-
# testing here to make sure the plugin is loaded
-
config = HTTPX::Plugins::AwsSdkAuthentication::Configuration.new("default")
-
assert config.respond_to?(:balls)
-
assert config.balls.nil?
-
end
-
-
1
private
-
-
1
def aws_s3_session(**options)
-
HTTPX.plugin(:aws_sdk_authentication, aws_profile: "default").aws_sdk_authentication(service: "s3", **options)
-
end
-
-
1
def amz_uri
-
uri = scheme == "https://" ? AWSS_URI : AWS_URI
-
origin(uri)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Brotli
-
1
def test_brotli
-
session = HTTPX.plugin(:brotli)
-
response = session.get("http://nghttp2.org/httpbin/brotli")
-
-
if (response.respond_to?(:status) && response.status.between?(502, 504)) ||
-
(response.respond_to?(:error) && response.error.is_a?(HTTPX::TimeoutError))
-
skip "`#{response.uri}` is down again"
-
end
-
-
verify_status(response, 200)
-
body = json_body(response)
-
assert body["brotli"], "response should be deflated"
-
-
# but gzip still works
-
uri = build_uri("/gzip")
-
response = session.get(uri)
-
verify_status(response, 200)
-
assert response.headers["content-length"].to_i != response.body.bytesize
-
body = json_body(response)
-
assert body["gzipped"]
-
end
-
-
1
def test_brotli_post
-
session = HTTPX.plugin(:brotli)
-
uri = build_uri("/post")
-
response = session.with_headers("content-encoding" => "br")
-
.post(uri, body: "a" * 8012)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/octet-stream")
-
compressed_data = body["data"]
-
compressed_data = compressed_data.delete_prefix("data:application/octet-stream;base64,")
-
compressed_data = Base64.decode64(compressed_data)
-
assert compressed_data.bytesize < 8012, "body hasn't been compressed"
-
assert ::Brotli.inflate(compressed_data) == "a" * 8012
-
-
# but gzip still works
-
uri = build_uri("/post")
-
response = session.with_headers("content-encoding" => "gzip")
-
.post(uri, body: "a" * 8012)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/octet-stream")
-
compressed_data = body["data"]
-
compressed_data = compressed_data.delete_prefix("data:application/octet-stream;base64,")
-
compressed_data = Base64.decode64(compressed_data)
-
assert compressed_data.bytesize < 8012, "body hasn't been compressed"
-
assert inflate_test_data(compressed_data) == "a" * 8012
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "securerandom"
-
-
1
module Requests
-
1
module Plugins
-
1
module Cache
-
1
def test_plugin_cache_options
-
cache_client = HTTPX.plugin(:cache, response_cache_store: :store)
-
assert cache_client.class.default_options.response_cache_store.is_a?(HTTPX::Plugins::Cache::Store)
-
cache_client = HTTPX.plugin(:cache, response_cache_store: :file_store)
-
assert cache_client.class.default_options.response_cache_store.is_a?(HTTPX::Plugins::Cache::FileStore)
-
end
-
-
1
def test_plugin_cache_cacheable_request_and_response
-
cache_client = HTTPX.plugin(
-
:cache,
-
cache_key: ->(req) { req.uri.path },
-
cacheable_request: ->(req) { req.uri.path.end_with?("/200", "/202") },
-
cacheable_response: ->(_, res) { res.status == 200 },
-
valid_cached_response: ->(_, _) { true },
-
)
-
-
cacheable_request_uri = build_uri("/status/200")
-
uncacheable_request_uri = build_uri("/status/201")
-
uncacheable_response_uri = build_uri("/status/202")
-
-
# cacheable request path
-
-
original = cache_client.get(cacheable_request_uri)
-
verify_status(original, 200)
-
cached = cache_client.get(cacheable_request_uri)
-
verify_status(cached, 200)
-
assert original.body == cached.body
-
cache_client.clear_response_cache
-
uncached = cache_client.get(cacheable_request_uri)
-
verify_status(uncached, 200)
-
assert uncached != original
-
-
# uncacheable request
-
original = cache_client.get(uncacheable_request_uri)
-
verify_status(original, 201)
-
uncached = cache_client.get(uncacheable_request_uri)
-
verify_status(uncached, 201)
-
assert uncached != original
-
-
# uncacheable response
-
original = cache_client.get(uncacheable_response_uri)
-
verify_status(original, 202)
-
uncached = cache_client.get(uncacheable_response_uri)
-
verify_status(uncached, 202)
-
assert uncached != original
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module CircuitBreaker
-
1
using HTTPX::URIExtensions
-
-
1
def test_plugin_circuit_breaker_lifecycles
-
return unless origin.start_with?("http://")
-
-
unknown_uri = "http://www.qwwqjqwdjqiwdj.com"
-
-
session = HTTPX.plugin(:circuit_breaker,
-
circuit_breaker_max_attempts: 2,
-
circuit_breaker_break_in: 2,
-
circuit_breaker_half_open_drip_rate: 1.0)
-
-
# circuit closed
-
response1 = session.get(unknown_uri)
-
verify_error_response(response1)
-
-
response2 = session.get(unknown_uri)
-
verify_error_response(response2)
-
assert response2 != response1
-
-
# circuit open
-
response3 = session.get(unknown_uri)
-
verify_error_response(response3)
-
assert response3 == response2
-
-
sleep 3
-
-
# circuit half-closed
-
response4 = session.get(unknown_uri)
-
assert response4 != response3
-
end
-
-
1
def test_plugin_circuit_breaker_reset_attempts
-
return unless origin.start_with?("http://")
-
-
unknown_uri = URI("http://www.qwwqjqwdjqiwdj.com")
-
-
session = HTTPX.plugin(:circuit_breaker,
-
circuit_breaker_max_attempts: 2,
-
circuit_breaker_reset_attempts_in: 2)
-
-
store = session.instance_variable_get(:@circuit_store)
-
circuit = store.instance_variable_get(:@circuits)[unknown_uri.origin]
-
-
# circuit closed
-
response1 = session.get(unknown_uri)
-
verify_error_response(response1)
-
assert circuit.instance_variable_get(:@attempts) == 1
-
sleep 2
-
response1 = session.get(unknown_uri)
-
verify_error_response(response1)
-
# because it reset
-
assert circuit.instance_variable_get(:@attempts) == 1
-
end
-
-
1
def test_plugin_circuit_breaker_break_on
-
break_on = ->(response) { response.is_a?(HTTPX::ErrorResponse) || response.status == 404 }
-
session = HTTPX.plugin(:circuit_breaker, circuit_breaker_max_attempts: 1, circuit_breaker_break_on: break_on)
-
-
response1 = session.get(build_uri("/status/404"))
-
verify_status(response1, 404)
-
-
response2 = session.get(build_uri("/status/404"))
-
verify_status(response2, 404)
-
assert response1 == response2
-
end
-
-
1
def test_plugin_circuit_breaker_on_circuit_open
-
return unless origin.start_with?("http://")
-
-
unknown_uri = "http://www.qwwqjqwdjqiwdj.com"
-
-
circuit_opened = false
-
session = HTTPX.plugin(:circuit_breaker,
-
circuit_breaker_max_attempts: 1,
-
circuit_breaker_break_in: 2,
-
circuit_breaker_half_open_drip_rate: 1.0)
-
.on_circuit_open { circuit_opened = true }
-
-
# circuit closed
-
response1 = session.get(unknown_uri)
-
verify_error_response(response1)
-
-
# circuit open
-
response2 = session.get(unknown_uri)
-
verify_error_response(response2)
-
assert response2 == response1
-
-
assert circuit_opened
-
end
-
-
1
def test_plugin_circuit_breaker_half_open_drip_rate
-
delay_url = URI(build_uri("/delay/2"))
-
-
session = HTTPX.plugin(:circuit_breaker, circuit_breaker_max_attempts: 2, circuit_breaker_half_open_drip_rate: 0.5,
-
circuit_breaker_break_in: 1)
-
-
store = session.instance_variable_get(:@circuit_store)
-
circuit = store.instance_variable_get(:@circuits)[delay_url.origin]
-
-
response1 = session.get(delay_url, timeout: { request_timeout: 0.5 })
-
response2 = session.get(delay_url, timeout: { request_timeout: 0.5 })
-
verify_error_response(response1, HTTPX::RequestTimeoutError)
-
verify_error_response(response2, HTTPX::RequestTimeoutError)
-
-
# circuit open
-
assert circuit.instance_variable_get(:@attempts) == 2
-
assert circuit.instance_variable_get(:@state) == :open
-
-
sleep 1.5
-
-
# circuit half-open
-
response3 = session.get(delay_url)
-
verify_status(response3, 200)
-
-
assert circuit.instance_variable_get(:@attempts) == 1
-
assert circuit.instance_variable_get(:@state) == :half_open
-
-
response4 = session.get(delay_url)
-
verify_error_response(response4, HTTPX::RequestTimeoutError)
-
-
assert circuit.instance_variable_get(:@attempts) == 2
-
assert circuit.instance_variable_get(:@state) == :half_open
-
-
# circuit closed again
-
response5 = session.get(delay_url)
-
verify_status(response5, 200)
-
-
assert circuit.instance_variable_get(:@state) == :closed
-
-
response1 = session.get(delay_url, timeout: { request_timeout: 0.5 })
-
response2 = session.get(delay_url, timeout: { request_timeout: 0.5 })
-
verify_error_response(response1, HTTPX::RequestTimeoutError)
-
verify_error_response(response2, HTTPX::RequestTimeoutError)
-
-
# circuit open
-
assert circuit.instance_variable_get(:@attempts) == 2
-
assert circuit.instance_variable_get(:@state) == :open
-
-
sleep 1.5
-
-
# circuit half-open
-
response3 = session.get(delay_url, timeout: { request_timeout: 0.5 })
-
verify_error_response(response3, HTTPX::RequestTimeoutError)
-
-
# attempts reset, haf-open -> open transition
-
assert circuit.instance_variable_get(:@attempts) == 1
-
assert circuit.instance_variable_get(:@state) == :open
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module ContentDigest
-
1
IGNORE_MISSING_HEADER = ->(res) { res.headers.key?("content-digest") }
-
-
1
def test_content_digest_missing_no_validation
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: false)
-
-
%w[/no_content_digest /invalid_content_digest].each do |path|
-
response = http.get(server.origin + path)
-
-
verify_status(response, 200)
-
assert response.body.content_digest_buffer.nil?
-
end
-
end
-
end
-
-
1
def test_content_digest_missing_validation_if_present
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: IGNORE_MISSING_HEADER)
-
-
response = http.get("#{server.origin}/no_content_digest")
-
-
verify_status(response, 200)
-
assert response.body.content_digest_buffer.nil?
-
end
-
end
-
-
1
def test_content_digest_missing_validation_always
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: true)
-
-
response = http.get("#{server.origin}/no_content_digest")
-
-
verify_error_response(response, HTTPX::Plugins::ContentDigest::MissingContentDigestError)
-
end
-
end
-
-
1
def test_content_digest_present_validation_if_present
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: IGNORE_MISSING_HEADER)
-
-
response = http.get("#{server.origin}/valid_content_digest")
-
-
verify_status(response, 200)
-
assert !response.body.content_digest_buffer.nil?
-
response.close
-
assert response.body.content_digest_buffer.nil?
-
end
-
end
-
-
1
def test_content_digest_present_validation_always
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: true)
-
-
response = http.get("#{server.origin}/valid_content_digest")
-
-
verify_status(response, 200)
-
assert !response.body.content_digest_buffer.nil?
-
response.close
-
assert response.body.content_digest_buffer.nil?
-
end
-
end
-
-
1
def test_content_digest_invalid_validation_if_present
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: IGNORE_MISSING_HEADER)
-
-
response = http.get("#{server.origin}/invalid_content_digest")
-
-
verify_error_response(response, HTTPX::Plugins::ContentDigest::InvalidContentDigestError)
-
response.close
-
end
-
end
-
-
1
def test_content_digest_invalid_validation_always
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: true)
-
-
response = http.get("#{server.origin}/invalid_content_digest")
-
-
verify_error_response(response, HTTPX::Plugins::ContentDigest::InvalidContentDigestError)
-
response.close
-
end
-
end
-
-
1
def test_content_digest_multiple_validation_always
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: true)
-
-
response = http.get("#{server.origin}/multiple_content_digests")
-
-
verify_status(response, 200)
-
assert !response.body.content_digest_buffer.nil?
-
response.close
-
assert response.body.content_digest_buffer.nil?
-
end
-
end
-
-
1
def test_content_digest_gzip_encoding
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: true)
-
-
response = http.get("#{server.origin}/gzip_content_digest")
-
-
verify_status(response, 200)
-
assert !response.body.content_digest_buffer.nil?
-
response.close
-
assert response.body.content_digest_buffer.nil?
-
end
-
end
-
-
1
def test_content_digest_large_response_body
-
start_test_servlet(ContentDigestServer) do |server|
-
http = HTTPX.plugin(:content_digest, validate_content_digest: true)
-
-
response = http.get("#{server.origin}/large_body_content_digest")
-
-
verify_status(response, 200)
-
assert !response.body.content_digest_buffer.nil?
-
response.close
-
assert response.body.content_digest_buffer.nil?
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Cookies
-
1
using HTTPX::URIExtensions
-
-
1
def test_plugin_cookies_get
-
session = HTTPX.plugin(:cookies)
-
response = session.get(cookies_uri)
-
body = json_body(response)
-
assert body.key?("cookies")
-
assert body["cookies"].empty?
-
-
session_response = session.with(cookies: [%w[abc def]]).get(cookies_uri)
-
body = json_body(session_response)
-
assert body.key?("cookies")
-
assert body["cookies"]["abc"] == "def", "abc wasn't properly set"
-
end
-
-
1
def test_plugin_cookies_get_with_hash
-
session = HTTPX.plugin(:cookies)
-
session_response = session.with(cookies: [{ "name" => "abc", "value" => "def" }]).get(cookies_uri)
-
body = json_body(session_response)
-
assert body.key?("cookies")
-
assert body["cookies"]["abc"] == "def", "abc wasn't properly set"
-
end
-
-
1
def test_plugin_cookies_get_with_cookie
-
session = HTTPX.plugin(:cookies)
-
session_response = session.with(cookies: [HTTPX::Plugins::Cookies::Cookie.new("abc", "def")]).get(cookies_uri)
-
body = json_body(session_response)
-
assert body.key?("cookies")
-
assert body["cookies"]["abc"] == "def", "abc wasn't properly set"
-
end
-
-
1
def test_plugin_cookies_set
-
session = HTTPX.plugin(:cookies)
-
session_cookies = { "a" => "b", "c" => "d" }
-
session_uri = cookies_set_uri(session_cookies)
-
session_response = session.get(session_uri)
-
verify_status(session_response, 302)
-
verify_cookies(session.cookies[session_uri], session_cookies)
-
-
# first request sets the session
-
response = session.get(cookies_uri)
-
body = json_body(response)
-
assert body.key?("cookies")
-
verify_cookies(body["cookies"], session_cookies)
-
-
# second request reuses the session
-
extra_cookie_response = session.with(cookies: { "e" => "f" }).get(cookies_uri)
-
body = json_body(extra_cookie_response)
-
assert body.key?("cookies")
-
verify_cookies(body["cookies"], session_cookies.merge("e" => "f"))
-
-
# redirect to a different origin only uses the option cookies
-
other_origin_response = session.with(cookies: { "e" => "f" }).get(redirect_uri(origin("google.com")))
-
verify_status(other_origin_response, 302)
-
assert !other_origin_response.headers.key?("set-cookie"), "cookies should not transition to next origin"
-
end
-
-
1
def test_cookies_wrap
-
session = HTTPX.plugin(:cookies).with(cookies: { "abc" => "def" })
-
-
session.wrap do |_http|
-
set_cookie_uri = cookies_set_uri("123" => "456")
-
session_response = session.get(set_cookie_uri)
-
verify_status(session_response, 302)
-
-
session_response = session.get(cookies_uri)
-
body = json_body(session_response)
-
assert body.key?("cookies")
-
assert body["cookies"]["abc"] == "def", "abc wasn't properly set"
-
assert body["cookies"]["123"] == "456", "123 wasn't properly set"
-
-
set_cookie_uri = cookies_set_uri("abc" => "123")
-
session_response = session.get(set_cookie_uri)
-
verify_status(session_response, 302)
-
-
session_response = session.get(cookies_uri)
-
body = json_body(session_response)
-
assert body.key?("cookies")
-
assert body["cookies"]["abc"] == "123", "abc wasn't properly set"
-
end
-
-
session_response = session.get(cookies_uri)
-
body = json_body(session_response)
-
assert body.key?("cookies")
-
assert body["cookies"]["abc"] == "def", "abc wasn't properly set"
-
end
-
-
1
def test_plugin_cookies_follow_redirects
-
session = HTTPX.plugin(:follow_redirects).plugin(:cookies)
-
session_cookies = { "a" => "b", "c" => "d" }
-
session_uri = cookies_set_uri(session_cookies)
-
-
response = session.get(session_uri)
-
verify_status(response, 200)
-
assert response.uri.to_s == cookies_uri
-
body = json_body(response)
-
assert body.key?("cookies")
-
verify_cookies(body["cookies"], session_cookies)
-
end
-
-
1
def test_plugin_cookies_jar_management
-
cookie_header = lambda do |response|
-
JSON.parse(response.to_s)["headers"]
-
end
-
uri = build_uri("/headers")
-
-
http = HTTPX.plugin(:cookies).with(cookies: { :a => 1, :b => 2 })
-
verify_header(cookie_header.call(http.get(uri)), "Cookie", "a=1; b=2")
-
-
http = http.with(cookies: { :a => 3 })
-
verify_header(cookie_header.call(http.get(uri)), "Cookie", "a=3; b=2")
-
-
verify_header(cookie_header.call(http.get(uri, cookies: { :a => 4 })), "Cookie", "a=4; b=2")
-
-
http = http.with(headers: { "Cookie" => "a=1;f=6" })
-
verify_header(cookie_header.call(http.get(uri)), "Cookie", "a=1; b=2; f=6")
-
-
verify_header(cookie_header.call(http.get(uri, cookies: { :a => 4 })), "Cookie", "a=4; b=2; f=6")
-
end
-
-
1
private
-
-
1
def cookies_uri
-
build_uri("/cookies")
-
end
-
-
1
def cookies_set_uri(cookies)
-
URI(build_uri("/cookies/set?#{URI.encode_www_form(cookies)}"))
-
end
-
-
1
def verify_cookies(jar, cookies)
-
assert !jar.nil? && !jar.empty?, "there should be cookies in the response"
-
assert jar.all? { |cookie|
-
case cookie
-
when HTTPX::Plugins::Cookies::Cookie
-
cookies.one? { |k, v| k == cookie.name && v == cookie.value }
-
else
-
cookie_name, cookie_value = cookie
-
cookies.one? { |k, v| k == cookie_name && v == cookie_value }
-
end
-
}, "jar should contain all expected cookies"
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Expect
-
1
def test_plugin_expect_100_form_params
-
uri = build_uri("/post")
-
response = HTTPX.plugin(:expect).post(uri, form: { "foo" => "bar" })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/x-www-form-urlencoded")
-
verify_header(body["headers"], "Expect", "100-continue")
-
verify_uploaded(body, "form", "foo" => "bar")
-
end
-
-
1
def test_plugin_expect_100_with_delay_form_params
-
# run this only for http/1.1 mode, as this is a local test server
-
return unless origin.start_with?("https://")
-
-
start_test_servlet(Expect100Server) do |server|
-
http = HTTPX.plugin(:expect)
-
uri = build_uri("/delay?delay=4", server.origin)
-
response = http.post(uri, body: "helloworld")
-
-
# sometimes httpbin delivers back the intermediate response after the body is sent after the delay
-
skip if response.respond_to?(:error) && response.error.is_a?(EOFError)
-
-
verify_status(response, 200)
-
body = response.body.to_s
-
assert body == "echo: helloworld"
-
verify_header(response.instance_variable_get(:@request).headers, "expect", "100-continue")
-
-
next_request = http.build_request("POST", build_uri("/", server.origin), body: "helloworld")
-
verify_header(next_request.headers, "expect", "100-continue")
-
end
-
end
-
-
1
def test_plugin_expect_100_form_params_under_threshold
-
uri = build_uri("/post")
-
session = HTTPX.plugin(:expect, expect_threshold_size: 4)
-
response = session.post(uri, body: "a" * 3)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_no_header(body["headers"], "Expect")
-
-
response = session.post(uri, body: "a" * 5)
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Expect", "100-continue")
-
end
-
-
1
def test_plugin_expect_100_send_body_after_delay
-
# run this only for http/1.1 mode, as this is a local test server
-
return unless origin.start_with?("http://")
-
-
start_test_servlet(Expect100Server) do |server|
-
http = HTTPX.plugin(:expect)
-
uri = build_uri("/no-expect", server.origin)
-
response = http.post(uri, body: "helloworld")
-
verify_status(response, 200)
-
body = response.body.to_s
-
assert body == "echo: helloworld"
-
verify_no_header(response.instance_variable_get(:@request).headers, "expect")
-
-
next_request = http.build_request("POST", build_uri("/", server.origin), body: "helloworld")
-
verify_no_header(next_request.headers, "expect")
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module FollowRedirects
-
1
def test_plugin_follow_redirects
-
no_redirect_response = HTTPX.get(redirect_uri)
-
verify_status(no_redirect_response, 302)
-
verify_header(no_redirect_response.headers, "location", redirect_location)
-
-
session = HTTPX.plugin(:follow_redirects)
-
redirect_response = session.get(redirect_uri)
-
verify_status(redirect_response, 200)
-
body = json_body(redirect_response)
-
assert body.key?("url"), "url should be set"
-
assert body["url"] == redirect_location, "url should have been the given redirection url"
-
end
-
-
1
def test_plugin_follow_redirects_on_post_302
-
session = HTTPX.plugin(:follow_redirects)
-
redirect_response = session.post(redirect_uri, body: "bang")
-
verify_status(redirect_response, 200)
-
body = json_body(redirect_response)
-
assert body.key?("url"), "url should be set"
-
assert body["url"] == redirect_location, "url should have been the given redirection url"
-
-
request = redirect_response.instance_variable_get(:@request)
-
assert request.uri.to_s == redirect_location
-
assert request.verb == "GET"
-
verify_no_header(request.headers, "content-type")
-
verify_no_header(request.headers, "content-length")
-
-
root_request = request.root_request
-
assert root_request.uri.to_s == redirect_uri
-
assert root_request.verb == "POST"
-
verify_header(root_request.headers, "content-type", "application/octet-stream")
-
verify_header(root_request.headers, "content-length", "4")
-
end
-
-
1
def test_plugin_follow_redirects_on_post_307
-
return unless origin.start_with?("http://")
-
-
start_test_servlet(Redirector307Server) do |server|
-
uri = "#{server.origin}/307"
-
session = HTTPX.plugin(:follow_redirects)
-
redirect_response = session.post(uri, body: "bang")
-
verify_status(redirect_response, 200)
-
assert redirect_response.body == "ok"
-
-
request = redirect_response.instance_variable_get(:@request)
-
assert request.uri.to_s == "#{server.origin}/"
-
assert request.verb == "POST"
-
verify_header(request.headers, "content-type", "application/octet-stream")
-
verify_header(request.headers, "content-length", "4")
-
-
root_request = request.root_request
-
assert root_request.uri.to_s == "#{server.origin}/307"
-
assert root_request.verb == "POST"
-
verify_header(root_request.headers, "content-type", "application/octet-stream")
-
verify_header(root_request.headers, "content-length", "4")
-
end
-
end
-
-
1
def test_plugin_follow_redirects_no_location_no_follow
-
session = HTTPX.plugin(:follow_redirects)
-
-
response = session.with(headers: { "if-none-match" => "justforcingcachedresponse" }).get(redirect_no_follow_uri)
-
verify_status(response, 304)
-
end
-
-
1
def test_plugin_follow_redirects_relative_path
-
session = HTTPX.plugin(:follow_redirects)
-
uri = redirect_uri("../get")
-
-
redirect_response = session.get(uri)
-
body = json_body(redirect_response)
-
assert body.key?("url"), "url should be set"
-
assert body["url"] == redirect_location, "url should have been the given redirection url"
-
end
-
-
1
def test_plugin_follow_redirects_default_max_redirects
-
session = HTTPX.plugin(:follow_redirects)
-
-
response = session.get(max_redirect_uri(3))
-
verify_status(response, 200)
-
-
response = session.get(max_redirect_uri(4))
-
verify_status(response, 302)
-
end
-
-
1
def test_plugin_follow_redirects_max_redirects
-
session = HTTPX.plugin(:follow_redirects)
-
-
response = session.max_redirects(1).get(max_redirect_uri(1))
-
verify_status(response, 200)
-
-
response = session.max_redirects(1).get(max_redirect_uri(2))
-
verify_status(response, 302)
-
end
-
-
1
def test_plugin_follow_redirects_retry_after
-
session = HTTPX.plugin(SessionWithMockResponse, mock_status: 302, mock_headers: { "retry-after" => "2" }).plugin(:follow_redirects)
-
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
response = session.get(max_redirect_uri(1))
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
-
verify_status(response, 200)
-
-
total_time = after_time - before_time
-
assert total_time >= 2, "request didn't take as expected to redirect (#{total_time} secs)"
-
end
-
-
1
def test_plugin_follow_redirects_retry_after_with_request_timeout
-
session = HTTPX.plugin(SessionWithMockResponse, mock_status: 302, mock_headers: { "retry-after" => "2" }).plugin(:follow_redirects)
-
-
timeout_response = session.get(max_redirect_uri(2), timeout: { request_timeout: 1 })
-
verify_error_response(timeout_response, HTTPX::RequestTimeoutError)
-
end
-
-
1
def test_plugin_follow_redirects_total_request_timeout_across_redirects
-
uri = max_redirect_uri(20) # high enough that the timeout should apply
-
session = HTTPX.plugin(RequestInspector)
-
.plugin(SessionWithMockResponse, mock_tries: 4, mock_status: 302, mock_headers: { "retry-after" => "2" })
-
.plugin(:follow_redirects)
-
.max_redirects(200)
-
.with(timeout: { total_request_timeout: 8 })
-
response = session.get(uri)
-
verify_error_response(response, HTTPX::TotalRequestTimeoutError)
-
assert session.total_responses.size > 2, "not enough redirections happening"
-
end
-
-
1
def test_plugin_follow_insecure_no_insecure_downgrade
-
return unless origin.start_with?("https")
-
-
session = HTTPX.plugin(:follow_redirects).max_redirects(1)
-
response = session.get(insecure_redirect_uri)
-
verify_error_response(response)
-
-
insecure_session = HTTPX.plugin(:follow_redirects)
-
.max_redirects(1)
-
.with(follow_insecure_redirects: true)
-
insecure_response = insecure_session.get(insecure_redirect_uri)
-
verify_status(insecure_response, 200)
-
-
assert insecure_response.is_a?(HTTPX::Response),
-
"request should follow insecure URLs (instead: #{insecure_response.status})"
-
end
-
-
1
def test_plugin_follow_redirects_removes_authorization_header
-
return unless origin.start_with?("http://")
-
-
session = HTTPX.plugin(:follow_redirects).with(headers: { "authorization" => "Bearer SECRET" })
-
-
# response = session.get(max_redirect_uri(1))
-
# verify_status(response, 200)
-
# body = json_body(response)
-
# assert body["headers"].key?("Authorization")
-
-
response = session.get(redirect_uri("#{httpbin_no_proxy}/get"))
-
verify_status(response, 200)
-
body = json_body(response)
-
assert !body["headers"].key?("Authorization")
-
-
response = session.with(allow_auth_to_other_origins: true).get(redirect_uri("#{httpbin_no_proxy}/get"))
-
verify_status(response, 200)
-
body = json_body(response)
-
assert body["headers"].key?("Authorization")
-
end
-
-
1
def test_plugin_follow_redirects_redirect_on
-
session = HTTPX.plugin(:follow_redirects).with(redirect_on: ->(location_uri) { !location_uri.path.end_with?("1") })
-
redirect_response = session.get(max_redirect_uri(3))
-
-
verify_status(redirect_response, 302)
-
verify_header(redirect_response.headers, "location", "/relative-redirect/1")
-
end
-
-
1
private
-
-
1
def redirect_uri(redirect_uri = redirect_location)
-
build_uri("/redirect-to?url=#{redirect_uri}")
-
end
-
-
1
def redirect_no_follow_uri
-
build_uri("/cache") # 304
-
end
-
-
1
def max_redirect_uri(n)
-
build_uri("/redirect/#{n}")
-
end
-
-
1
def insecure_redirect_uri
-
build_uri("/redirect-to?url=http://www.google.com")
-
end
-
-
1
def redirect_location
-
build_uri("/get")
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module GRPC
-
1
include GRPCHelpers
-
-
1
def test_plugin_grpc_stub_rpc_defines_snake_case_methods
-
server_port = run_rpc(TestService)
-
grpc = grpc_plugin
-
-
# build service
-
stub = grpc.build_stub("localhost:#{server_port}")
-
-
sv = stub.rpc(:aCamelCaseRpc, EchoMsg, EchoMsg, marshal_method: :marshal, unmarshal_method: :unmarshal)
-
assert sv.respond_to? :a_camel_case_rpc
-
assert sv.respond_to? :aCamelCaseRpc
-
end
-
-
1
def test_plugin_grpc_unary_plain_bytestreams
-
no_marshal = proc { |x| x }
-
-
server_port = run_request_response("a_reply", OK, marshal: no_marshal) do |call|
-
assert call.remote_read == "a_request"
-
assert call.metadata["k1"] == "v1"
-
assert call.metadata["k2"] == "v2"
-
end
-
-
grpc = grpc_plugin
-
# build service
-
stub = grpc.build_stub("localhost:#{server_port}")
-
result = stub.execute("an_rpc_method", "a_request", metadata: { k1: "v1", k2: "v2" })
-
-
assert result.to_s == "a_reply"
-
end
-
-
1
def test_plugin_grpc_call_credentials
-
return unless origin.start_with?("https")
-
-
call_credentials = -> { { "k1" => "updated-k1" } }
-
no_marshal = proc { |x| x }
-
-
server_port = run_request_response("a_reply", OK, marshal: no_marshal) do |call|
-
assert call.remote_read == "a_request"
-
assert call.metadata["k1"] == "updated-k1"
-
assert call.metadata["k2"] == "v2"
-
end
-
-
grpc = grpc_plugin
-
# build service
-
stub = grpc.with_call_credentials(call_credentials).build_stub("localhost:#{server_port}")
-
result = stub.execute("an_rpc_method", "a_request", metadata: { k1: "v1", k2: "v2" })
-
-
assert result.to_s == "a_reply"
-
end
-
-
1
def test_plugin_grpc_compressed_request
-
no_marshal = proc { |x| x }
-
-
server_port = run_request_response("a_reply", OK, marshal: no_marshal) do |call|
-
# assert call.metadata["grpc-encoding"] == "gzip", "request wasn't compressed"
-
# TODO: find a way to test if request payload was indeed compressed
-
assert call.remote_read == "A" * 2000
-
end
-
-
grpc = grpc_plugin
-
# build service
-
stub = grpc.build_stub("localhost:#{server_port}", compression: "gzip")
-
result = stub.execute("an_rpc_method", "A" * 2000)
-
-
assert result.to_s == "a_reply"
-
end
-
-
1
def test_plugin_grpc_compressed_response
-
no_marshal = proc { |x| x }
-
-
server_port = run_request_response("A" * 2000, OK, marshal: no_marshal,
-
server_initial_md: { "grpc-internal-encoding-request" => "gzip" }) do |call|
-
assert call.remote_read == "a_request"
-
end
-
-
grpc = grpc_plugin
-
# build service
-
stub = grpc.build_stub("localhost:#{server_port}")
-
result = stub.execute("an_rpc_method", "a_request")
-
-
assert result.to_s == "A" * 2000
-
end
-
-
# Cancellation on error
-
-
1
def test_plugin_grpc_deadline_exceeded
-
no_marshal = proc { |x| x }
-
-
server_port = run_request_response("a_reply", OK, marshal: no_marshal) do |call|
-
sleep(3)
-
assert call.remote_read == "a_request"
-
end
-
-
grpc = grpc_plugin
-
# build service
-
stub = grpc.build_stub("localhost:#{server_port}")
-
-
error = assert_raises(HTTPX::GRPCError) { stub.execute("an_rpc_method", "a request", deadline: 2).to_s }
-
assert error.status == ::GRPC::Core::StatusCodes::DEADLINE_EXCEEDED
-
end
-
-
1
def test_plugin_grpc_cancellation_on_client_error
-
no_marshal = proc { |x| x }
-
-
input = Enumerator.new do |_y|
-
# y << "a_request"
-
raise "oh crap"
-
end
-
-
server_port = run_request_response("a_reply", OK, marshal: no_marshal) do |call|
-
# not supposed to arrive here
-
begin
-
call.remote_read
-
rescue StandardError
-
nil
-
end
-
end
-
-
grpc = grpc_plugin
-
# build service
-
stub = grpc.build_stub("localhost:#{server_port}")
-
-
error = assert_raises(HTTPX::Error) { stub.execute("an_rpc_method", input).to_s }
-
assert error.message.include?("oh crap")
-
end
-
-
1
def test_plugin_grpc_cancellation_on_server_error
-
server_port = run_rpc(TestService)
-
-
grpc = grpc_plugin
-
-
# build service
-
test_service_stub = grpc.build_stub("localhost:#{server_port}", service: TestService)
-
error = assert_raises(HTTPX::GRPCError) { test_service_stub.a_cancellable_rpc(EchoMsg.new(msg: "ping")).to_s }
-
-
assert error.status == 1
-
assert error.details == "dump"
-
end
-
-
1
def test_plugin_grpc_unary_protobuf
-
server_port = run_rpc(TestService)
-
-
grpc = grpc_plugin
-
-
# build service
-
test_service_stub = grpc.build_stub("localhost:#{server_port}", service: TestService)
-
echo_response = test_service_stub.an_rpc(EchoMsg.new(msg: "ping"))
-
-
assert echo_response.msg == "ping"
-
assert echo_response.trailing_metadata["grpc-message"] == "OK"
-
end
-
-
1
def test_plugin_grpc_client_stream_protobuf
-
server_port = run_rpc(TestService)
-
-
grpc = grpc_plugin
-
-
# build service
-
test_service_stub = grpc.build_stub("localhost:#{server_port}", service: TestService)
-
-
input = [EchoMsg.new(msg: "ping"), EchoMsg.new(msg: "ping")]
-
response = test_service_stub.a_client_streaming_rpc(input)
-
-
assert response.msg == "client stream pong"
-
assert response.trailing_metadata["grpc-message"] == "OK"
-
end
-
-
1
def test_plugin_grpc_server_stream_protobuf
-
server_port = run_rpc(TestService)
-
-
grpc = grpc_plugin
-
-
# build service
-
test_service_stub = grpc.build_stub("localhost:#{server_port}", service: TestService)
-
streaming_response = test_service_stub.a_server_streaming_rpc(EchoMsg.new(msg: "ping"))
-
-
assert streaming_response.respond_to?(:each)
-
assert streaming_response.trailing_metadata.nil?
-
-
echo_responses = streaming_response.each.to_a
-
assert echo_responses.map(&:msg) == ["server stream pong", "server stream pong"]
-
assert streaming_response.trailing_metadata["grpc-message"] == "OK"
-
end
-
-
1
def test_plugin_grpc_bidi_stream_protobuf
-
server_port = run_rpc(TestService)
-
-
grpc = grpc_plugin
-
-
# build service
-
test_service_stub = grpc.build_stub("localhost:#{server_port}", service: TestService)
-
input = [EchoMsg.new(msg: "ping"), EchoMsg.new(msg: "ping")]
-
streaming_response = test_service_stub.a_bidi_rpc(input)
-
-
assert streaming_response.respond_to?(:each)
-
assert streaming_response.trailing_metadata.nil?
-
-
echo_responses = streaming_response.each.to_a
-
assert echo_responses.map(&:msg) == ["bidi pong", "bidi pong"]
-
assert streaming_response.trailing_metadata["grpc-message"] == "OK"
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module H2C
-
1
def test_plugin_h2c
-
HTTPX.plugin(SessionWithPool).plugin(:h2c).wrap do |session|
-
uri = build_uri("/get")
-
-
request = session.build_request("GET", uri)
-
request2 = session.build_request("GET", uri)
-
response = session.request(request)
-
verify_status(response, 200)
-
assert response.version == "2.0", "http h2c requests should be in HTTP/2"
-
response.close
-
# verifies that first request was used to upgrade the connection
-
verify_header(request.headers, "connection", "upgrade, http2-settings")
-
-
response = session.request(request2)
-
verify_status(response, 200)
-
assert response.version == "2.0", "http h2c requests should be in HTTP/2"
-
response.close
-
# verifies that first request was used to upgrade the connection
-
verify_no_header(request2.headers, "connection")
-
end
-
end
-
-
1
def test_plugin_h2c_multiple
-
session = HTTPX.plugin(SessionWithPool).plugin(:h2c)
-
uri = build_uri("/get")
-
responses = session.get(uri, uri, uri)
-
responses.each do |response|
-
verify_status(response, 200)
-
assert response.version == "2.0", "http h2c requests should be in HTTP/2"
-
response.close
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module OAuth
-
1
def test_plugin_oauth_oauth_session
-
with_oauth_metadata do |server|
-
# from options
-
oauth_session = HTTPX.plugin(:oauth).with_oauth_options(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
scope: "all"
-
).send(:oauth_session)
-
-
assert oauth_session.token_endpoint == "#{server.origin}/token"
-
assert oauth_session.token_endpoint_auth_method == "client_secret_basic"
-
assert oauth_session.instance_variable_get(:@grant_type) == "client_credentials"
-
assert oauth_session.instance_variable_get(:@scope) == %w[all]
-
assert oauth_session.instance_variable_get(:@audience).nil?
-
-
# with audience
-
oauth_session = HTTPX.plugin(:oauth).with_oauth_options(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
scope: "all",
-
audience: "audience"
-
).send(:oauth_session)
-
-
assert oauth_session.token_endpoint.to_s == "#{server.origin}/token"
-
assert oauth_session.token_endpoint_auth_method == "client_secret_basic"
-
assert oauth_session.instance_variable_get(:@grant_type) == "client_credentials"
-
assert oauth_session.instance_variable_get(:@scope) == %w[all]
-
assert oauth_session.instance_variable_get(:@audience) == "audience"
-
-
# from options, pointing to refresh
-
oauth_session = HTTPX.plugin(:oauth).with_oauth_options(
-
issuer: "https://smthelse",
-
token_endpoint_auth_method: "client_secret_post",
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
refresh_token: "REFRESH_TOKEN", access_token: "TOKEN",
-
scope: %w[foo bar]
-
).send(:oauth_session)
-
assert oauth_session.token_endpoint.to_s == "https://smthelse/token"
-
assert oauth_session.token_endpoint_auth_method == "client_secret_post"
-
assert oauth_session.instance_variable_get(:@grant_type) == "refresh_token"
-
assert oauth_session.instance_variable_get(:@scope) == %w[foo bar]
-
-
# from oauth server metadata url
-
session = HTTPX.plugin(:oauth).with_oauth_options(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
)
-
oauth_session = session.send(:oauth_session)
-
oauth_session.send(:load, session)
-
-
assert oauth_session.token_endpoint.to_s == "#{server.origin}/token"
-
assert oauth_session.token_endpoint_auth_method == "client_secret_basic"
-
assert oauth_session.instance_variable_get(:@grant_type) == "client_credentials"
-
assert oauth_session.instance_variable_get(:@scope) == %w[openid profile email address phone offline_access]
-
-
# from hash
-
HTTPX.plugin(:oauth).with_oauth_options(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
scope: "all"
-
).send(:oauth_session)
-
assert oauth_session.token_endpoint.to_s == "#{server.origin}/token"
-
assert oauth_session.token_endpoint_auth_method == "client_secret_basic"
-
assert oauth_session.instance_variable_get(:@grant_type) == "client_credentials"
-
assert oauth_session.instance_variable_get(:@scope) == %w[openid profile email address phone offline_access]
-
-
assert_raises(HTTPX::Error) do
-
HTTPX.plugin(:oauth).with_oauth_options(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
token_endpoint_auth_method: "unsupported"
-
)
-
end
-
-
assert_raises(HTTPX::Error) do
-
HTTPX.plugin(:oauth).with_oauth_options(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
grant_type: "implicit_grant" # not supported
-
)
-
end
-
-
assert_raises(ArgumentError) do
-
HTTPX.plugin(:oauth).with_oauth_options("wrong")
-
end
-
end
-
end
-
-
1
def test_plugin_oauth_access_token_audience
-
with_oauth_metadata do |server|
-
http = HTTPX.plugin(
-
:oauth,
-
oauth_options: {
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
scope: "all"
-
}
-
)
-
http_aud = http.with_oauth_options(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
scope: "all", audience: "audience"
-
)
-
-
access_token = http.send(:oauth_session).fetch_access_token(http)
-
aud_access_token = http_aud.send(:oauth_session).fetch_access_token(http_aud)
-
-
assert access_token == "CLIENT-CREDS-AUTH"
-
assert aud_access_token == "CLIENT-CREDS-AUTH-audience"
-
end
-
end
-
-
1
def test_plugin_oauth_client_credentials
-
with_oauth_metadata do |server|
-
session = HTTPX.plugin(
-
:oauth, oauth_options: {
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET", scope: "all"
-
}
-
)
-
-
client_creds_uri = build_uri("/client-credentials-authed", server.origin)
-
-
response = HTTPX.get(client_creds_uri)
-
verify_status(response, 401)
-
-
response = session.get(client_creds_uri)
-
verify_status(response, 200)
-
end
-
end
-
-
1
def test_plugin_oauth_refresh_oauth_tokens
-
with_oauth_metadata do |server|
-
session = HTTPX.plugin(
-
:oauth, oauth_options: {
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET", scope: "all"
-
}
-
)
-
oauth_session = session.send(:oauth_session)
-
assert oauth_session.access_token.nil?
-
session.refresh_oauth_tokens!
-
assert !oauth_session.access_token.nil?
-
end
-
end
-
-
1
def test_plugin_oauth_expires_in
-
with_oauth_metadata(expires_in: 1) do |server|
-
session = HTTPX.plugin(
-
:oauth,
-
oauth_options: {
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET", scope: "all"
-
}
-
)
-
oauth_session = session.send(:oauth_session)
-
token = oauth_session.fetch_access_token(session)
-
assert token == oauth_session.access_token
-
sleep(2)
-
assert oauth_session.access_token.nil?
-
end
-
end
-
-
1
def test_plugin_oauth_retries_refresh_token_on_retry
-
with_oauth_metadata do |server|
-
session = HTTPX.plugin(:retries).plugin(
-
:oauth,
-
oauth_options: {
-
issuer: server.origin,
-
token_endpoint_auth_method: "client_secret_post",
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
refresh_token: "REFRESH_TOKEN", access_token: "TOKEN", scope: %w[foo bar]
-
}
-
)
-
-
refresh_token_uri = build_uri("/refresh-token-authed", server.origin)
-
-
response = HTTPX.get(refresh_token_uri)
-
verify_status(response, 401)
-
-
response = session.get(refresh_token_uri)
-
verify_status(response, 200)
-
end
-
end
-
-
1
def test_plugin_oauth_deprecated_oauth_session_option
-
with_oauth_metadata do |server|
-
oauth_session = nil
-
assert_output(nil, /DEPRECATION WARNING: option `:oauth_session` is deprecated/) do
-
# from options
-
oauth_session = HTTPX.plugin(:oauth).with_oauth_session(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
scope: "all"
-
).send(:oauth_session)
-
end
-
-
assert oauth_session.token_endpoint == "#{server.origin}/token"
-
assert oauth_session.token_endpoint_auth_method == "client_secret_basic"
-
assert oauth_session.instance_variable_get(:@grant_type) == "client_credentials"
-
assert oauth_session.instance_variable_get(:@scope) == %w[all]
-
assert oauth_session.instance_variable_get(:@audience).nil?
-
end
-
end
-
-
1
def test_plugin_oauth_deprecated_oauth_auth
-
with_oauth_metadata do |server|
-
oauth_session = nil
-
assert_output(nil, /DEPRECATION WARNING: `oauth_auth` is deprecated/) do
-
# from options
-
oauth_session = HTTPX.plugin(:oauth).oauth_auth(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
scope: "all"
-
).send(:oauth_session)
-
end
-
-
assert oauth_session.token_endpoint == "#{server.origin}/token"
-
assert oauth_session.token_endpoint_auth_method == "client_secret_basic"
-
assert oauth_session.instance_variable_get(:@grant_type) == "client_credentials"
-
assert oauth_session.instance_variable_get(:@scope) == %w[all]
-
assert oauth_session.instance_variable_get(:@audience).nil?
-
end
-
end
-
-
1
def test_plugin_oauth_deprecated_with_access_token
-
with_oauth_metadata do |server|
-
oauth_session = nil
-
assert_output(nil, /DEPRECATION WARNING: `with_access_token` is deprecated/) do
-
# from oauth server metadata url
-
session = HTTPX.plugin(:oauth).with_oauth_options(
-
issuer: server.origin,
-
client_id: "CLIENT_ID", client_secret: "SECRET",
-
)
-
oauth_session = session.with_access_token.send(:oauth_session)
-
end
-
-
assert oauth_session.token_endpoint == "#{server.origin}/token"
-
assert oauth_session.token_endpoint_auth_method == "client_secret_basic"
-
assert oauth_session.instance_variable_get(:@grant_type) == "client_credentials"
-
assert oauth_session.instance_variable_get(:@scope) == %w[openid profile email address phone offline_access]
-
assert oauth_session.access_token == "CLIENT-CREDS-AUTH"
-
end
-
end
-
-
1
private
-
-
1
def with_oauth_metadata(metadata = {}, **kwargs)
-
start_test_servlet(OAuthProviderServer, **kwargs) do |server|
-
server.metadata.merge!(metadata)
-
yield server
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Persistent
-
1
def test_persistent
-
uri = build_uri("/get")
-
-
non_persistent_session = HTTPX.plugin(SessionWithPool)
-
response = non_persistent_session.get(uri)
-
verify_status(response, 200)
-
assert non_persistent_session.connections.size == 1, "should have been just 1"
-
assert non_persistent_session.connections.one?(&:closed?), "should have been no open connections"
-
-
persistent_session = non_persistent_session.plugin(:persistent)
-
response = persistent_session.get(uri)
-
verify_status(response, 200)
-
assert persistent_session.connections.size == 1, "should have been just 1"
-
assert persistent_session.connections.none?(&:closed?), "should have been open connections"
-
-
persistent_session.close
-
assert persistent_session.connections.one?(&:closed?), "should have been no connections"
-
end
-
-
1
def test_persistent_options
-
retry_persistent_session = HTTPX.plugin(:persistent).plugin(:retries, max_retries: 4)
-
options = retry_persistent_session.send(:default_options)
-
assert options.max_retries == 4
-
assert options.persistent
-
-
persistent_retry_session = HTTPX.plugin(:retries, max_retries: 4).plugin(:persistent)
-
options = persistent_retry_session.send(:default_options)
-
assert options.max_retries == 4
-
assert options.persistent
-
end
-
-
1
def test_plugin_persistent_does_not_retry_timeout_requests
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
persistent_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:persistent)
-
.with(timeout: { request_timeout: 3 })
-
retries_response = persistent_session.get(build_uri("/delay/10"))
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
total_time = after_time - before_time
-
-
verify_error_response(retries_response, HTTPX::RequestTimeoutError)
-
assert persistent_session.calls.zero?, "expect request to not be resent (was #{persistent_session.calls})"
-
verify_execution_delta(3, total_time, 1)
-
end
-
-
1
def test_plugin_persistent_does_not_retry_change_requests_on_timeouts
-
check_error = ->(response) { response.is_a?(HTTPX::ErrorResponse) || response.status == 405 }
-
persistent_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:persistent, retry_on: check_error) # because CI
-
.with(timeout: { request_timeout: 3 })
-
-
response = persistent_session.post(build_uri("/delay/10"), body: ["a" * 1024])
-
assert check_error[response]
-
assert persistent_session.calls.zero?, "expect request to be built 0 times (was #{persistent_session.calls})"
-
end
-
-
1
def test_plugin_persistent_does_not_retry_change_requests_on_keep_alive_interval_timeouts
-
start_test_servlet(KeepAlivePongThenTimeoutSocketServer) do |server|
-
check_error = ->(response) { response.is_a?(HTTPX::ErrorResponse) || response.status == 405 }
-
persistent_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:persistent, retry_on: check_error)
-
.with(
-
ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE },
-
timeout: { keep_alive_timeout: 1, request_timeout: 2 }
-
)
-
-
response = persistent_session.post(server.origin, body: "test")
-
verify_status(response, 200)
-
assert persistent_session.calls.zero?, "expect request to be built 0 times (was #{persistent_session.calls})"
-
sleep(2)
-
response = persistent_session.post(server.origin, body: "test")
-
assert check_error[response]
-
assert persistent_session.calls == 1, "expect request to be built 1 time (was #{persistent_session.calls})"
-
end
-
end
-
-
def test_plugin_persistent_coalescing
-
coalesced_origin = "https://#{ENV["HTTPBIN_COALESCING_HOST"]}"
-
http = HTTPX.plugin(SessionWithPool).plugin(:persistent)
-
-
response1 = http.get(origin)
-
verify_status(response1, 200)
-
response2 = http.get(coalesced_origin)
-
verify_status(response2, 200)
-
# introspection time
-
connections = http.connections
-
assert connections.size == 2
-
origins = connections.map(&:origins)
-
assert origins.any? { |orgs| orgs.sort == [origin, coalesced_origin].sort },
-
"connections for #{[origin, coalesced_origin]} didn't coalesce (expected connection with both origins (#{origins}))"
-
-
assert http.pool.connections.size == 1, "coalesced connection should have been dropped"
-
assert http.pool.connections_counter == 1, "coalesced connection should not have been accounted for in the pool"
-
-
unsafe_origin = URI(origin)
-
unsafe_origin.scheme = "http"
-
response3 = http.get(unsafe_origin)
-
verify_status(response3, 200)
-
-
# introspection time
-
connections = http.connections
-
assert connections.size == 3
-
origins = connections.map(&:origins)
-
refute origins.any?([origin]),
-
"connection coalesced inexpectedly (expected connection with both origins (#{origins}))"
-
-
http.close
-
1
end if ENV.key?("HTTPBIN_COALESCING_HOST")
-
-
1
def test_persistent_with_io
-
return unless origin.start_with?("https")
-
-
io = origin_io
-
uri = build_uri("/get")
-
-
session = HTTPX.plugin(SessionWithPool).plugin(:persistent)
-
response = session.get(uri)
-
verify_status(response, 200)
-
-
assert !io.closed?, "io should have been left open"
-
connections = session.pool.connections
-
assert connections.size == 1
-
-
response = session.get(uri)
-
verify_status(response, 200)
-
-
assert !io.closed?, "io should have been left open"
-
connections = session.pool.connections
-
assert connections.size == 1
-
-
session.close
-
assert session.connections.one?(&:closed?), "should have been no connections"
-
end
-
-
1
def test_persistent_retry_http2_ping_timeout
-
return unless origin.start_with?("https")
-
-
start_test_servlet(DelayedPingServer, ping_delay: 2) do |server|
-
http = HTTPX.plugin(SessionWithPool)
-
.plugin(RequestInspector)
-
.plugin(:persistent) # implicit max_retries == 1
-
.with(timeout: { keep_alive_timeout: 0, ping_timeout: 4 }, ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
-
uri = "#{server.origin}/"
-
response = http.get(uri)
-
verify_status(response, 200)
-
response = http.get(uri)
-
verify_status(response, 200)
-
assert http.calls == 1, "expect request to be built 1 time (was #{http.calls})"
-
http.close
-
-
other_http = http.with(timeout: { ping_timeout: 1 })
-
uri = "#{server.origin}/"
-
response = other_http.get(uri)
-
verify_status(response, 200)
-
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
response = other_http.get(uri)
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
total_time = after_time - before_time
-
verify_status(response, 200)
-
assert other_http.calls == 2, "expect request to be built 2 times (was #{other_http.calls})"
-
verify_execution_delta(1, total_time, 1)
-
http.close
-
end
-
end
-
-
def test_persistent_retry_http2_goaway
-
return unless origin.start_with?("https")
-
-
start_test_servlet(KeepAlivePongThenGoawayServer) do |server|
-
http = HTTPX.plugin(SessionWithPool)
-
.plugin(RequestInspector)
-
.plugin(:persistent) # implicit max_retries == 1
-
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
-
uri = "#{server.origin}/"
-
response = http.get(uri)
-
verify_status(response, 200)
-
response = http.get(uri)
-
verify_status(response, 200)
-
assert http.calls == 2, "expect request to be built 2 times (was #{http.calls})"
-
http.close
-
end
-
1
end unless RUBY_ENGINE == "jruby"
-
-
def test_persistent_proxy_retry_http2_goaway
-
return unless origin.start_with?("https")
-
-
start_test_servlet(KeepAlivePongThenGoawayServer) do |server|
-
start_test_servlet(ProxyServer) do |proxy|
-
http = HTTPX.plugin(SessionWithPool)
-
.plugin(RequestInspector)
-
.plugin(:persistent) # implicit max_retries == 1
-
.plugin(:proxy)
-
.with(
-
proxy: { uri: proxy.origin },
-
ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE }
-
)
-
uri = "#{server.origin}/"
-
response = http.get(uri)
-
verify_status(response, 200)
-
response = http.get(uri)
-
verify_status(response, 200)
-
assert http.calls == 2, "expect request to be built 2 times (was #{http.calls})"
-
http.close
-
end
-
end
-
1
end unless RUBY_ENGINE == "jruby"
-
-
1
unless RUBY_ENGINE != "jruby" || JRUBY_VERSION >= "10.0.0.0"
-
def test_persistent_on_stream_close_get_retried
-
return unless origin.start_with?("https")
-
-
start_test_servlet(CloseAfterXThenDelaySeconds, seconds_to_close: 1) do |server|
-
uri = "#{server.origin}/"
-
-
http = HTTPX.plugin(SessionWithPool)
-
.plugin(RequestInspector)
-
.plugin(:persistent) # implicit max_retries == 1
-
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
-
-
response = http.get(uri)
-
verify_status(response, 200)
-
-
sleep 2
-
-
response = http.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
-
_, error_response, _ = http.total_responses
-
verify_error_response(error_response)
-
-
assert http.calls == 2, "expect request to be built 2 times (was #{http.calls})"
-
end
-
end
-
-
def test_persistent_on_stream_close_post_body_string_retried
-
return unless origin.start_with?("https")
-
-
start_test_servlet(CloseAfterXThenDelaySeconds, seconds_to_close: 1) do |server|
-
uri = "#{server.origin}/"
-
-
http = HTTPX.plugin(SessionWithPool)
-
.plugin(RequestInspector)
-
.plugin(:persistent) # implicit max_retries == 1
-
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
-
-
response = http.get(uri)
-
verify_status(response, 200)
-
-
sleep 2
-
-
response = http.post(uri, body: "BANGARANG")
-
verify_status(response, 200)
-
assert response.body.to_s == "BANGARANG"
-
-
_, error_response, _ = http.total_responses
-
verify_error_response(error_response)
-
-
assert http.calls == 2, "expect request to be built 2 times (was #{http.calls})"
-
end
-
end
-
-
def test_persistent_on_stream_close_post_body_stringio_retried
-
return unless origin.start_with?("https")
-
-
start_test_servlet(CloseAfterXThenDelaySeconds, seconds_to_close: 1) do |server|
-
uri = "#{server.origin}/"
-
-
http = HTTPX.plugin(SessionWithPool)
-
.plugin(RequestInspector)
-
.plugin(:persistent) # implicit max_retries == 1
-
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
-
-
response = http.get(uri)
-
verify_status(response, 200)
-
-
sleep 2
-
-
response = http.post(uri, body: StringIO.new("BANGARANG"))
-
verify_status(response, 200)
-
assert response.body.to_s == "BANGARANG"
-
-
_, error_response, _ = http.total_responses
-
verify_error_response(error_response)
-
-
assert http.calls == 2, "expect request to be built 2 times (was #{http.calls})"
-
end
-
end
-
-
def test_persistent_on_stream_close_post_body_file_retried
-
return unless origin.start_with?("https")
-
-
start_test_servlet(CloseAfterXThenDelaySeconds, seconds_to_close: 1) do |server|
-
uri = "#{server.origin}/"
-
-
http = HTTPX.plugin(SessionWithPool)
-
.plugin(RequestInspector)
-
.plugin(:persistent) # implicit max_retries == 1
-
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
-
-
rng = Random.new(42)
-
req_body = Tempfile.new("httpx-body", binmode: true)
-
-
begin
-
response = http.get(uri)
-
verify_status(response, 200)
-
-
sleep 2
-
-
req_body.write(rng.bytes(16_385))
-
req_body.rewind
-
-
response = http.post(uri, body: req_body)
-
verify_status(response, 200)
-
assert response.body.bytesize == 16_385
-
-
_, error_response, _ = http.total_responses
-
verify_error_response(error_response)
-
-
assert http.calls == 2, "expect request to be built 2 times (was #{http.calls})"
-
ensure
-
req_body.close
-
req_body.unlink
-
end
-
end
-
end
-
-
def test_persistent_on_stream_close_post_body_multipart_retried
-
return unless origin.start_with?("https")
-
-
start_test_servlet(CloseAfterXThenDelaySeconds, seconds_to_close: 1) do |server|
-
uri = "#{server.origin}/"
-
-
http = HTTPX.plugin(SessionWithPool)
-
.plugin(RequestInspector)
-
.plugin(:persistent) # implicit max_retries == 1
-
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
-
-
begin
-
response = http.get(uri)
-
verify_status(response, 200)
-
-
sleep 2
-
-
file = File.new(fixture_file_path)
-
response = http.post(uri, form: [
-
["image1", file],
-
])
-
verify_status(response, 200)
-
assert response.body.bytesize > File.size(fixture_file_path)
-
-
_, error_response, _ = http.total_responses
-
verify_error_response(error_response)
-
-
assert http.calls == 2, "expect request to be built 2 times (was #{http.calls})"
-
end
-
end
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "resolv"
-
-
1
module Requests
-
1
module Plugins
-
1
module Proxy
-
1
include ProxyHelper
-
-
1
using HTTPX::URIExtensions
-
-
1
RESOLVER = Resolv::DNS.new
-
-
1
def test_plugin_no_proxy_defined
-
http = HTTPX.plugin(:proxy)
-
uri = build_uri("/get")
-
res = http.with_proxy(uri: []).get(uri)
-
verify_error_response(res, HTTPX::ProxyError)
-
end
-
-
1
def test_plugin_http_http_proxy
-
return unless origin.start_with?("http://")
-
-
session = HTTPX.plugin(:proxy, fallback_protocol: "http/1.1").plugin(ProxyResponseDetector).with_proxy(uri: http_proxy)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
1
def test_plugin_http_no_proxy
-
return unless origin.start_with?("http://")
-
-
session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: http_proxy, no_proxy: [httpbin_no_proxy.host])
-
-
# proxy
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
-
# no proxy
-
no_proxy_uri = build_uri("/get", httpbin_no_proxy)
-
no_proxy_response = session.get(no_proxy_uri)
-
verify_status(no_proxy_response, 200)
-
verify_body_length(no_proxy_response)
-
assert !no_proxy_response.proxied?
-
end
-
-
1
def test_plugin_http_h2_proxy
-
return unless origin.start_with?("http://")
-
-
session = HTTPX.plugin(:proxy, fallback_protocol: "h2").plugin(ProxyResponseDetector).with_proxy(uri: http2_proxy)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
1
def test_plugin_https_connect_http1_proxy
-
# return unless origin.start_with?("https://")
-
session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: http_proxy)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
# TODO: uncomment when supporting H2 CONNECT
-
# def test_plugin_https_connect_h2_proxy
-
# return unless origin.start_with?("https://")
-
-
# session = HTTPX.plugin(:proxy, alpn_protocols: %w[h2]).with_proxy(uri: http2_proxy)
-
# uri = build_uri("/get")
-
# response = session.get(uri)
-
# verify_status(response, 200)
-
# verify_body_length(response)
-
# end
-
-
1
def test_plugin_http_next_proxy
-
session = HTTPX.plugin(SessionWithPool)
-
.plugin(:proxy)
-
.plugin(ProxyResponseDetector)
-
.with_proxy(uri: ["http://unavailable-proxy", *http_proxy])
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
1
def test_plugin_http_proxy_auth_options
-
auth_proxy = URI(http_proxy.first)
-
return unless auth_proxy.user
-
-
user = auth_proxy.user
-
pass = auth_proxy.password
-
auth_proxy.user = nil
-
auth_proxy.password = nil
-
-
session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(
-
uri: auth_proxy.to_s,
-
username: user,
-
password: pass
-
)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
1
def test_plugin_http_proxy_auth_error
-
no_auth_proxy = URI(http_proxy.first)
-
return unless no_auth_proxy.user
-
-
no_auth_proxy.user = nil
-
no_auth_proxy.password = nil
-
-
session = HTTPX.plugin(:proxy).with_proxy(uri: no_auth_proxy.to_s)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 407)
-
end
-
-
1
def test_plugin_http_proxy_basic_auth_wrong_error
-
wrong_auth_proxy = URI(http_proxy.first)
-
return unless wrong_auth_proxy.user
-
-
wrong_auth_proxy.password = "wrongpass"
-
-
session = HTTPX.plugin(:proxy).with_proxy(uri: wrong_auth_proxy.to_s)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 407)
-
end
-
-
1
def test_plugin_http_proxy_digest_auth
-
auth_proxy = URI(http_proxy.first)
-
return unless auth_proxy.user
-
-
user = auth_proxy.user
-
pass = auth_proxy.password
-
auth_proxy.user = nil
-
auth_proxy.password = nil
-
-
session = HTTPX.plugin(:proxy)
-
.plugin(ProxyResponseDetector)
-
.with_proxy_digest_auth(
-
uri: auth_proxy.to_s,
-
username: user,
-
password: pass
-
)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
def test_plugin_http_proxy_connection_coalescing
-
return unless origin.start_with?("https://")
-
-
coalesced_origin = "https://#{ENV["HTTPBIN_COALESCING_HOST"]}"
-
HTTPX.plugin(:proxy).with_proxy(uri: http_proxy).plugin(SessionWithPool).wrap do |http|
-
response1 = http.get(origin)
-
verify_status(response1, 200)
-
response2 = http.get(coalesced_origin)
-
verify_status(response2, 200)
-
# introspection time
-
connections = http.connections
-
origins = connections.map(&:origins)
-
assert origins.any? { |orgs| orgs.sort == [origin, coalesced_origin].sort },
-
"connections for #{[origin, coalesced_origin]} didn't coalesce (expected connection with both origins (#{origins}))"
-
-
unsafe_origin = URI(origin)
-
unsafe_origin.scheme = "http"
-
response3 = http.get(unsafe_origin)
-
verify_status(response3, 200)
-
-
# introspection time
-
connections = http.connections
-
origins = connections.map(&:origins)
-
refute origins.any?([origin]),
-
"connection coalesced inexpectedly (expected connection with both origins (#{origins}))"
-
end
-
1
end if ENV.key?("HTTPBIN_COALESCING_HOST")
-
-
1
def test_plugin_http_proxy_redirect_305
-
return unless origin.start_with?("http://")
-
-
start_test_servlet(ProxyServer) do |proxy|
-
start_test_servlet(ProxyRedirectorServer, proxy.origin) do |server|
-
session = HTTPX.plugin(:follow_redirects)
-
.plugin(:proxy)
-
.plugin(ProxyResponseDetector)
-
-
uri = "#{server.origin}/"
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.body.to_s == proxy.origin.to_s
-
end
-
end
-
end
-
-
1
def test_plugin_socks4_proxy
-
session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks4_proxy)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
1
def test_plugin_socks4_proxy_ip
-
proxy = URI(socks4_proxy.first)
-
-
# doing this bit of song and dance due to URI's CVE "fix" from 1.0.4
-
# https://github.com/ruby/uri/issues/184
-
user, _ = proxy.userinfo
-
proxy.host = Resolv.getaddress(proxy.host)
-
proxy.user = user
-
-
session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: [proxy])
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
1
def test_plugin_socks4_proxy_error
-
proxy = URI(socks4_proxy.first)
-
proxy.user = nil
-
-
session = HTTPX.plugin(:proxy).with_proxy(uri: [proxy])
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_error_response(response, HTTPX::Socks4Error)
-
end
-
-
1
def test_plugin_socks4a_proxy
-
session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks4a_proxy)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
1
def test_plugin_socks5_proxy
-
session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks5_proxy)
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
1
def test_plugin_socks5_ipv4_proxy
-
session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks5_proxy)
-
uri = URI(build_uri("/get"))
-
hostname = uri.host
-
-
ipv4 = RESOLVER.getresource(hostname, Resolv::DNS::Resource::IN::A).address.to_s
-
uri.hostname = ipv4
-
-
response = session.get(uri, headers: { "host" => uri.authority }, ssl: { hostname: hostname })
-
verify_status(response, 200)
-
verify_body_length(response)
-
assert response.proxied?
-
end
-
-
# TODO: enable when docker-compose supports ipv6 out of the box
-
# def test_plugin_socks5_ipv6_proxy
-
# session = HTTPX.plugin(:proxy).with_proxy(uri: socks5_proxy)
-
# uri = URI(build_uri("/get"))
-
# hostname = uri.host
-
-
# ipv6 = RESOLVER.getresource(hostname, Resolv::DNS::Resource::IN::AAAA).address.to_s
-
# uri.hostname = ipv6
-
-
# response = session.get(uri, headers: { "host" => uri.authority }, ssl: { hostname: hostname })
-
# verify_status(response, 200)
-
# verify_body_length(response)
-
# end
-
-
1
def test_plugin_socks5_proxy_negotiation_error
-
proxy = URI(socks5_proxy.first)
-
proxy.password = nil
-
-
session = HTTPX.plugin(:proxy).with_proxy(uri: [proxy])
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_error_response(response, /negotiation error/)
-
end
-
-
1
def test_plugin_socks5_proxy_authentication_error
-
proxy = URI(socks5_proxy.first)
-
proxy.password = "1"
-
-
session = HTTPX.plugin(:proxy).with_proxy(uri: [proxy])
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_error_response(response, /authentication error:/)
-
end
-
-
1
def test_plugin_socks5_proxy_none_error
-
start_test_servlet(Sock5WithNoneServer) do |server|
-
proxy = server.origin
-
session = HTTPX.plugin(:proxy).with_proxy(uri: [proxy])
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_error_response(response, /no supported authorization methods/)
-
end
-
end
-
-
def test_plugin_ssh_proxy
-
session = HTTPX.plugin(:"proxy/ssh")
-
.with_proxy(uri: ssh_proxy,
-
username: "root",
-
auth_methods: %w[publickey],
-
host_key: "ssh-rsa",
-
keys: %w[test/support/ssh/ssh_host_ed25519_key])
-
uri = build_uri("/get")
-
response = session.get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
1
end if ENV.key?("HTTPX_SSH_PROXY") && RUBY_ENGINE == "ruby" &&
-
# TODO: remove after https://bugs.ruby-lang.org/issues/22083 is fixed
-
RUBY_VERSION < "4.0.0"
-
-
1
def test_plugin_retries_on_proxy_error
-
start_test_servlet(Sock5WithNoneServer) do |server|
-
proxy = server.origin
-
uri = build_uri("/get")
-
session = HTTPX.plugin(RequestInspector).plugin(:proxy).plugin(:retries).with_proxy(uri: [proxy])
-
res = session.get(uri)
-
verify_error_response(res, /no supported authorization methods/)
-
assert session.calls == 3, "expect request to be built 3 times (was #{session.calls})"
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module PushPromise
-
1
def test_plugin_no_push_promise
-
html, css = HTTPX.get(push_html_uri, push_css_uri, max_concurrent_requests: 1,
-
http2_settings: { settings_enable_push: 1 })
-
verify_status(html, 200)
-
verify_status(css, 200)
-
verify_no_header(css.headers, "x-http2-push")
-
html.close
-
css.close
-
end
-
-
1
def test_plugin_push_promise_get
-
session = HTTPX.plugin(:push_promise)
-
html, css = session.get(push_html_uri, push_css_uri)
-
verify_status(html, 200)
-
verify_status(css, 200)
-
verify_header(css.headers, "x-http2-push", "1")
-
assert css.pushed?
-
html.close
-
css.close
-
end
-
-
1
def test_plugin_push_promise_concurrent
-
session = HTTPX.plugin(:push_promise).with(max_concurrent_requests: 100)
-
html, css = session.get(push_html_uri, push_css_uri)
-
verify_status(html, 200)
-
verify_status(css, 200)
-
verify_no_header(css.headers, "x-http2-push")
-
assert !css.pushed?
-
html.close
-
css.close
-
end
-
-
1
private
-
-
1
def push_origin
-
"https://nghttp2.org"
-
end
-
-
1
def push_html_uri
-
"#{push_origin}/"
-
end
-
-
1
def push_css_uri
-
"#{push_origin}/stylesheets/screen.css"
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Query
-
1
QUERY_FAILED_STATUS_CODE = ENV.key?("CI") ? 501 : 405
-
-
1
def test_plugin_query
-
session = HTTPX.plugin(:query)
-
assert session.respond_to?(:query)
-
-
uri = build_uri("/get")
-
-
response = session.query(uri)
-
verify_status(response, QUERY_FAILED_STATUS_CODE) # not implemented yet
-
-
request = response.instance_variable_get(:@request)
-
assert request.verb == "QUERY"
-
end
-
-
1
def test_plugin_retries_query_can_be_retried
-
check_error = ->(response) {
-
response.is_a?(HTTPX::ErrorResponse) || response.status == QUERY_FAILED_STATUS_CODE
-
}
-
retries_session = HTTPX.plugin(RequestInspector).plugin(:query).plugin(:retries, retry_on: check_error)
-
uri = build_uri("/get")
-
retries_response = retries_session.query(uri)
-
verify_status(retries_response, QUERY_FAILED_STATUS_CODE) # not implemented yet
-
assert retries_session.calls == 3, "expect request to be built 3 times (was #{retries_session.calls})"
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module RateLimiter
-
1
def test_plugin_rate_limiter_429
-
rate_limiter_session = HTTPX.plugin(RequestInspector)
-
.plugin(SessionWithMockResponse, mock_status: 429)
-
.plugin(:rate_limiter)
-
-
uri = build_uri("/get")
-
-
rate_limiter_session.get(uri)
-
-
verify_rated_responses(rate_limiter_session, 429)
-
end
-
-
1
def test_plugin_rate_limiter_503
-
rate_limiter_session = HTTPX.plugin(RequestInspector)
-
.plugin(SessionWithMockResponse, mock_status: 503)
-
.plugin(:rate_limiter)
-
-
uri = build_uri("/get")
-
-
rate_limiter_session.get(uri)
-
-
verify_rated_responses(rate_limiter_session, 503)
-
end
-
-
1
def test_plugin_rate_limiter_retry_after_integer
-
rate_limiter_session = HTTPX.plugin(RequestInspector)
-
.plugin(SessionWithMockResponse, mock_status: 429, mock_headers: { "retry-after" => "2" })
-
.plugin(:rate_limiter)
-
-
uri = build_uri("/get")
-
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
rate_limiter_session.get(uri)
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
-
verify_rated_responses(rate_limiter_session, 429)
-
-
total_time = after_time - before_time
-
verify_execution_delta(2, total_time, 1)
-
end
-
-
1
def test_plugin_rate_limiter_retry_after_date
-
retry_after = (Time.now + 3).httpdate
-
rate_limiter_session = HTTPX.plugin(RequestInspector)
-
.plugin(SessionWithMockResponse, mock_status: 429, mock_headers: { "retry-after" => retry_after })
-
.plugin(:rate_limiter)
-
-
uri = build_uri("/get")
-
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
rate_limiter_session.get(uri)
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
-
verify_rated_responses(rate_limiter_session, 429)
-
total_time = after_time - before_time
-
verify_execution_delta(2, total_time, 1)
-
end
-
-
1
private
-
-
1
def verify_rated_responses(session, rated_status)
-
assert session.total_responses.size == 2, "expected 2 responses(was #{session.total_responses.size})"
-
rated_response, response = session.total_responses
-
verify_status(rated_response, rated_status)
-
verify_status(response, 200)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "securerandom"
-
-
1
module Requests
-
1
module Plugins
-
1
module ResponseCache
-
1
def test_plugin_response_cache_options
-
cache_client = HTTPX.plugin(:response_cache, response_cache_store: :store)
-
assert cache_client.class.default_options.response_cache_store.is_a?(HTTPX::Plugins::Cache::Store)
-
cache_client = HTTPX.plugin(:response_cache, response_cache_store: :file_store)
-
assert cache_client.class.default_options.response_cache_store.is_a?(HTTPX::Plugins::Cache::FileStore)
-
end
-
-
1
def test_plugin_response_cache_etag
-
cache_client = HTTPX.plugin(:response_cache)
-
-
etag_uri = build_uri("/cache")
-
-
original = cache_client.get(etag_uri)
-
verify_status(original, 200)
-
assert original.instance_variable_get(:@revalidated_at).nil?
-
cached = cache_client.get(etag_uri)
-
verify_status(cached, 304)
-
-
assert original.body == cached.body
-
refute original.instance_variable_get(:@revalidated_at).nil?
-
-
cache_client.clear_response_cache
-
-
uncached = cache_client.get(etag_uri)
-
verify_status(uncached, 200)
-
assert uncached != original
-
end
-
-
1
def test_plugin_response_cache_cache_control
-
cache_client = HTTPX.plugin(:response_cache)
-
-
cache_control_uri = build_uri("/cache")
-
original = cache_client.get(cache_control_uri)
-
verify_status(original, 200)
-
assert original.instance_variable_get(:@revalidated_at).nil?
-
cached = cache_client.get(cache_control_uri)
-
verify_status(cached, 304)
-
refute original.instance_variable_get(:@revalidated_at).nil?
-
-
assert original.body == cached.body
-
end
-
-
1
def test_plugin_response_cache_do_not_cache_on_error_status
-
cache_client = HTTPX.plugin(SessionWithPool).plugin(:response_cache)
-
-
store = cache_client.instance_variable_get(:@options).response_cache_store.instance_variable_get(:@store)
-
-
response_404 = cache_client.get(build_uri("/status/404"))
-
verify_status(response_404, 404)
-
assert !store.value?(response_404)
-
-
response_410 = cache_client.get(build_uri("/status/410"))
-
verify_status(response_410, 410)
-
assert store.value?(response_410)
-
end
-
-
1
def test_plugin_response_cache_do_not_store_on_no_store_header
-
return if origin.start_with?("https")
-
-
start_test_servlet(ResponseCacheServer) do |server|
-
cache_client = HTTPX.plugin(:response_cache)
-
store = cache_client.instance_variable_get(:@options).response_cache_store.instance_variable_get(:@store)
-
-
response = cache_client.get("#{server.origin}/no-store")
-
verify_status(response, 200)
-
assert store.empty?, "request should not have been cached with no-store header"
-
end
-
end
-
-
1
def test_plugin_response_cache_return_cached_while_fresh
-
cache_client = HTTPX.plugin(SessionWithPool).plugin(:response_cache)
-
-
cache_control_uri = build_uri("/cache/2")
-
-
store = cache_client.instance_variable_get(:@options).response_cache_store.instance_variable_get(:@store)
-
-
original = cache_client.get(cache_control_uri)
-
verify_status(original, 200)
-
assert cache_client.connection_count == 1, "a request should have been made"
-
assert store.value?(original)
-
-
cached = cache_client.get(cache_control_uri)
-
verify_status(cached, 200)
-
assert cache_client.connection_count == 1, "no request should have been performed"
-
assert original.body == cached.body, "bodies should have the same value"
-
assert !original.body.eql?(cached.body), "bodies should have different references"
-
assert store.value?(original)
-
-
sleep(2)
-
after_expired = cache_client.get(cache_control_uri)
-
verify_status(after_expired, 200)
-
assert cache_client.connection_count == 2, "a conditional request should have been made"
-
assert !store.value?(original)
-
assert store.value?(after_expired)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Retries
-
1
def test_plugin_retries
-
no_retries_session = HTTPX.plugin(RequestInspector).with(timeout: { request_timeout: 3 })
-
no_retries_response = no_retries_session.get(build_uri("/delay/10"))
-
verify_error_response(no_retries_response)
-
assert no_retries_session.calls.zero?, "expected request to be retried 1 time (was #{no_retries_session.calls})"
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries)
-
.with(timeout: { request_timeout: 3 })
-
retries_response = retries_session.get(build_uri("/delay/10"))
-
verify_error_response(retries_response)
-
assert retries_session.calls == 3, "expected request to be retried 3 times (was #{retries_session.calls})"
-
end
-
-
1
def test_plugin_retries_total_request_timeout_across_attempts
-
uri = build_uri("/delay/10")
-
session = HTTPX.plugin(RequestInspector)
-
.plugin(:retries, max_retries: 3)
-
.with(timeout: { total_request_timeout: 8, read_timeout: 5 })
-
response = session.get(uri)
-
verify_error_response(response, HTTPX::TotalRequestTimeoutError)
-
assert session.total_responses.size == 2
-
end
-
-
1
def test_plugin_retries_change_requests
-
check_error = ->(response) { response.is_a?(HTTPX::ErrorResponse) || response.status == 405 }
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries, retry_on: check_error) # because CI
-
.with(timeout: { request_timeout: 3 })
-
-
retries_response = retries_session.post(build_uri("/delay/10"), body: ["a" * 1024])
-
assert check_error[retries_response]
-
assert retries_session.calls.zero?, "expected request to be built 0 times (was #{retries_session.calls})"
-
-
retries_session.reset
-
-
retries_response = retries_session.post(build_uri("/delay/10"), body: ["a" * 1024], retry_change_requests: true)
-
assert check_error[retries_response]
-
assert retries_session.calls == 3, "expected request to be retried 3 times (was #{retries_session.calls})"
-
end
-
-
1
def test_plugin_retries_multi_request
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries)
-
.with(timeout: { request_timeout: 3 })
-
.max_retries(1)
-
-
uri = build_uri("/delay/10")
-
expected = 6 # each request should retry once and fail after 3 seconds
-
-
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
responses = retries_session.get(uri, uri, uri)
-
actual = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
-
-
assert responses.size == 3
-
responses.each do |response|
-
verify_error_response(response)
-
# we're comparing against max-retries + 1, because the calls increment will happen
-
# also in the last call, where the request is not going to be retried.
-
end
-
assert retries_session.calls == 5, "expected each request to be retried 2 times (was #{retries_session.calls})"
-
-
assert_in_delta expected, actual, 2, "expected to have executed in #{expected} secs (actual: #{actual} secs)"
-
end
-
-
1
def test_plugin_retries_max_retries
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries)
-
.with(timeout: { request_timeout: 3 })
-
.max_retries(2)
-
retries_response = retries_session.get(build_uri("/delay/10"))
-
-
verify_error_response(retries_response)
-
# we're comparing against max-retries + 1, because the calls increment will happen
-
# also in the last call, where the request is not going to be retried.
-
assert retries_session.calls == 2, "expected request to be retried 2 times (was #{retries_session.calls})"
-
end
-
-
1
def test_plugin_retries_retry_on
-
retry_callback = lambda do |response|
-
response.is_a?(HTTPX::Response) && response.status == 400
-
end
-
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries, retry_on: retry_callback)
-
.with(timeout: { request_timeout: 3 })
-
.max_retries(2)
-
-
retries_response = retries_session.get(build_uri("/status/400"))
-
verify_status(retries_response, 400)
-
assert retries_session.calls == 2, "expected request to be retried for 400 status code (it was, #{retries_session.calls} times)"
-
retries_session.reset
-
-
retries_response = retries_session.get(build_uri("/status/401"))
-
verify_status(retries_response, 401)
-
assert retries_session.calls.zero?,
-
"expected request not to be retried for 401 status code (it was, #{retries_session.calls} times)"
-
retries_session.reset
-
-
retries_response = retries_session.get(build_uri("/delay/10"))
-
verify_error_response(retries_response)
-
assert retries_session.calls == 2,
-
"expected request to still be retried for regular socket errors (it was, #{retries_session.calls} times)"
-
end
-
-
1
def test_plugin_retries_retry_after
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries, retry_after: 2)
-
.with(timeout: { request_timeout: 3 })
-
.max_retries(1)
-
retries_response = retries_session.get(build_uri("/delay/10"))
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
total_time = after_time - before_time
-
-
verify_error_response(retries_response, HTTPX::RequestTimeoutError)
-
verify_execution_delta(3 + 2 + 3, total_time, 1)
-
end
-
-
1
def test_plugin_retries_retry_after_with_jitter
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries, retry_after: 2, retry_jitter: ->(_) { 1 })
-
.with(timeout: { request_timeout: 3 })
-
.max_retries(1)
-
retries_response = retries_session.get(build_uri("/delay/10"))
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
total_time = after_time - before_time
-
-
verify_error_response(retries_response, HTTPX::RequestTimeoutError)
-
verify_execution_delta(3 + 2 + 1 + 3, total_time, 1)
-
end
-
-
1
def test_plugin_retries_retry_after_exponential
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries, retry_after: :exponential_backoff)
-
.with(timeout: { request_timeout: 3 })
-
.max_retries(2)
-
retries_response = retries_session.get(build_uri("/delay/10"))
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
total_time = after_time - before_time
-
-
verify_error_response(retries_response, HTTPX::RequestTimeoutError)
-
verify_execution_delta(3 + 3 + 2 + 3 + 4, total_time, 1)
-
end
-
-
1
def test_plugin_retries_retry_after_polynomial
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries, retry_after: :polynomial_backoff)
-
.with(timeout: { request_timeout: 3 })
-
.max_retries(2)
-
retries_response = retries_session.get(build_uri("/delay/10"))
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
total_time = after_time - before_time
-
-
verify_error_response(retries_response, HTTPX::RequestTimeoutError)
-
verify_execution_delta(3 + 3 + 1 + 3 + 1, total_time, 1)
-
end
-
-
1
def test_plugin_retries_retry_after_callable
-
retries = 0
-
exponential = ->(*) { (retries += 1) * 2 }
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(:retries, retry_after: exponential)
-
.with(timeout: { request_timeout: 3 })
-
.max_retries(2)
-
retries_response = retries_session.get(build_uri("/delay/10"))
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
total_time = after_time - before_time
-
-
verify_error_response(retries_response, HTTPX::RequestTimeoutError)
-
verify_execution_delta(3 + 2 + 3 + 4 + 3, total_time, 1)
-
end
-
-
1
def test_plugin_retries_resumable
-
resumable_uri = build_uri("/range/200?chunk_size=100")
-
full_payload = HTTPX.get(resumable_uri).raise_for_status.to_s
-
-
retries_session = HTTPX
-
.plugin(RequestInspector)
-
.plugin(RequestFailAfter100Bytes)
-
.plugin(:retries)
-
.max_retries(2)
-
.with(retry_on: ->(res) {
-
res.error && res.error.message == "over 100 bytes"
-
}, window_size: 50, buffer_size: 50, http2_settings: { settings_initial_window_size: 100 })
-
retries_response = retries_session.get(resumable_uri)
-
verify_status(retries_response, 200)
-
assert retries_response.to_s == full_payload
-
-
total_responses = retries_session.total_responses
-
assert total_responses.size == 2
-
total_requests = total_responses.map { |res| res.instance_variable_get(:@request) }
-
-
assert total_requests.uniq.size == 1
-
request = total_requests.first
-
assert request.headers.key?("range")
-
assert request.headers["range"].match(/bytes=\d+-/)
-
end
-
-
# safety-check test only check if request is successfully rewinded
-
1
def test_plugin_retries_multipart_file_post
-
check_error = lambda { |response|
-
(response.is_a?(HTTPX::ErrorResponse) && response.error.is_a?(HTTPX::TimeoutError)) || response.status == 405
-
}
-
uri = build_uri("/delay/4")
-
retries_session = HTTPX.plugin(RequestInspector)
-
.plugin(:retries, max_retries: 1, retry_on: check_error) # because CI...
-
.with_timeout(request_timeout: 2)
-
retries_response = retries_session.post(uri, retry_change_requests: true, form: { image: File.new(fixture_file_path) })
-
assert check_error[retries_response], "expected #{retries_response} to be an error response"
-
assert retries_session.calls == 1, "expected request to be retried 1 time (was #{retries_session.calls})"
-
end
-
-
1
def test_plugin_retries_multipart_tempfile_post
-
check_error = lambda { |response|
-
(response.is_a?(HTTPX::ErrorResponse) && response.error.is_a?(HTTPX::TimeoutError)) || response.status == 405
-
}
-
uri = build_uri("/delay/4")
-
retries_session = HTTPX.plugin(RequestInspector)
-
.plugin(:retries, max_retries: 1, retry_on: check_error) # because CI...
-
.with_timeout(request_timeout: 2)
-
-
retries_response = nil
-
Tempfile.open do |file|
-
retries_response = retries_session.post(uri, retry_change_requests: true, form: { image: file })
-
end
-
assert check_error[retries_response], "expected #{retries_response} to be an error response"
-
assert retries_session.calls == 1, "expected request to be retried 1 time (was #{retries_session.calls})"
-
end
-
-
1
module RequestFailAfter100Bytes
-
1
class BiggerThan100Bytes < HTTPX::Error; end
-
-
1
module ResponseBodyMethods
-
1
def write(chunk)
-
val = super
-
-
raise(BiggerThan100Bytes, "over 100 bytes") if (100..199).cover?(@length)
-
-
val
-
end
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module ServerSentEvents
-
1
def test_plugin_server_sent_events
-
session = HTTPX.plugin(:server_sent_events).with(ssl: { verify_hostname: false })
-
-
start_test_servlet(SSE, tls: tls?, messages: [{ data: "test" }]) do |server|
-
uri = "#{server.origin}/"
-
-
no_sse_response = session.get(uri)
-
sse_response = session.get(uri, event_stream: true)
-
-
assert sse_response.respond_to?(:headers) # test respond_to_missing?
-
-
no_sse_headers = no_sse_response.headers
-
no_sse_headers.delete("date")
-
sse_headers = sse_response.headers
-
sse_headers.delete("date")
-
-
verify_header(no_sse_headers, "content-type", "text/plain")
-
verify_header(sse_headers, "content-type", "text/event-stream")
-
verify_header(sse_headers, "cache-control", "no-cache")
-
verify_no_header(sse_headers, "content-length")
-
end
-
end
-
-
1
def test_plugin_server_sent_events_each_message
-
messages = [
-
{ event: "test", data: "test1", id: 1 },
-
{ event: "test", data: "test2", id: 2 },
-
{ comment: "this is a comment" },
-
{ event: "test", data: "test3" },
-
]
-
session = HTTPX.plugin(:server_sent_events).with(ssl: { verify_hostname: false })
-
start_test_servlet(SSE, tls: tls?, messages: messages) do |server|
-
uri = "#{server.origin}/"
-
-
response = session.get(uri, event_stream: true)
-
messages = response.each_message.to_a
-
assert messages.size == 3, "all the messages should have been yielded"
-
assert(messages.all? { |m| m.event == "test" })
-
assert messages[0].id == "1"
-
assert messages[0].data == "test1"
-
assert messages[1].id == "2"
-
assert messages[1].data == "test2"
-
assert messages[2].id.nil?
-
assert messages[2].data == "test3"
-
end
-
end
-
-
1
def test_plugin_server_sent_events_multiple_datas_concat_single_message
-
messages = [
-
{ data: "test1" },
-
{ data: %w[test2 test3] },
-
]
-
session = HTTPX.plugin(:server_sent_events).with(ssl: { verify_hostname: false })
-
start_test_servlet(SSE, tls: tls?, messages: messages) do |server|
-
uri = "#{server.origin}/"
-
-
response = session.get(uri, event_stream: true)
-
messages = response.each_message.to_a
-
assert messages.size == 2, "all the messages should have been yielded"
-
assert messages[0].data == "test1"
-
assert messages[1].data == "test2\ntest3"
-
end
-
end
-
-
1
def test_plugin_server_sent_events_retries_last_used_id
-
return unless tls?
-
-
messages = [
-
{ data: "test1", id: 1 },
-
{ data: "test2", id: 2 },
-
{ data: "test3", id: 3 },
-
]
-
session = HTTPX.plugin(RequestInspector)
-
.plugin(:retries)
-
.plugin(:server_sent_events)
-
.with(ssl: { verify_hostname: false })
-
start_test_servlet(SSE, tls: tls?, messages: messages, close_after: 2) do |server|
-
uri = "#{server.origin}/"
-
-
response = session.get(uri, event_stream: true)
-
messages = response.each_message.to_a
-
total_requests = session.total_requests
-
-
# assert that there was an actual retry
-
assert total_requests.size == 2
-
first_request, retry_request = total_requests
-
verify_no_header(first_request.headers, "last-event-id")
-
verify_header(retry_request.headers, "last-event-id", "2")
-
-
# assert that the messages before the retry aren't repeated
-
assert messages.size == 3, "all the messages should have been yielded"
-
assert messages[0].id == "1"
-
assert messages[0].data == "test1"
-
assert messages[1].id == "2"
-
assert messages[1].data == "test2"
-
assert messages[2].id == "3"
-
assert messages[2].data == "test3"
-
end
-
end
-
-
1
def test_plugin_server_sent_events_retries_last_used_id_retry_after
-
return unless tls?
-
-
messages = [
-
{ data: "test1", id: 1, retry: 2000 },
-
{ data: "test2", id: 2, retry: 2000 },
-
{ data: "test3", id: 3, retry: 2000 },
-
]
-
session = HTTPX.plugin(RequestInspector).plugin(:retries).plugin(:server_sent_events).with(ssl: { verify_hostname: false })
-
start_test_servlet(SSE, tls: tls?, messages: messages, close_after: 2) do |server|
-
uri = "#{server.origin}/"
-
-
before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
response = session.get(uri, event_stream: true)
-
messages = response.each_message.to_a
-
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
total_time = after_time - before_time
-
total_requests = session.total_requests
-
-
# assert that there was an actual retry
-
assert total_requests.size == 2
-
first_request, retry_request = total_requests
-
verify_no_header(first_request.headers, "last-event-id")
-
verify_header(retry_request.headers, "last-event-id", "2")
-
-
verify_execution_delta(2, total_time, 1)
-
-
# assert that the messages before the retry aren't repeated
-
assert messages.size == 3, "all the messages should have been yielded"
-
assert messages[0].id == "1"
-
assert messages[0].data == "test1"
-
assert messages[1].id == "2"
-
assert messages[1].data == "test2"
-
assert messages[2].data == "test3"
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module SsrfFilter
-
1
def test_plugin_ssrf_filter_allows
-
uri = "#{scheme}nghttp2.org"
-
-
session = HTTPX.plugin(:ssrf_filter)
-
response = session.get(uri)
-
verify_status(response, 200)
-
end
-
-
1
def test_plugin_ssrf_filter_not_allowed_scheme
-
return unless origin.start_with?("http://")
-
-
session = HTTPX.plugin(:ssrf_filter, allowed_schemes: %w[https])
-
response = session.get("#{scheme}localhost/get")
-
verify_error_response(response, HTTPX::ServerSideRequestForgeryError)
-
end
-
-
1
def test_plugin_ssrf_filter_localhost
-
session = HTTPX.plugin(:ssrf_filter)
-
response = session.get("#{scheme}localhost/get")
-
verify_error_response(response, HTTPX::ServerSideRequestForgeryError)
-
response = session.get("#{scheme}google.com", addresses: %w[127.0.0.1])
-
verify_error_response(response, HTTPX::ServerSideRequestForgeryError)
-
end
-
-
1
def test_plugin_ssrf_filter_extra_unsafe_ranges
-
# Intentionally blocking IP addresses resolved from nghttp2.org, which normally works
-
session = HTTPX.plugin(:ssrf_filter, extra_unsafe_ranges: Resolv.getaddresses("nghttp2.org"))
-
response = session.get("#{scheme}nghttp2.org")
-
verify_error_response(response, HTTPX::ServerSideRequestForgeryError)
-
end
-
-
1
def test_plugin_ssrf_filter_safe_private_ranges
-
session = HTTPX.plugin(:ssrf_filter, safe_private_ranges: ["127.0.0.1", "::1"])
-
response = session.get("#{scheme}localhost/get")
-
-
# connection error means that a connection was attempted, not blocked by SSRF filtering
-
verify_error_response(response, HTTPX::ConnectionError)
-
end
-
-
1
def test_plugin_ssrf_filter_aws_metadata_endpoint
-
session = HTTPX.plugin(:ssrf_filter)
-
response = session.get("#{scheme}169.254.169.254/latest/meta-data")
-
verify_error_response(response, HTTPX::ServerSideRequestForgeryError)
-
end
-
-
def test_plugin_ssrf_filter_dns_answer_spoof
-
dns_spoof_resolver = Class.new(TestDNSResolver) do
-
def resolve(_, family)
-
family == 1 ? ["255.255.255.255"] : []
-
end
-
end
-
start_test_servlet(dns_spoof_resolver) do |spoof_dns|
-
HTTPX.plugin(SessionWithPool).plugin(:ssrf_filter).wrap do |session|
-
response = session.get("https://wqwereasdsada.xyz", resolver_options: { nameserver: [spoof_dns.nameserver], cache: false })
-
verify_error_response(response, "wqwereasdsada.xyz has no allowed IP addresses")
-
end
-
end
-
1
end unless RUBY_ENGINE == "jruby"
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Stream
-
1
include FiberSchedulerTestHelpers
-
-
1
def test_plugin_stream
-
session = HTTPX.plugin(:stream)
-
-
uri = build_uri("/get")
-
-
no_stream_response = session.get(uri)
-
stream_response = session.get(uri, stream: true)
-
-
assert no_stream_response.to_s != stream_response.to_s, "stream response should only eager load the first chunk"
-
-
assert stream_response.respond_to?(:headers) # test respond_to_missing?
-
-
no_stream_headers = no_stream_response.headers.to_h
-
no_stream_headers.delete("date")
-
stream_headers = no_stream_response.headers.to_h
-
stream_headers.delete("date")
-
-
assert no_stream_headers == stream_headers, "headers should be the same " \
-
"(h1: #{no_stream_response.headers}, " \
-
"(h2: #{stream_response.headers}) "
-
end
-
-
1
def test_plugin_stream_each
-
session = HTTPX.plugin(:stream)
-
-
response = session.get(build_uri("/stream/3"), stream: true)
-
body = response.each
-
payload = body.to_a.join
-
assert payload.lines.size == 3, "all the lines should have been yielded"
-
end
-
-
1
def test_plugin_stream_each_after_buffering_some_content
-
session = HTTPX.plugin(:stream)
-
-
response = session.get(build_uri("/stream/3"), stream: true)
-
verify_status(response, 200) # forces buffering
-
payload = response.each.to_a.join
-
assert payload.lines.size == 3, "all the lines should have been yielded"
-
end
-
-
1
def test_plugin_stream_each_line
-
session = HTTPX.plugin(:stream)
-
-
response = session.get(build_uri("/stream/3"), stream: true)
-
lines = response.each_line.with_index.map do |line, idx|
-
assert !line.end_with?("\n")
-
data = JSON.parse(line)
-
assert data["id"] == idx
-
end
-
-
assert lines.size == 3, "all the lines should have been yielded"
-
end
-
-
1
def test_plugin_stream_compressed
-
session = HTTPX.plugin(:stream)
-
-
response = session.get(build_uri("/gzip"), stream: true)
-
payload = response.each.to_a.join
-
assert response.headers["content-length"].to_i != payload.lines.sum(&:bytesize), "all the lines should have been yielded"
-
end
-
-
1
def test_plugin_stream_multiple_responses_error
-
session = HTTPX.plugin(:stream)
-
-
assert_raises(HTTPX::Error, "support only 1 response at a time") do
-
response = session.get(build_uri("/stream/2"), build_uri("/stream/3"), stream: true)
-
# force request
-
response.each_line.to_a
-
end
-
end
-
-
1
def test_plugin_stream_response_error
-
session = HTTPX.plugin(:stream)
-
-
assert_raises(HTTPX::HTTPError) do
-
response = session.get(build_uri("/status/404"), stream: true)
-
# force request
-
response.each_line.to_a
-
end
-
end
-
-
1
def test_plugin_stream_connection_error
-
session = HTTPX.with_timeout(request_timeout: 1).plugin(:stream)
-
-
assert_raises(HTTPX::TimeoutError) do
-
response = session.get(build_uri("/delay/10"), stream: true)
-
# force request
-
response.each_line.to_a
-
end
-
end
-
-
1
def test_plugin_stream_follow_redirects
-
session = HTTPX.plugin(:follow_redirects).plugin(:stream)
-
-
stream_uri = build_uri("/stream/3")
-
redirect_to_stream_uri = redirect_uri(stream_uri)
-
-
response = session.get(redirect_to_stream_uri, stream: true)
-
payload = response.each.to_a.join
-
assert payload.lines.size == 3, "all the lines should have been yielded"
-
end
-
-
def test_plugin_stream_fiber_concurrency_close_stream_before_request_starts
-
skip unless scheme == "https://"
-
-
start_test_servlet(SettingsTimeoutServer) do |server|
-
delay_uri = "#{server.origin}/"
-
session = HTTPX.plugin(:fiber_concurrency)
-
.plugin(:stream)
-
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
-
-
err = Class.new(StandardError)
-
-
with_test_fiber_scheduler do
-
err = Class.new(StandardError)
-
-
stream_response = error = nil
-
req_fiber = Fiber.schedule do
-
begin
-
stream_response = session.get(delay_uri, timeout: { settings_timeout: 20 }, stream: true)
-
stream_response.raise_for_status
-
rescue HTTPX::Error => e
-
error = e
-
stream_response.close
-
end
-
end
-
-
Fiber.schedule do
-
sleep 1
-
-
# sometimes, in CI, it takes quite long for this thread to be prioritized
-
# back, to the point where settings timeout expires before we have a chance
-
# to inspect
-
unless error.is_a?(HTTPX::SettingsTimeoutError)
-
-
assert stream_response
-
request = stream_response.request
-
assert request.state == :idle
-
assert request.response.nil?
-
stream_response.close
-
assert request.state == :idle
-
assert request.response.nil?
-
begin
-
req_fiber.raise(err) unless req_fiber.alive?
-
rescue FiberError
-
# this may happen if this thread takes too long being picked up by the scheduler, and the request timeout
-
# triggers and reactivates the request fiber, at which point we won't be able to raise an error on it, as
-
# one can't wait an error on a resuming fiber.
-
nil
-
end
-
end
-
end
-
end
-
-
assert true
-
end
-
1
end if Fiber.respond_to?(:set_scheduler) && RUBY_VERSION >= "3.1.0"
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module StreamBidi
-
1
def test_plugin_stream_bidi
-
uri = build_uri("/get")
-
session = HTTPX.plugin(:stream_bidi)
-
response = session.get(uri)
-
verify_status(response, 200)
-
end
-
-
1
def test_plugin_stream_bidi_does_not_support_non_body_request
-
uri = build_uri("/post")
-
session = HTTPX.plugin(:stream_bidi)
-
assert_raises(HTTPX::Error) do
-
session.build_request("POST", uri, json: { foo: "bar" }, stream: true)
-
end
-
end
-
-
1
def test_plugin_stream_bidi_persistent_session_close
-
session = HTTPX.with(persistent: true).plugin(:stream_bidi)
-
session.close # should not raise NoMethodError: undefined method `inflight?' for Signal
-
end
-
-
1
def test_plugin_stream_bidi_each
-
start_test_servlet(Bidi, tls: false) do |server|
-
uri = "#{server.origin}/"
-
-
start_msg = "{\"message\":\"started\"}\n"
-
pong_msg = "{\"message\":\"pong\"}\n"
-
-
session = HTTPX.plugin(:stream_bidi)
-
request = session.build_request(
-
"POST",
-
uri,
-
headers: { "content-type" => "application/x-ndjson" },
-
body: [start_msg],
-
stream: true
-
)
-
-
response = session.request(request)
-
chunks = []
-
response.each.each_with_index do |chunk, idx| # rubocop:disable Style/RedundantEach
-
if idx < 4
-
request << pong_msg
-
else
-
request.close
-
end
-
chunks << chunk
-
end
-
assert chunks.size == 5, "all the lines should have been yielded"
-
end
-
end
-
-
1
def test_plugin_stream_bidi_buffer_data_from_separate_thread
-
start_test_servlet(Bidi, tls: false) do |server|
-
uri = "#{server.origin}/"
-
q = Queue.new
-
-
start_msg = "{\"message\":\"started\"}\n"
-
pong_msg = "{\"message\":\"pong\"}\n"
-
-
session = HTTPX.plugin(:stream_bidi)
-
request = session.build_request(
-
"POST",
-
uri,
-
headers: { "content-type" => "application/x-ndjson" },
-
body: [start_msg],
-
stream: true
-
)
-
-
response = session.request(request)
-
-
th = Thread.start do
-
4.times do
-
msg = q.pop
-
request << msg
-
end
-
request.close
-
end
-
-
chunks = []
-
response.each.each_with_index do |chunk, _idx| # rubocop:disable Style/RedundantEach
-
chunks << chunk
-
q << pong_msg
-
end
-
-
th.join
-
-
assert chunks.size == 5, "all the lines should have been yielded"
-
end
-
end
-
-
1
def test_plugin_stream_bidi_reuse_persistent_connection_across_threads
-
start_test_servlet(Bidi, tls: false) do |server|
-
uri = "#{server.origin}/"
-
-
start_msg = "{\"message\":\"started\"}\n"
-
pong_msg = "{\"message\":\"pong\"}\n"
-
-
# Create persistent session (connection will be reused across threads)
-
session = HTTPX.plugin(:stream_bidi)
-
-
begin
-
# Thread A: First request (creates the connection)
-
thread_a_chunks = []
-
thread_a = Thread.start do
-
request = session.build_request(
-
"POST",
-
uri,
-
headers: { "content-type" => "application/x-ndjson" },
-
body: [start_msg],
-
stream: true
-
)
-
response = session.request(request)
-
response.each.each_with_index do |chunk, idx| # rubocop:disable Style/RedundantEach
-
if idx < 4
-
request << pong_msg
-
else
-
request.close
-
end
-
thread_a_chunks << chunk
-
end
-
end
-
thread_a.join
-
-
thread_b_chunks = []
-
thread_b = Thread.start do
-
request = session.build_request(
-
"POST",
-
uri,
-
headers: { "content-type" => "application/x-ndjson" },
-
body: [start_msg],
-
stream: true
-
)
-
response = session.request(request)
-
response.each.each_with_index do |chunk, idx| # rubocop:disable Style/RedundantEach
-
if idx < 4
-
request << pong_msg
-
else
-
request.close
-
end
-
thread_b_chunks << chunk
-
end
-
end
-
thread_b.join
-
-
# Both requests should succeed
-
assert thread_a_chunks.size == 5, "thread A should receive all chunks"
-
assert thread_b_chunks.size == 5, "thread B should receive all chunks"
-
ensure
-
session.close
-
end
-
end
-
end
-
-
1
def test_plugin_stream_bidi_retry_after_headers_sent
-
start_test_servlet(BidiFailOnce, tls: false) do |server|
-
uri = "#{server.origin}/"
-
-
start_msg = "{\"message\":\"started\"}\n"
-
-
# Use both stream_bidi and retries plugins
-
# retry_change_requests: true because POST is not idempotent
-
session = HTTPX.plugin(:stream_bidi)
-
.plugin(:retries, retry_change_requests: true, max_retries: 2)
-
-
request = session.build_request(
-
"POST",
-
uri,
-
headers: { "content-type" => "application/x-ndjson" },
-
body: [start_msg],
-
stream: true
-
)
-
-
# Close the request immediately - we just want to test that
-
# the retry doesn't crash due to @headers_sent not being reset
-
request.close
-
-
response = session.request(request)
-
-
# If the bug exists (headers_sent not reset), this will raise
-
# HTTP2::Error::InternalError or cause a deadlock
-
# With the fix, the response should be a valid StreamResponse
-
refute response.is_a?(HTTPX::ErrorResponse),
-
"expected successful response after retry, got #{response.class}: #{response.error if response.respond_to?(:error)}"
-
verify_status(response, 200)
-
end
-
end
-
-
# Tests that stream_bidi + retries works correctly when user continues
-
# to write data after a retry is triggered. This specifically tests
-
# the callback leak fix where stale :body callbacks from previous
-
# connection attempts could cause protocol errors.
-
1
def test_plugin_stream_bidi_retry_with_ongoing_writes
-
start_test_servlet(BidiFailAfterData, tls: false) do |server|
-
uri = "#{server.origin}/"
-
-
start_msg = "{\"message\":\"started\"}\n"
-
-
session = HTTPX.plugin(:stream_bidi)
-
.plugin(:retries, retry_change_requests: true, max_retries: 2)
-
-
request = session.build_request(
-
"POST",
-
uri,
-
headers: { "content-type" => "application/x-ndjson" },
-
body: [start_msg],
-
stream: true
-
)
-
-
response = session.request(request)
-
-
# Read response in a separate thread while we continue writing
-
chunks = []
-
error = nil
-
reader = Thread.start do
-
Thread.abort_on_exception = true
-
response.each do |chunk|
-
chunks << chunk
-
end
-
end
-
-
# Continue writing data - this is where the callback leak bug manifests
-
# Without the fix, stale callbacks fire and cause protocol_error
-
3.times do |i|
-
request << "{\"message\":\"update_#{i}\"}\n"
-
sleep 0.05
-
end
-
-
request.close
-
reader.join
-
-
refute error, "expected no error during response reading, got: #{error}"
-
refute response.is_a?(HTTPX::ErrorResponse), "expected successful response after retry"
-
end
-
end
-
-
# Tests that stream_bidi + retries correctly handles the case where
-
# user writes data from a separate thread while a retry happens.
-
1
def test_plugin_stream_bidi_retry_with_concurrent_writes
-
start_test_servlet(BidiFailOnce, tls: false) do |server|
-
uri = "#{server.origin}/"
-
q = Queue.new
-
-
start_msg = "{\"message\":\"started\"}\n"
-
pong_msg = "{\"message\":\"pong\"}\n"
-
-
session = HTTPX.plugin(:stream_bidi)
-
.plugin(:retries, retry_change_requests: true, max_retries: 2)
-
-
request = session.build_request(
-
"POST",
-
uri,
-
headers: { "content-type" => "application/x-ndjson" },
-
body: [start_msg],
-
stream: true
-
)
-
-
response = session.request(request)
-
-
# Writer thread - continues writing data regardless of retry
-
writer_error = nil
-
writer = Thread.start do
-
4.times do
-
msg = q.pop
-
request << msg
-
end
-
request.close
-
rescue StandardError => e
-
writer_error = e
-
end
-
-
# Read responses and signal writer to send more
-
chunks = []
-
reader_error = nil
-
begin
-
response.each do |chunk|
-
chunks << chunk
-
q << pong_msg
-
end
-
rescue StandardError => e
-
reader_error = e
-
end
-
-
writer.join(10)
-
-
refute writer_error, "writer thread should not error: #{writer_error&.class}: #{writer_error&.message}"
-
refute reader_error, "reader should not error: #{reader_error&.class}: #{reader_error&.message}"
-
refute response.is_a?(HTTPX::ErrorResponse), "expected successful response"
-
assert chunks.size >= 1, "expected to receive response chunks"
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Tracing
-
1
def test_plugin_tracing_request_callbacks
-
http = HTTPX.plugin(:tracing, tracer: test_tracer)
-
uri = build_uri("/get")
-
request = http.build_request("GET", uri)
-
response = http.request(request)
-
verify_status(response, 200)
-
assert test_tracer.started[request] == 1
-
assert test_tracer.finished[request] == 1
-
end
-
-
1
def test_plugin_tracing_multiple_tracers_propagates
-
tracer1 = TestTracer.new
-
tracer2 = TestTracer.new
-
http = HTTPX.plugin(:tracing, tracer: tracer1).with(tracer: tracer2)
-
uri = build_uri("/get")
-
request = http.build_request("GET", uri)
-
response = http.request(request)
-
verify_status(response, 200)
-
assert tracer1.started[request] == 1
-
assert tracer1.finished[request] == 1
-
assert tracer2.started[request] == 1
-
assert tracer2.finished[request] == 1
-
end
-
-
1
def test_plugin_tracing_retries_one_for_each
-
http = HTTPX.plugin(RequestInspector)
-
.plugin(:retries)
-
.plugin(:tracing, tracer: test_tracer)
-
.with(timeout: { request_timeout: 3 })
-
request = http.build_request("GET", build_uri("/delay/10"))
-
retries_response = http.request(request)
-
verify_error_response(retries_response)
-
assert http.calls == 3, "expect request to be retried 3 times (was #{http.calls})"
-
-
assert test_tracer.started[request] == 4
-
assert test_tracer.finished[request] == 4
-
assert test_tracer.reset_times[request].size == 3
-
test_tracer.reset_times[request].each do |time|
-
assert_in_delta(3, time, 3, "expected all requests to have taken 3 seconds")
-
end
-
end
-
-
1
def test_plugin_tracing_retries_with_delayed_ping
-
start_test_servlet(DelayedPingServer, ping_delay: 2) do |server|
-
uri = "#{server.origin}/"
-
HTTPX.plugin(RequestInspector)
-
.plugin(:retries)
-
.plugin(:tracing, tracer: test_tracer)
-
.with(timeout: { keep_alive_timeout: 1 }, ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE,
-
verify_hostname: false }).wrap do |http|
-
response1 = http.get(uri)
-
sleep 2
-
response2 = http.get(uri)
-
sleep 2
-
response3 = http.get(uri)
-
-
verify_status(response1, 200)
-
verify_status(response2, 200)
-
verify_status(response3, 200)
-
-
assert test_tracer.total_times.size == 3
-
test_tracer.total_times.each_value.with_index do |times, idx|
-
next unless idx.positive?
-
-
assert_in_delta(2, times.first, 2, "expected all requests to have taken 2 seconds to ping")
-
end
-
end
-
end
-
end
-
-
1
def test_plugin_tracing_merge_tracers
-
tracer1 = TestTracer.new
-
tracer2 = TestTracer.new
-
tracer3 = TestTracer.new
-
-
http1 = HTTPX.plugin(:tracing, tracer: tracer1)
-
-
def http1.options
-
@options
-
end
-
-
assert http1.options.tracer.is_a?(TestTracer)
-
assert http1.options.tracer == tracer1
-
-
http2 = http1.with(tracer: tracer2)
-
def http2.options
-
@options
-
end
-
assert !http2.options.tracer.is_a?(TestTracer)
-
assert http2.options.tracer.send(:tracers) == [tracer1, tracer2]
-
-
http3 = http2.with(tracer: tracer3)
-
def http3.options
-
@options
-
end
-
assert !http3.options.tracer.is_a?(TestTracer)
-
assert http3.options.tracer.send(:tracers) == [tracer1, tracer2, tracer3]
-
end
-
-
1
private
-
-
1
def test_tracer
-
@test_tracer ||= TestTracer.new
-
end
-
-
1
class TestTracer
-
1
attr_reader :requests, :started, :finished, :errored, :reset_times, :total_times
-
-
1
def initialize(enabled = true)
-
@enabled = enabled
-
@requests = []
-
@started = Hash.new(0)
-
@finished = Hash.new(0)
-
@errored = Hash.new(0)
-
@reset_times = Hash.new { |hs, k| hs[k] = [] }
-
@started_at = {}
-
@total_times = Hash.new { |hs, k| hs[k] = [] }
-
end
-
-
1
def enabled?(_)
-
@enabled
-
end
-
-
1
def start(request)
-
@requests << request
-
@started[request] += 1
-
@started_at[request] = Time.now
-
end
-
-
1
def reset(request)
-
@reset_times[request] << (Time.now - request.init_time)
-
end
-
-
1
def finish(request, _response)
-
@finished[request] += 1
-
@total_times[request] << (Time.now - @started_at[request])
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module Upgrade
-
def test_plugin_upgrade_h2
-
return unless origin.start_with?("https://")
-
-
start_test_servlet(H2Upgrade, alpn_protocols: %w[http/1.1 h2]) do |server|
-
http = HTTPX.plugin(SessionWithPool)
-
-
http = http.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE, alpn_protocols: %w[http/1.1] }) # disable alpn negotiation
-
-
http.plugin(:upgrade).wrap do |session|
-
uri = "#{server.origin}/"
-
-
request = session.build_request("GET", uri)
-
request2 = session.build_request("GET", uri)
-
-
response = session.request(request)
-
verify_status(response, 200)
-
assert response.version == "1.1", "first request should be in HTTP/1.1"
-
response.close
-
# verifies that first request was used to upgrade the connection
-
verify_header(response.headers, "upgrade", "h2")
-
response2 = session.request(request2)
-
verify_status(response2, 200)
-
assert response2.version == "2.0", "second request should already be in HTTP/2"
-
response2.close
-
end
-
end
-
1
end unless RUBY_ENGINE == "jruby"
-
-
1
def test_plugin_upgrade_websockets
-
return unless origin.start_with?("http://")
-
-
http = HTTPX.plugin(SessionWithPool).plugin(:upgrade)
-
-
response = http.get("http://ws-echo-server")
-
verify_status(response, 200)
-
-
http = http.plugin(WSTestPlugin)
-
-
response = http.get("http://ws-echo-server")
-
verify_status(response, 101)
-
-
websocket = response.websocket
-
-
assert !websocket.nil?, "websocket wasn't created"
-
-
websocket.send("ping")
-
websocket.send("pong")
-
-
sleep 2
-
-
echo_messages = websocket.messages
-
assert echo_messages.size >= 3
-
assert echo_messages.include?("handshake")
-
assert echo_messages.include?("ping")
-
assert echo_messages.include?("pong")
-
websocket.close
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Plugins
-
1
module WebDav
-
1
def test_plugin_webdav_mkcol
-
# put file
-
webdav_client.delete("/mkcol_dir_test/")
-
-
response = webdav_client.mkcol("/mkcol_dir_test/")
-
verify_status(response, 201)
-
end
-
-
1
def test_plugin_webdav_copy
-
# put file
-
webdav_client.delete("/copied_copy.html")
-
webdav_client.put("/copy.html", body: "<html></html>")
-
-
response = webdav_client.get("/copied_copy.html")
-
verify_status(response, 404)
-
response = webdav_client.copy("/copy.html", "/copied_copy.html")
-
verify_status(response, 201)
-
response = webdav_client.get("/copied_copy.html")
-
verify_status(response, 200)
-
response = webdav_client.get("/copy.html")
-
verify_status(response, 200)
-
end
-
-
1
def test_plugin_webdav_move
-
# put file
-
webdav_client.delete("/moved_move.html")
-
webdav_client.put("/move.html", body: "<html></html>")
-
-
response = webdav_client.get("/moved_move.html")
-
verify_status(response, 404)
-
response = webdav_client.move("/move.html", "/moved_move.html")
-
verify_status(response, 201)
-
response = webdav_client.get("/move.html")
-
verify_status(response, 404)
-
response = webdav_client.get("/moved_move.html")
-
verify_status(response, 200)
-
end
-
-
1
def test_plugin_webdav_lock
-
# put file
-
webdav_client.put("/lockfile.html", body: "bang")
-
response = webdav_client.lock("/lockfile.html")
-
verify_status(response, 200)
-
lock_token = response.headers["lock-token"]
-
-
response = webdav_client.delete("/lockfile.html")
-
verify_status(response, 423)
-
-
response = webdav_client.unlock("/lockfile.html", lock_token)
-
verify_status(response, 204)
-
-
response = webdav_client.delete("/lockfile.html")
-
verify_status(response, 204)
-
-
webdav_client.put("/lockfile.html", body: "bang")
-
response = webdav_client.lock("/lockfile.html", timeout: 2)
-
verify_status(response, 200)
-
-
response = webdav_client.delete("/lockfile.html")
-
verify_status(response, 423)
-
-
sleep 3
-
response = webdav_client.delete("/lockfile.html")
-
verify_status(response, 204)
-
end
-
-
1
def test_plugin_webdav_lock_blk
-
# put file
-
webdav_client.put("/lockfileblk.html", body: "bang")
-
webdav_client.lock("/lockfileblk.html") do |response|
-
verify_status(response, 200)
-
-
response = webdav_client.delete("/lockfileblk.html")
-
verify_status(response, 423)
-
end
-
response = webdav_client.delete("/lockfileblk.html")
-
verify_status(response, 204)
-
end
-
-
1
def test_plugin_webdav_propfind_proppatch
-
# put file
-
webdav_client.put("/propfind.html", body: "bang")
-
response = webdav_client.propfind("/propfind.html")
-
verify_status(response, 207)
-
xml = "<D:set>" \
-
"<D:prop>" \
-
"<Z:Authors>" \
-
"<Z:Author>Jim Bean</Z:Author>" \
-
"</Z:Authors>" \
-
"</D:prop>" \
-
"</D:set>"
-
response = webdav_client.proppatch("/propfind.html", xml)
-
verify_status(response, 207)
-
-
response = webdav_client.propfind("/propfind.html")
-
verify_status(response, 207)
-
assert response.to_s.include?("Jim Bean")
-
-
response = webdav_client.propfind("/propfind.html", :acl)
-
verify_status(response, 207)
-
-
response = webdav_client.propfind(
-
"/propfind.html",
-
'<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
-
)
-
verify_status(response, 207)
-
assert response.to_s.include?("Jim Bean")
-
end
-
-
1
private
-
-
1
def webdav_client
-
@webdav_client ||= HTTPX.plugin(:basic_auth).plugin(:webdav, origin: start_webdav_server).basic_auth("user", "pass")
-
end
-
-
1
def start_webdav_server
-
origin = ENV.fetch("WEBDAV_HOST")
-
"http://#{origin}"
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
require "nokogiri"
-
-
1
module Requests
-
1
module Plugins
-
1
module XML
-
1
def test_plugin_xml_request_body_document
-
uri = build_uri("/post")
-
response = HTTPX.plugin(:xml).post(uri, xml: Nokogiri::XML("<xml></xml>"))
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/xml; charset=utf-8")
-
# nokogiri in cruby adds \n trailer, jruby doesn't
-
assert body["data"].start_with?("<?xml version=\"1.0\"?>\n<xml/>")
-
end
-
-
1
def test_plugin_xml_request_body_string
-
uri = build_uri("/post")
-
response = HTTPX.plugin(:xml).post(uri, xml: "<xml></xml>")
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/xml; charset=utf-8")
-
assert body["data"] == "<xml></xml>"
-
end
-
-
1
def test_plugin_xml_response
-
uri = build_uri("/xml")
-
response = HTTPX.plugin(:xml).get(uri)
-
verify_status(response, 200)
-
verify_body_length(response)
-
xml = response.xml
-
assert xml.is_a?(Nokogiri::XML::Node)
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module Resolvers
-
1
using HTTPX::URIExtensions
-
-
1
ResolverTimeoutPlugin = Module.new do
-
1
self::ResolverNativeMethods = Module.new do
-
1
def dread
-
@io.read(16_384, "".b)
-
-
super
-
end
-
end
-
-
1
self::ResolverHTTPSMethods = Module.new do
-
# this forces the resolver connection to timeout by setting it to read
-
# when it'll be ready to write requests.
-
1
def resolver_connection
-
super.tap do |conn|
-
def conn.interests
-
return super unless @state == :open
-
-
:r
-
end
-
end
-
end
-
end
-
-
1
self::ResolverSystemMethods = Module.new do
-
# this forces the system resolver to timeout by cleaning the pipe signal
-
# telling the main thread that there's a response.
-
1
def consume
-
sleep(0.5)
-
@pipe_read.read_nonblock(1, exception: false) # drain
-
super
-
end
-
end
-
end
-
-
{
-
1
native: { cache: false },
-
system: { cache: false },
-
https: { uri: ENV["HTTPX_RESOLVER_URI"], cache: false },
-
}.each do |resolver_type, options|
-
3
define_method :"test_resolver_#{resolver_type}_multiple_errors" do
-
2.times do |i|
-
session = HTTPX.plugin(SessionWithPool)
-
unknown_uri = "http://www.sfjewjfwigiewpgwwg-native-#{i}.com"
-
response = session.get(unknown_uri, resolver_class: resolver_type, resolver_options: options)
-
verify_error_response(response, HTTPX::ResolveError)
-
end
-
end
-
-
3
define_method :"test_resolver_#{resolver_type}_request" do
-
session = HTTPX.plugin(SessionWithPool)
-
uri = build_uri("/get")
-
response = session.head(uri, resolver_class: resolver_type, resolver_options: options)
-
verify_status(response, 200)
-
response.close
-
end
-
-
3
define_method :"test_resolver_#{resolver_type}_alias_request" do
-
session = HTTPX.plugin(SessionWithPool)
-
uri = URI(build_uri("/get"))
-
# this google host will resolve to a CNAME
-
uri.host = "lh3.googleusercontent.com"
-
response = session.head(uri, resolver_class: resolver_type, resolver_options: options)
-
assert !response.is_a?(HTTPX::ErrorResponse), "response was an error (#{response})"
-
assert response.status < 500, "unexpected HTTP error (#{response})"
-
response.close
-
end
-
-
3
define_method :"test_resolver_#{resolver_type}_timeout" do
-
resolver_opts = options.merge(timeouts: [1, 2])
-
-
HTTPX.plugin(ResolverTimeoutPlugin).plugin(SessionWithPool).wrap do |session|
-
uri = build_uri("/get")
-
-
# before_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
response = session.get(uri, resolver_class: resolver_type, resolver_options: options.merge(resolver_opts))
-
# after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-
# total_time = after_time - before_time
-
-
verify_error_response(response, HTTPX::ResolveTimeoutError)
-
# assert_in_delta 2 + 1, total_time, 12, "request didn't take as expected to retry dns queries (#{total_time} secs)"
-
end
-
end
-
-
3
define_method :"test_resolver_#{resolver_type}_happy_eyeballs" do
-
skip if resolver_type == :system # still no way to pass the nameserver to getaddrinfo via ruby
-
-
uri = URI(build_uri("/get"))
-
start_test_servlet(TestDNSResolver) do |dns_server|
-
resolver_opts = options.merge(
-
nameserver: [dns_server.nameserver],
-
)
-
-
HTTPX
-
.plugin(SessionWithPool)
-
.with(ip_families: [Socket::AF_INET6, Socket::AF_INET]) do |session|
-
response = session.get(uri, resolver_class: resolver_type, resolver_options: options.merge(resolver_opts))
-
-
verify_status(response, 200)
-
conns = session.connections
-
-
if resolver_type == :https
-
assert conns.size == 3
-
resolver_uri = URI(resolver_opts[:uri])
-
conns.reject! { |c| c.origin.to_s == resolver_uri.origin }
-
else
-
assert conns.size == 2
-
end
-
-
assert(conns.all? { |c| c.origin.to_s == uri.origin })
-
assert(conns.one? { |c| c.family == Socket::AF_INET6 })
-
assert(conns.one? { |c| c.family == Socket::AF_INET })
-
assert(conns.one?(&:main_sibling))
-
end
-
end
-
end
-
-
3
case resolver_type
-
when :https
-
-
1
define_method :"test_resolver_#{resolver_type}_get_request" do
-
HTTPX.plugin(SessionWithPool).wrap do |http|
-
uri = build_uri("/get")
-
response = http.head(uri, resolver_class: resolver_type, resolver_options: options.merge(use_get: true))
-
verify_status(response, 200)
-
response.close
-
resolver_uri = URI(options[:uri])
-
resolver_conn = http.pool.connections.find { |c| c.origin.to_s == resolver_uri.origin }
-
assert resolver_conn, "https resolver connection should still be there"
-
assert resolver_conn.open?, "resolver connection should be kept around open"
-
end
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_unresolvable_servername" do
-
session = HTTPX.plugin(SessionWithPool)
-
uri = build_uri("/get")
-
response = session.head(uri, resolver_class: resolver_type, resolver_options: options.merge(uri: "https://unexisting-doh/dns-query"))
-
verify_error_response(response, HTTPX::ResolveError)
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_server_error" do
-
session = HTTPX.plugin(SessionWithPool)
-
uri = URI(build_uri("/get"))
-
resolver_class = Class.new(HTTPX::Resolver::HTTPS) do
-
def build_request(_hostname)
-
@options.request_class.new("POST", @uri, @options)
-
end
-
end
-
response = session.head(uri, resolver_class: resolver_class, resolver_options: options)
-
verify_error_response(response, HTTPX::ResolveError)
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_decoding_error" do
-
session = HTTPX.plugin(SessionWithPool)
-
uri = URI(build_uri("/get"))
-
resolver_class = Class.new(HTTPX::Resolver::HTTPS) do
-
def decode_response_body(_response)
-
[:decode_error, Resolv::DNS::DecodeError.new("smth")]
-
end
-
end
-
response = session.head(uri, resolver_class: resolver_class, resolver_options: options.merge(record_types: %w[]))
-
verify_error_response(response, HTTPX::ResolveError)
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_encoding_error" do
-
session = HTTPX.plugin(SessionWithPool)
-
uri = URI(build_uri("/get"))
-
resolver_class = Class.new(HTTPX::Resolver::HTTPS) do
-
def build_request(*)
-
raise Resolv::DNS::EncodeError, "ups"
-
end
-
end
-
assert_raises(Resolv::DNS::EncodeError) do
-
session.head(uri, resolver_class: resolver_class, resolver_options: options.merge(record_types: %w[]))
-
end
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_dns_error" do
-
session = HTTPX.plugin(SessionWithPool)
-
uri = URI(build_uri("/get"))
-
resolver_class = Class.new(HTTPX::Resolver::HTTPS) do
-
def decode_response_body(*)
-
[:dns_error, nil]
-
end
-
end
-
response = session.head(uri, resolver_class: resolver_class, resolver_options: options.merge(record_types: %w[]))
-
verify_error_response(response, HTTPX::ResolveError)
-
assert session.pool.connections.empty?
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_no_answers" do
-
HTTPX.plugin(SessionWithPool).wrap do |http|
-
uri = URI(build_uri("/get"))
-
resolver_class = Class.new(HTTPX::Resolver::HTTPS) do
-
def parse_addresses(_, request)
-
super([], request)
-
end
-
end
-
response = http.head(uri, resolver_class: resolver_class, resolver_options: options.merge(record_types: %w[]))
-
verify_error_response(response, HTTPX::ResolveError)
-
resolver_uri = URI(options[:uri])
-
resolver_conn = http.pool.connections.find { |c| c.origin.to_s == resolver_uri.origin }
-
assert resolver_conn, "https resolver connection should still be there"
-
assert resolver_conn.state == :closed, "resolver connection should have closed after error"
-
end
-
end
-
when :native
-
1
define_method :"test_resolver_#{resolver_type}_tcp_request" do
-
tcp_socket = nil
-
resolver_class = Class.new(HTTPX::Resolver::Native) do
-
define_method :build_socket do
-
tcp_socket = super()
-
end
-
end
-
-
session = HTTPX.plugin(SessionWithPool)
-
uri = build_uri("/get")
-
response = session.head(uri, resolver_class: resolver_class, resolver_options: options.merge(socket_type: :tcp))
-
verify_status(response, 200)
-
response.close
-
-
assert !tcp_socket.nil?
-
assert tcp_socket.is_a?(HTTPX::TCP)
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_same_relative_name" do
-
addresses = nil
-
resolver_class = Class.new(HTTPX::Resolver::Native) do
-
define_method :parse_addresses do |addrs|
-
addresses = addrs
-
super(addrs)
-
end
-
end
-
-
start_test_servlet(DNSSameRelativeName) do |slow_dns_server|
-
start_test_servlet(DNSSameRelativeName) do |not_so_slow_dns_server|
-
nameservers = [slow_dns_server.nameserver, not_so_slow_dns_server.nameserver]
-
-
resolver_opts = options.merge(nameserver: nameservers)
-
-
session = HTTPX.plugin(SessionWithPool)
-
uri = URI(build_uri("/get"))
-
response = session.head(uri, resolver_class: resolver_class, resolver_options: resolver_opts)
-
verify_status(response, 200)
-
response.close
-
-
assert !addresses.nil?
-
addr = addresses.first
-
assert addr["name"] != uri.host
-
end
-
end
-
end
-
-
# this test mocks the case where there's no nameserver set to send the DNS queries to.
-
1
define_method :"test_resolver_#{resolver_type}_no_nameserver" do
-
session = HTTPX.plugin(SessionWithPool)
-
uri = build_uri("/get")
-
-
response = session.head(uri, resolver_class: resolver_type, resolver_options: options.merge(nameserver: nil))
-
verify_error_response(response, HTTPX::ResolveError)
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_slow_nameserver" do
-
start_test_servlet(SlowDNSServer, 6) do |slow_dns_server|
-
start_test_servlet(SlowDNSServer, 1) do |not_so_slow_dns_server|
-
nameservers = [slow_dns_server.nameserver, not_so_slow_dns_server.nameserver]
-
-
resolver_opts = options.merge(nameserver: nameservers, timeouts: [3])
-
-
HTTPX.plugin(SessionWithPool).wrap do |session|
-
uri = build_uri("/get")
-
-
response = session.get(uri, resolver_class: resolver_type, resolver_options: options.merge(resolver_opts))
-
verify_status(response, 200)
-
-
resolver = session.resolver
-
assert resolver.instance_variable_get(:@ns_index) == 1
-
end
-
end
-
end
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_dns_error" do
-
start_test_servlet(DNSErrorServer) do |slow_dns_server|
-
start_test_servlet(DNSErrorServer) do |not_so_slow_dns_server|
-
nameservers = [slow_dns_server.nameserver, not_so_slow_dns_server.nameserver]
-
-
resolver_opts = options.merge(nameserver: nameservers)
-
-
HTTPX.plugin(SessionWithPool).wrap do |session|
-
uri = build_uri("/get")
-
-
response = session.get(uri, resolver_class: resolver_type, resolver_options: options.merge(resolver_opts))
-
verify_error_response(response, /unknown DNS error/)
-
end
-
end
-
end
-
end
-
-
# this test mocks a DNS server invalid messages back
-
1
define_method :"test_resolver_#{resolver_type}_decoding_error" do
-
HTTPX.plugin(SessionWithPool).wrap do |session|
-
uri = URI(build_uri("/get"))
-
before_connections = nil
-
resolver_class = Class.new(HTTPX::Resolver::Native) do
-
attr_reader :connections
-
-
define_method :parse do |buffer|
-
before_connections = @connections.size
-
super(buffer[0..-2])
-
end
-
end
-
response = session.head(uri, resolver_class: resolver_class, resolver_options: options.merge(record_types: %w[]))
-
verify_error_response(response, HTTPX::NativeResolveError)
-
assert session.resolvers.size == 1
-
resolver = session.resolvers.first
-
resolver = resolver.resolvers.first # because it's a multi
-
assert resolver.state == :closed
-
assert before_connections == 1, "resolver should have been resolving one connection"
-
assert resolver.connections.empty?, "resolver should not hold connections at this point anymore"
-
end
-
end
-
-
# this test mocks a DNS server breaking the socket with Errno::EHOSTUNREACH
-
1
define_method :"test_resolver_#{resolver_type}_unreachable" do
-
session = HTTPX.plugin(SessionWithPool)
-
uri = URI(build_uri("/get"))
-
resolver_class = Class.new(HTTPX::Resolver::Native) do
-
class << self
-
attr_accessor :attempts
-
end
-
self.attempts = 0
-
-
def dwrite
-
self.class.attempts += 1
-
raise Errno::EHOSTUNREACH, "host unreachable"
-
end
-
end
-
response = session.head(uri, resolver_class: resolver_class, resolver_options: options.merge(nameserver: %w[127.0.0.1] * 3))
-
verify_error_response(response, HTTPX::ResolveError)
-
assert resolver_class.attempts == 3, "should have attempted to use all 3 nameservers"
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_max_udp_size_exceeded" do
-
uri = origin("1024.size.dns.netmeister.org")
-
session = HTTPX.plugin(SessionWithPool)
-
-
resolver_class = Class.new(HTTPX::Resolver::Native) do
-
@ios = []
-
-
class << self
-
attr_reader :ios
-
end
-
-
private
-
-
def build_socket
-
io = super
-
self.class.ios << io
-
io
-
end
-
end
-
-
response = session.head(uri, timeout: { connect_timeout: 2 }, resolver_class: resolver_class,
-
resolver_options: options.merge(nameserver: %w[166.84.7.99]))
-
verify_error_response(response, HTTPX::Error)
-
-
assert resolver_class.ios.any?(HTTPX::TCP), "resolver did not upgrade to tcp"
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_max_udp_size_exceeded_with_cname" do
-
uri = origin("1024.size.dns.netmeister.org")
-
session = HTTPX.plugin(SessionWithPool)
-
-
resolver_class = Class.new(HTTPX::Resolver::Native) do
-
@ios = []
-
-
class << self
-
attr_reader :ios
-
end
-
-
def build_socket
-
io = super
-
self.class.ios << io
-
io
-
end
-
-
def parse_addresses(addresses)
-
addr = addresses.first
-
-
return super unless addr["name"] == "1024.size.dns.netmeister.org"
-
-
# insert bogus CNAME
-
addresses.unshift(
-
{
-
"name" => "1024.size.dns.netmeister.org",
-
"TTL" => 10,
-
"alias" => ENV.fetch("HTTPBIN_HOST", "nghttp2.org/httpbin"),
-
}
-
)
-
super
-
end
-
end
-
-
response = session.head(uri, timeout: { connect_timeout: 2 }, resolver_class: resolver_class,
-
resolver_options: options.merge(nameserver: %w[166.84.7.99]))
-
verify_error_response(response, HTTPX::Error)
-
-
assert resolver_class.ios.any?(HTTPX::TCP), "resolver did not upgrade to tcp"
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_no_addresses" do
-
start_test_servlet(DNSNoAddress) do |slow_dns_server|
-
start_test_servlet(DNSNoAddress) do |not_so_slow_dns_server|
-
nameservers = [slow_dns_server.nameserver, not_so_slow_dns_server.nameserver]
-
-
resolver_opts = options.merge(nameserver: nameservers)
-
-
HTTPX.plugin(SessionWithPool).wrap do |session|
-
uri = build_uri("/get")
-
-
response = session.get(uri, resolver_class: resolver_type, resolver_options: resolver_opts)
-
verify_error_response(response, /Can't resolve/)
-
end
-
end
-
end
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_ttl_expired" do
-
start_test_servlet(TestDNSResolver, ttl: 4) do |short_ttl_dns_server|
-
nameservers = [short_ttl_dns_server.nameserver]
-
-
resolver_opts = options.merge(nameserver: nameservers)
-
-
session = HTTPX.plugin(SessionWithPool)
-
-
2.times do
-
uri = URI(build_uri("/get"))
-
response = session.head(uri, resolver_class: resolver_type, resolver_options: resolver_opts)
-
verify_status(response, 200)
-
response.close
-
end
-
-
# expire ttl
-
sleep 4
-
uri = URI(build_uri("/get"))
-
response = session.head(uri, resolver_class: resolver_type, resolver_options: resolver_opts)
-
verify_status(response, 200)
-
response.close
-
-
num_answers = short_ttl_dns_server.answers
-
assert num_answers == 2, "should have only answered 2 times for DNS queries, instead is #{num_answers}"
-
end
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_candidate" do
-
uri = URI(build_uri("/get"))
-
-
only_to_candidate = Class.new(TestDNSResolver) do
-
define_method :dns_response do |query|
-
domain = extract_domain(query)
-
-
return unless domain == "#{uri.hostname}.local." # last condidate
-
-
super(query)
-
end
-
-
def resolve(domain, typevalue)
-
super(domain.delete_suffix(".local."), typevalue)
-
end
-
end
-
-
start_test_servlet(only_to_candidate) do |slow_dns_server|
-
dns_config = {
-
nameserver: [slow_dns_server.nameserver],
-
timeouts: [1, 2],
-
dots: 1,
-
search: "local",
-
}
-
resolver_opts = options.merge(dns_config)
-
-
HTTPX.plugin(SessionWithPool).wrap do |session|
-
response = session.get(uri, resolver_class: resolver_type, resolver_options: options.merge(resolver_opts))
-
-
verify_status(response, 200)
-
assert session.resolvers.size == 1
-
resolver = session.resolvers.first
-
resolver = resolver.resolvers.first # because it's a multi
-
assert resolver.state == :closed
-
-
tries = resolver.tries
-
assert tries.keys.size == 2
-
assert tries.key?(uri.hostname)
-
assert tries[uri.hostname] == 2, "should have tried canonical 2 times"
-
assert tries.key?("#{uri.hostname}.local")
-
-
assert resolver.timeouts.empty?, "should have cleaned up all candidate timeouts"
-
end
-
end
-
end
-
-
1
define_method :"test_resolver_#{resolver_type}_servfail_should_retry" do
-
uri = URI(build_uri("/get"))
-
-
start_test_servlet(DNSServFailOnce) do |dns_server|
-
resolver_opts = options.merge(
-
nameserver: [dns_server.nameserver],
-
timeouts: [1, 1, 2] # more than 2 in case one of the writes fail
-
)
-
-
session = HTTPX.plugin(SessionWithPool).with(ip_families: [Socket::AF_INET])
-
response = session.get(uri, resolver_class: resolver_type, resolver_options: options.merge(resolver_opts))
-
-
verify_status(response, 200)
-
-
assert dns_server.failed
-
assert dns_server.queries == 2
-
end
-
end
-
end
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module ResponseBody
-
1
def test_http_response_copy_to_file
-
file = Tempfile.new(%w[cat .jpeg])
-
uri = build_uri("/image")
-
response = HTTPX.get(uri, headers: { "accept" => "image/jpeg" })
-
verify_status(response, 200)
-
verify_header(response.headers, "content-type", "image/jpeg")
-
verify_body_length(response)
-
response.copy_to(file)
-
content_length = response.headers["content-length"].to_i
-
assert file.size == content_length, "file should contain the content of response"
-
ensure
-
if file
-
file.close
-
file.unlink
-
end
-
end
-
-
1
def test_http_response_copy_to_io
-
io = StringIO.new
-
uri = build_uri("/image")
-
response = HTTPX.get(uri, headers: { "accept" => "image/jpeg" })
-
verify_status(response, 200)
-
verify_header(response.headers, "content-type", "image/jpeg")
-
response.copy_to(io)
-
content_length = response.headers["content-length"].to_i
-
assert io.size == content_length, "file should contain the content of response"
-
ensure
-
io.close if io
-
end
-
-
1
def test_http_response_buffer_to_custom
-
uri = build_uri("/")
-
custom_body = Class.new(HTTPX::Response::Body) do
-
attr_reader :file
-
-
def initialize(_response, _opts)
-
super
-
@file = Tempfile.new("httpx-test")
-
end
-
-
def write(data)
-
@file << data
-
end
-
-
def close
-
return unless @file
-
-
@file.close
-
@file.unlink
-
end
-
end
-
-
response = HTTPX.with(response_body_class: custom_body).get(uri)
-
verify_status(response, 200)
-
assert response.body.is_a?(custom_body), "body should be from custom type"
-
file = response.body.file
-
file.rewind
-
content_length = response.headers["content-length"].to_i
-
assert file.size == content_length, "didn't buffer the whole body"
-
ensure
-
response.close if response
-
end
-
end
-
end
-
# frozen_string_literal: true
-
-
1
module Requests
-
1
module WithBody
-
1
%w[post put patch delete].each do |meth|
-
4
define_method :"test_#{meth}_query_params" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, params: { "q" => "this is a test" })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_uploaded(body, "args", "q" => "this is a test")
-
verify_uploaded(body, "url", build_uri("/#{meth}?q=this+is+a+test"))
-
-
return unless can_run_ractor_tests?
-
-
response2 = Ractor.new(meth, uri) do |meth, uri|
-
HTTPX.send(meth, uri, params: { "q" => "this is a test" })
-
end.value
-
-
verify_status(response2, 200)
-
body2 = json_body(response2)
-
verify_uploaded(body2, "args", "q" => "this is a test")
-
verify_uploaded(body2, "url", build_uri("/#{meth}?q=this+is+a+test"))
-
end
-
-
4
define_method :"test_#{meth}_query_params_empty" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, "#{uri}?foo=bar", params: {})
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_uploaded(body, "args", "foo" => "bar")
-
verify_uploaded(body, "url", build_uri("/#{meth}?foo=bar"))
-
end
-
-
4
define_method :"test_#{meth}_query_nested_params" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, params: { "q" => { "a" => "z" }, "a" => %w[1 2], "b" => [] })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_uploaded(body, "args", "q[a]" => "z", "a[]" => %w[1 2], "b[]" => "")
-
verify_uploaded(body, "url", build_uri("/#{meth}?q[a]=z&a[]=1&a[]=2&b[]"))
-
end
-
-
4
define_method :"test_#{meth}_form_params" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { "foo" => "bar" })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/x-www-form-urlencoded")
-
verify_uploaded(body, "form", "foo" => "bar")
-
-
return unless can_run_ractor_tests?
-
-
response2 = Ractor.new(meth, uri) do |meth, uri|
-
HTTPX.send(meth, uri, form: { "foo" => "bar" })
-
end.value
-
-
verify_status(response2, 200)
-
body2 = json_body(response2)
-
verify_header(body2["headers"], "Content-Type", "application/x-www-form-urlencoded")
-
verify_uploaded(body2, "form", "foo" => "bar")
-
end
-
-
4
define_method :"test_#{meth}_form_nested_params" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, form: { "q" => { "a" => "z" }, "a" => %w[1 2], "b" => [] })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/x-www-form-urlencoded")
-
verify_uploaded(body, "form", "q[a]" => "z", "a[]" => %w[1 2], "b[]" => "")
-
end
-
-
4
define_method :"test_#{meth}_expect_100_form_params" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.with_headers("expect" => "100-continue")
-
.send(meth, uri, form: { "foo" => "bar" })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/x-www-form-urlencoded")
-
verify_header(body["headers"], "Expect", "100-continue")
-
verify_uploaded(body, "form", "foo" => "bar")
-
end
-
-
4
define_method :"test_#{meth}_json_params" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, json: { "foo" => "bar" })
-
verify_status(response, 200)
-
body = json_body(response)
-
verify_header(body["headers"], "Content-Type", "application/json")
-
verify_uploaded(body, "json", "foo" => "bar")
-
-
return unless can_run_ractor_tests?
-
-
response2 = Ractor.new(meth, uri) do |meth, uri|
-
HTTPX.send(meth, uri, json: { "foo" => "bar" })
-
end.value
-
-
verify_status(response2, 200)
-
body2 = json_body(response2)
-
verify_header(body2["headers"], "Content-Type", "application/json")
-
verify_uploaded(body2, "json", "foo" => "bar")
-
end
-
-
4
define_method :"test_#{meth}_body_params" do
-
uri = build_uri("/#{meth}")
-
response = HTTPX.send(meth, uri, body: "data")
-