loading
Generated 2025-03-28T00:09:56+00:00

All Files ( 96.0% covered at 88054.01 hits/line )

102 files in total.
7269 relevant lines, 6978 lines covered and 291 lines missed. ( 96.0% )
File % covered Lines Relevant Lines Lines covered Lines missed Avg. Hits / Line
lib/httpx.rb 100.00 % 66 39 39 0 772.59
lib/httpx/adapters/datadog.rb 86.09 % 324 151 130 21 38.34
lib/httpx/adapters/faraday.rb 98.13 % 298 160 157 3 133.92
lib/httpx/adapters/sentry.rb 100.00 % 121 62 62 0 88.19
lib/httpx/adapters/webmock.rb 100.00 % 172 91 91 0 96.13
lib/httpx/altsvc.rb 96.39 % 163 83 80 3 236.58
lib/httpx/buffer.rb 100.00 % 50 21 21 0 550033.38
lib/httpx/callbacks.rb 100.00 % 35 19 19 0 164233.21
lib/httpx/chainable.rb 95.65 % 106 46 44 2 1047.15
lib/httpx/connection.rb 92.47 % 933 478 442 36 197422.64
lib/httpx/connection/http1.rb 89.19 % 401 222 198 24 3780.68
lib/httpx/connection/http2.rb 94.90 % 428 255 242 13 315689.61
lib/httpx/domain_name.rb 95.56 % 145 45 43 2 246.96
lib/httpx/errors.rb 97.83 % 118 46 45 1 81.46
lib/httpx/extensions.rb 67.86 % 59 28 19 9 405.11
lib/httpx/headers.rb 100.00 % 175 71 71 0 18884.70
lib/httpx/io.rb 100.00 % 11 5 5 0 26.00
lib/httpx/io/ssl.rb 96.47 % 162 85 82 3 2161.36
lib/httpx/io/tcp.rb 91.15 % 206 113 103 10 8386.32
lib/httpx/io/udp.rb 100.00 % 62 35 35 0 341.63
lib/httpx/io/unix.rb 94.29 % 70 35 33 2 20.17
lib/httpx/loggable.rb 100.00 % 41 17 17 0 51920.24
lib/httpx/options.rb 98.73 % 360 157 155 2 17869.27
lib/httpx/parser/http1.rb 100.00 % 186 109 109 0 7622.06
lib/httpx/plugins/auth.rb 100.00 % 25 9 9 0 24.00
lib/httpx/plugins/auth/basic.rb 100.00 % 20 10 10 0 86.60
lib/httpx/plugins/auth/digest.rb 100.00 % 102 61 61 0 127.85
lib/httpx/plugins/auth/ntlm.rb 100.00 % 35 19 19 0 2.95
lib/httpx/plugins/auth/socks5.rb 100.00 % 22 11 11 0 23.82
lib/httpx/plugins/aws_sdk_authentication.rb 100.00 % 109 43 43 0 13.21
lib/httpx/plugins/aws_sigv4.rb 100.00 % 236 120 120 0 101.51
lib/httpx/plugins/basic_auth.rb 100.00 % 29 12 12 0 35.33
lib/httpx/plugins/brotli.rb 100.00 % 50 25 25 0 10.80
lib/httpx/plugins/callbacks.rb 100.00 % 102 48 48 0 139.46
lib/httpx/plugins/circuit_breaker.rb 100.00 % 144 65 65 0 71.78
lib/httpx/plugins/circuit_breaker/circuit.rb 100.00 % 100 47 47 0 58.21
lib/httpx/plugins/circuit_breaker/circuit_store.rb 100.00 % 53 23 23 0 98.48
lib/httpx/plugins/content_digest.rb 100.00 % 202 102 102 0 79.71
lib/httpx/plugins/cookies.rb 100.00 % 107 51 51 0 118.76
lib/httpx/plugins/cookies/cookie.rb 100.00 % 174 77 77 0 318.21
lib/httpx/plugins/cookies/jar.rb 100.00 % 97 47 47 0 265.34
lib/httpx/plugins/cookies/set_cookie_parser.rb 100.00 % 140 70 70 0 161.01
lib/httpx/plugins/digest_auth.rb 100.00 % 65 29 29 0 99.24
lib/httpx/plugins/expect.rb 100.00 % 118 56 56 0 87.95
lib/httpx/plugins/follow_redirects.rb 100.00 % 229 106 106 0 172337.21
lib/httpx/plugins/grpc.rb 100.00 % 280 133 133 0 117.79
lib/httpx/plugins/grpc/call.rb 90.91 % 63 33 30 3 40.18
lib/httpx/plugins/grpc/grpc_encoding.rb 97.87 % 90 47 46 1 75.57
lib/httpx/plugins/grpc/message.rb 95.83 % 55 24 23 1 39.50
lib/httpx/plugins/h2c.rb 94.64 % 111 56 53 3 14.54
lib/httpx/plugins/ntlm_auth.rb 100.00 % 60 30 30 0 3.97
lib/httpx/plugins/oauth.rb 100.00 % 175 87 87 0 70.23
lib/httpx/plugins/persistent.rb 100.00 % 52 19 19 0 226.16
lib/httpx/plugins/proxy.rb 98.01 % 308 151 148 3 324.74
lib/httpx/plugins/proxy/http.rb 100.00 % 182 102 102 0 190.03
lib/httpx/plugins/proxy/socks4.rb 97.47 % 135 79 77 2 177.80
lib/httpx/plugins/proxy/socks5.rb 99.12 % 194 113 112 1 279.48
lib/httpx/plugins/proxy/ssh.rb 92.31 % 92 52 48 4 8.19
lib/httpx/plugins/push_promise.rb 100.00 % 81 41 41 0 10.49
lib/httpx/plugins/rate_limiter.rb 100.00 % 55 18 18 0 41.72
lib/httpx/plugins/response_cache.rb 100.00 % 181 82 82 0 76.43
lib/httpx/plugins/response_cache/store.rb 100.00 % 93 47 47 0 115.91
lib/httpx/plugins/retries.rb 97.89 % 216 95 93 2 215270.25
lib/httpx/plugins/ssrf_filter.rb 100.00 % 145 61 61 0 133.23
lib/httpx/plugins/stream.rb 100.00 % 157 74 74 0 118.30
lib/httpx/plugins/upgrade.rb 100.00 % 78 34 34 0 47.24
lib/httpx/plugins/upgrade/h2.rb 91.67 % 54 24 22 2 8.29
lib/httpx/plugins/webdav.rb 100.00 % 86 39 39 0 23.10
lib/httpx/plugins/xml.rb 100.00 % 76 34 34 0 84.35
lib/httpx/pmatch_extensions.rb 100.00 % 33 17 17 0 28.00
lib/httpx/pool.rb 100.00 % 132 68 68 0 5005.65
lib/httpx/punycode.rb 100.00 % 22 9 9 0 18.22
lib/httpx/request.rb 100.00 % 301 124 124 0 6594.81
lib/httpx/request/body.rb 100.00 % 153 66 66 0 3090.30
lib/httpx/resolver.rb 100.00 % 161 82 82 0 1628.05
lib/httpx/resolver/https.rb 86.11 % 248 144 124 20 33.88
lib/httpx/resolver/multi.rb 88.24 % 93 51 45 6 3504.35
lib/httpx/resolver/native.rb 88.39 % 538 310 274 36 1458.07
lib/httpx/resolver/resolver.rb 83.13 % 165 83 69 14 1624.00
lib/httpx/resolver/system.rb 81.06 % 230 132 107 25 17.57
lib/httpx/response.rb 100.00 % 294 111 111 0 1752.91
lib/httpx/response/body.rb 100.00 % 253 115 115 0 2662.05
lib/httpx/response/buffer.rb 96.00 % 96 50 48 2 1881.90
lib/httpx/selector.rb 92.52 % 217 107 99 8 2025370.52
lib/httpx/session.rb 94.24 % 496 243 229 14 99792.76
lib/httpx/session_extensions.rb 100.00 % 29 14 14 0 6.43
lib/httpx/timers.rb 95.38 % 125 65 62 3 2101956.42
lib/httpx/transcoder.rb 100.00 % 91 52 52 0 281.52
lib/httpx/transcoder/body.rb 100.00 % 43 26 26 0 944.62
lib/httpx/transcoder/chunker.rb 100.00 % 115 66 66 0 219.70
lib/httpx/transcoder/deflate.rb 100.00 % 37 20 20 0 30.20
lib/httpx/transcoder/form.rb 100.00 % 80 42 42 0 439.10
lib/httpx/transcoder/gzip.rb 100.00 % 71 40 40 0 96.05
lib/httpx/transcoder/json.rb 100.00 % 71 33 33 0 42.06
lib/httpx/transcoder/multipart.rb 100.00 % 17 10 10 0 921.00
lib/httpx/transcoder/multipart/decoder.rb 93.98 % 139 83 78 5 28.93
lib/httpx/transcoder/multipart/encoder.rb 100.00 % 119 70 70 0 1945.69
lib/httpx/transcoder/multipart/mime_type_detector.rb 92.11 % 78 38 35 3 177.18
lib/httpx/transcoder/multipart/part.rb 100.00 % 35 18 18 0 504.89
lib/httpx/transcoder/utils/body_reader.rb 92.00 % 46 25 23 2 94.44
lib/httpx/transcoder/utils/deflater.rb 100.00 % 75 37 37 0 102.59
lib/httpx/utils.rb 100.00 % 75 39 39 0 584677.15

lib/httpx.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "httpx/version"
  3. # Top-Level Namespace
  4. #
  5. 26 module HTTPX
  6. 26 EMPTY = [].freeze
  7. 26 EMPTY_HASH = {}.freeze
  8. # All plugins should be stored under this module/namespace. Can register and load
  9. # plugins.
  10. #
  11. 26 module Plugins
  12. 26 @plugins = {}
  13. 26 @plugins_mutex = Thread::Mutex.new
  14. # Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
  15. # it from the load path under "httpx/plugins/" directory.
  16. #
  17. 26 def self.load_plugin(name)
  18. 5454 h = @plugins
  19. 5454 m = @plugins_mutex
  20. 10908 unless (plugin = m.synchronize { h[name] })
  21. 299 require "httpx/plugins/#{name}"
  22. 598 raise "Plugin #{name} hasn't been registered" unless (plugin = m.synchronize { h[name] })
  23. end
  24. 5454 plugin
  25. end
  26. # Registers a plugin (+mod+) in the central store indexed by +name+.
  27. #
  28. 26 def self.register_plugin(name, mod)
  29. 303 h = @plugins
  30. 303 m = @plugins_mutex
  31. 578 m.synchronize { h[name] = mod }
  32. end
  33. end
  34. end
  35. 26 require "httpx/extensions"
  36. 26 require "httpx/errors"
  37. 26 require "httpx/utils"
  38. 26 require "httpx/punycode"
  39. 26 require "httpx/domain_name"
  40. 26 require "httpx/altsvc"
  41. 26 require "httpx/callbacks"
  42. 26 require "httpx/loggable"
  43. 26 require "httpx/transcoder"
  44. 26 require "httpx/timers"
  45. 26 require "httpx/pool"
  46. 26 require "httpx/headers"
  47. 26 require "httpx/request"
  48. 26 require "httpx/response"
  49. 26 require "httpx/options"
  50. 26 require "httpx/chainable"
  51. 26 require "httpx/session"
  52. 26 require "httpx/session_extensions"
  53. # load integrations when possible
  54. 26 require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog::Tracing)
  55. 26 require "httpx/adapters/sentry" if defined?(Sentry)
  56. 26 require "httpx/adapters/webmock" if defined?(WebMock)

lib/httpx/adapters/datadog.rb

86.09% lines covered

151 relevant lines. 130 lines covered and 21 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "datadog/tracing/contrib/integration"
  3. 6 require "datadog/tracing/contrib/configuration/settings"
  4. 6 require "datadog/tracing/contrib/patcher"
  5. 6 module Datadog::Tracing
  6. 6 module Contrib
  7. 6 module HTTPX
  8. 6 DATADOG_VERSION = defined?(::DDTrace) ? ::DDTrace::VERSION : ::Datadog::VERSION
  9. 6 METADATA_MODULE = Datadog::Tracing::Metadata
  10. 6 TYPE_OUTBOUND = Datadog::Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
  11. 6 TAG_PEER_SERVICE = Datadog::Tracing::Metadata::Ext::TAG_PEER_SERVICE
  12. 6 TAG_URL = Datadog::Tracing::Metadata::Ext::HTTP::TAG_URL
  13. 6 TAG_METHOD = Datadog::Tracing::Metadata::Ext::HTTP::TAG_METHOD
  14. 6 TAG_TARGET_HOST = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_HOST
  15. 6 TAG_TARGET_PORT = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_PORT
  16. 6 TAG_STATUS_CODE = Datadog::Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE
  17. # HTTPX Datadog Plugin
  18. #
  19. # Enables tracing for httpx requests.
  20. #
  21. # A span will be created for each request transaction; the span is created lazily only when
  22. # buffering a request, and it is fed the start time stored inside the tracer object.
  23. #
  24. 6 module Plugin
  25. 6 module RequestTracer
  26. 6 extend Contrib::HttpAnnotationHelper
  27. 6 module_function
  28. 6 SPAN_REQUEST = "httpx.request"
  29. # initializes tracing on the +request+.
  30. 6 def call(request)
  31. 228 return unless configuration(request).enabled
  32. 84 span = nil
  33. # request objects are reused, when already buffered requests get rerouted to a different
  34. # connection due to connection issues, or when they already got a response, but need to
  35. # be retried. In such situations, the original span needs to be extended for the former,
  36. # while a new is required for the latter.
  37. 84 request.on(:idle) do
  38. 18 span = nil
  39. end
  40. # the span is initialized when the request is buffered in the parser, which is the closest
  41. # one gets to actually sending the request.
  42. 84 request.on(:headers) do
  43. 96 next if span
  44. 96 span = initialize_span(request, now)
  45. end
  46. 84 request.on(:response) do |response|
  47. 96 unless span
  48. 6 next unless response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
  49. # handles the case when the +error+ happened during name resolution, which means
  50. # that the tracing start point hasn't been triggered yet; in such cases, the approximate
  51. # initial resolving time is collected from the connection, and used as span start time,
  52. # and the tracing object in inserted before the on response callback is called.
  53. 6 span = initialize_span(request, response.error.connection.init_time)
  54. end
  55. 96 finish(response, span)
  56. end
  57. end
  58. 6 def finish(response, span)
  59. 96 if response.is_a?(::HTTPX::ErrorResponse)
  60. 6 span.set_error(response.error)
  61. else
  62. 90 span.set_tag(TAG_STATUS_CODE, response.status.to_s)
  63. 90 span.set_error(::HTTPX::HTTPError.new(response)) if response.status >= 400 && response.status <= 599
  64. end
  65. 96 span.finish
  66. end
  67. # return a span initialized with the +@request+ state.
  68. 6 def initialize_span(request, start_time)
  69. 102 verb = request.verb
  70. 102 uri = request.uri
  71. 102 config = configuration(request)
  72. 102 span = create_span(request, config, start_time)
  73. 102 span.resource = verb
  74. # Add additional request specific tags to the span.
  75. 102 span.set_tag(TAG_URL, request.path)
  76. 102 span.set_tag(TAG_METHOD, verb)
  77. 102 span.set_tag(TAG_TARGET_HOST, uri.host)
  78. 102 span.set_tag(TAG_TARGET_PORT, uri.port)
  79. # Tag as an external peer service
  80. 102 span.set_tag(TAG_PEER_SERVICE, span.service)
  81. 102 if config[:distributed_tracing]
  82. 96 propagate_trace_http(
  83. Datadog::Tracing.active_trace,
  84. request.headers
  85. )
  86. end
  87. # Set analytics sample rate
  88. 102 if Contrib::Analytics.enabled?(config[:analytics_enabled])
  89. 12 Contrib::Analytics.set_sample_rate(span, config[:analytics_sample_rate])
  90. end
  91. 102 span
  92. rescue StandardError => e
  93. Datadog.logger.error("error preparing span for http request: #{e}")
  94. Datadog.logger.error(e.backtrace)
  95. end
  96. 6 def now
  97. 96 ::Datadog::Core::Utils::Time.now.utc
  98. end
  99. 6 def configuration(request)
  100. 330 Datadog.configuration.tracing[:httpx, request.uri.host]
  101. end
  102. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
  103. 3 def propagate_trace_http(trace, headers)
  104. 48 Datadog::Tracing::Contrib::HTTP.inject(trace, headers)
  105. end
  106. 3 def create_span(request, configuration, start_time)
  107. 51 Datadog::Tracing.trace(
  108. SPAN_REQUEST,
  109. service: service_name(request.uri.host, configuration),
  110. type: TYPE_OUTBOUND,
  111. start_time: start_time
  112. )
  113. end
  114. else
  115. 3 def propagate_trace_http(trace, headers)
  116. 48 Datadog::Tracing::Propagation::HTTP.inject!(trace.to_digest, headers)
  117. end
  118. 3 def create_span(request, configuration, start_time)
  119. 51 Datadog::Tracing.trace(
  120. SPAN_REQUEST,
  121. service: service_name(request.uri.host, configuration),
  122. span_type: TYPE_OUTBOUND,
  123. start_time: start_time
  124. )
  125. end
  126. end
  127. end
  128. 6 module RequestMethods
  129. # intercepts request initialization to inject the tracing logic.
  130. 6 def initialize(*)
  131. 228 super
  132. 228 return unless Datadog::Tracing.enabled?
  133. 228 RequestTracer.call(self)
  134. end
  135. end
  136. 6 module ConnectionMethods
  137. 6 attr_reader :init_time
  138. 6 def initialize(*)
  139. 216 super
  140. 216 @init_time = ::Datadog::Core::Utils::Time.now.utc
  141. end
  142. end
  143. end
  144. 6 module Configuration
  145. # Default settings for httpx
  146. #
  147. 6 class Settings < Datadog::Tracing::Contrib::Configuration::Settings
  148. 6 DEFAULT_ERROR_HANDLER = lambda do |response|
  149. Datadog::Ext::HTTP::ERROR_RANGE.cover?(response.status)
  150. end
  151. 6 option :service_name, default: "httpx"
  152. 6 option :distributed_tracing, default: true
  153. 6 option :split_by_domain, default: false
  154. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  155. 6 option :enabled do |o|
  156. 6 o.type :bool
  157. 6 o.env "DD_TRACE_HTTPX_ENABLED"
  158. 6 o.default true
  159. end
  160. 6 option :analytics_enabled do |o|
  161. 6 o.type :bool
  162. 6 o.env "DD_TRACE_HTTPX_ANALYTICS_ENABLED"
  163. 6 o.default false
  164. end
  165. 6 option :analytics_sample_rate do |o|
  166. 6 o.type :float
  167. 6 o.env "DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE"
  168. 6 o.default 1.0
  169. end
  170. else
  171. option :enabled do |o|
  172. o.default { env_to_bool("DD_TRACE_HTTPX_ENABLED", true) }
  173. o.lazy
  174. end
  175. option :analytics_enabled do |o|
  176. o.default { env_to_bool(%w[DD_TRACE_HTTPX_ANALYTICS_ENABLED DD_HTTPX_ANALYTICS_ENABLED], false) }
  177. o.lazy
  178. end
  179. option :analytics_sample_rate do |o|
  180. o.default { env_to_float(%w[DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE DD_HTTPX_ANALYTICS_SAMPLE_RATE], 1.0) }
  181. o.lazy
  182. end
  183. end
  184. 6 if defined?(Datadog::Tracing::Contrib::SpanAttributeSchema)
  185. 6 option :service_name do |o|
  186. 6 o.default do
  187. 66 Datadog::Tracing::Contrib::SpanAttributeSchema.fetch_service_name(
  188. "DD_TRACE_HTTPX_SERVICE_NAME",
  189. "httpx"
  190. )
  191. end
  192. 6 o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  193. end
  194. else
  195. option :service_name do |o|
  196. o.default do
  197. ENV.fetch("DD_TRACE_HTTPX_SERVICE_NAME", "httpx")
  198. end
  199. o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  200. end
  201. end
  202. 6 option :distributed_tracing, default: true
  203. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.15.0")
  204. 6 option :error_handler do |o|
  205. 6 o.type :proc
  206. 6 o.default_proc(&DEFAULT_ERROR_HANDLER)
  207. end
  208. elsif Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  209. option :error_handler do |o|
  210. o.type :proc
  211. o.experimental_default_proc(&DEFAULT_ERROR_HANDLER)
  212. end
  213. else
  214. option :error_handler, default: DEFAULT_ERROR_HANDLER
  215. end
  216. end
  217. end
  218. # Patcher enables patching of 'httpx' with datadog components.
  219. #
  220. 6 module Patcher
  221. 6 include Datadog::Tracing::Contrib::Patcher
  222. 6 module_function
  223. 6 def target_version
  224. 12 Integration.version
  225. end
  226. # loads a session instannce with the datadog plugin, and replaces the
  227. # base HTTPX::Session with the patched session class.
  228. 6 def patch
  229. 6 datadog_session = ::HTTPX.plugin(Plugin)
  230. 6 ::HTTPX.send(:remove_const, :Session)
  231. 6 ::HTTPX.send(:const_set, :Session, datadog_session.class)
  232. end
  233. end
  234. # Datadog Integration for HTTPX.
  235. #
  236. 6 class Integration
  237. 6 include Contrib::Integration
  238. 6 MINIMUM_VERSION = Gem::Version.new("0.10.2")
  239. 6 register_as :httpx
  240. 6 def self.version
  241. 246 Gem.loaded_specs["httpx"] && Gem.loaded_specs["httpx"].version
  242. end
  243. 6 def self.loaded?
  244. 78 defined?(::HTTPX::Request)
  245. end
  246. 6 def self.compatible?
  247. 78 super && version >= MINIMUM_VERSION
  248. end
  249. 6 def new_configuration
  250. 156 Configuration::Settings.new
  251. end
  252. 6 def patcher
  253. 156 Patcher
  254. end
  255. end
  256. end
  257. end
  258. end

lib/httpx/adapters/faraday.rb

98.13% lines covered

160 relevant lines. 157 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 15 require "delegate"
  3. 15 require "httpx"
  4. 15 require "faraday"
  5. 15 module Faraday
  6. 15 class Adapter
  7. 15 class HTTPX < Faraday::Adapter
  8. 15 module RequestMixin
  9. 15 def build_connection(env)
  10. 287 return @connection if defined?(@connection)
  11. 287 @connection = ::HTTPX.plugin(:persistent).plugin(ReasonPlugin)
  12. 287 @connection = @connection.with(@connection_options) unless @connection_options.empty?
  13. 287 connection_opts = options_from_env(env)
  14. 287 if (bind = env.request.bind)
  15. 8 @bind = TCPSocket.new(bind[:host], bind[:port])
  16. 7 connection_opts[:io] = @bind
  17. end
  18. 287 @connection = @connection.with(connection_opts)
  19. 287 if (proxy = env.request.proxy)
  20. 8 proxy_options = { uri: proxy.uri }
  21. 8 proxy_options[:username] = proxy.user if proxy.user
  22. 8 proxy_options[:password] = proxy.password if proxy.password
  23. 8 @connection = @connection.plugin(:proxy).with(proxy: proxy_options)
  24. end
  25. 287 @connection = @connection.plugin(OnDataPlugin) if env.request.stream_response?
  26. 287 @connection = @config_block.call(@connection) || @connection if @config_block
  27. 287 @connection
  28. end
  29. 15 def close
  30. 294 @connection.close if @connection
  31. 294 @bind.close if @bind
  32. end
  33. 15 private
  34. 15 def connect(env, &blk)
  35. 287 connection(env, &blk)
  36. rescue ::HTTPX::TLSError => e
  37. 8 raise Faraday::SSLError, e
  38. rescue Errno::ECONNABORTED,
  39. Errno::ECONNREFUSED,
  40. Errno::ECONNRESET,
  41. Errno::EHOSTUNREACH,
  42. Errno::EINVAL,
  43. Errno::ENETUNREACH,
  44. Errno::EPIPE,
  45. ::HTTPX::ConnectionError => e
  46. 8 raise Faraday::ConnectionFailed, e
  47. end
  48. 15 def build_request(env)
  49. 295 meth = env[:method]
  50. 30 request_options = {
  51. 264 headers: env.request_headers,
  52. body: env.body,
  53. **options_from_env(env),
  54. }
  55. 295 [meth.to_s.upcase, env.url, request_options]
  56. end
  57. 15 def options_from_env(env)
  58. 582 timeout_options = {}
  59. 582 req_opts = env.request
  60. 582 if (sec = request_timeout(:read, req_opts))
  61. 28 timeout_options[:read_timeout] = sec
  62. end
  63. 582 if (sec = request_timeout(:write, req_opts))
  64. 14 timeout_options[:write_timeout] = sec
  65. end
  66. 582 if (sec = request_timeout(:open, req_opts))
  67. 14 timeout_options[:connect_timeout] = sec
  68. end
  69. 59 {
  70. 522 ssl: ssl_options_from_env(env),
  71. timeout: timeout_options,
  72. }
  73. end
  74. 15 if defined?(::OpenSSL)
  75. 15 def ssl_options_from_env(env)
  76. 582 ssl_options = {}
  77. 582 unless env.ssl.verify.nil?
  78. 28 ssl_options[:verify_mode] = env.ssl.verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
  79. end
  80. 582 ssl_options[:ca_file] = env.ssl.ca_file if env.ssl.ca_file
  81. 582 ssl_options[:ca_path] = env.ssl.ca_path if env.ssl.ca_path
  82. 582 ssl_options[:cert_store] = env.ssl.cert_store if env.ssl.cert_store
  83. 582 ssl_options[:cert] = env.ssl.client_cert if env.ssl.client_cert
  84. 582 ssl_options[:key] = env.ssl.client_key if env.ssl.client_key
  85. 582 ssl_options[:ssl_version] = env.ssl.version if env.ssl.version
  86. 582 ssl_options[:verify_depth] = env.ssl.verify_depth if env.ssl.verify_depth
  87. 582 ssl_options[:min_version] = env.ssl.min_version if env.ssl.min_version
  88. 582 ssl_options[:max_version] = env.ssl.max_version if env.ssl.max_version
  89. 582 ssl_options
  90. end
  91. else
  92. skipped # :nocov:
  93. skipped def ssl_options_from_env(*)
  94. skipped {}
  95. skipped end
  96. skipped # :nocov:
  97. end
  98. end
  99. 15 include RequestMixin
  100. 15 module OnDataPlugin
  101. 15 module RequestMethods
  102. 15 attr_writer :response_on_data
  103. 15 def response=(response)
  104. 16 super
  105. 16 return if response.is_a?(::HTTPX::ErrorResponse)
  106. 16 response.body.on_data = @response_on_data
  107. end
  108. end
  109. 15 module ResponseBodyMethods
  110. 15 attr_writer :on_data
  111. 15 def write(chunk)
  112. 49 return super unless @on_data
  113. 49 @on_data.call(chunk, chunk.bytesize)
  114. end
  115. end
  116. end
  117. 15 module ReasonPlugin
  118. 15 def self.load_dependencies(*)
  119. 287 require "net/http/status"
  120. end
  121. 15 module ResponseMethods
  122. 15 def reason
  123. 241 Net::HTTP::STATUS_CODES.fetch(@status, "Non-Standard status code")
  124. end
  125. end
  126. end
  127. 15 class ParallelManager
  128. 15 class ResponseHandler < SimpleDelegator
  129. 15 attr_reader :env
  130. 15 def initialize(env)
  131. 32 @env = env
  132. 32 super
  133. end
  134. 15 def on_response(&blk)
  135. 64 if blk
  136. 32 @on_response = ->(response) do
  137. 32 blk.call(response)
  138. end
  139. 32 self
  140. else
  141. 32 @on_response
  142. end
  143. end
  144. 15 def on_complete(&blk)
  145. 32 if blk
  146. @on_complete = blk
  147. self
  148. else
  149. 32 @on_complete
  150. end
  151. end
  152. end
  153. 15 include RequestMixin
  154. 15 def initialize(options)
  155. 32 @handlers = []
  156. 32 @connection_options = options
  157. end
  158. 15 def enqueue(request)
  159. 32 handler = ResponseHandler.new(request)
  160. 32 @handlers << handler
  161. 32 handler
  162. end
  163. 15 def run
  164. 32 return unless @handlers.last
  165. 24 env = @handlers.last.env
  166. 24 connect(env) do |session|
  167. 56 requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
  168. 24 if env.request.stream_response?
  169. 8 requests.each do |request|
  170. 8 request.response_on_data = env.request.on_data
  171. end
  172. end
  173. 24 responses = session.request(*requests)
  174. 24 Array(responses).each_with_index do |response, index|
  175. 32 handler = @handlers[index]
  176. 32 handler.on_response.call(response)
  177. 32 handler.on_complete.call(handler.env) if handler.on_complete
  178. end
  179. end
  180. rescue ::HTTPX::TimeoutError => e
  181. raise Faraday::TimeoutError, e
  182. end
  183. # from Faraday::Adapter#connection
  184. 15 def connection(env)
  185. 24 conn = build_connection(env)
  186. 24 return conn unless block_given?
  187. 24 yield conn
  188. end
  189. 15 private
  190. # from Faraday::Adapter#request_timeout
  191. 15 def request_timeout(type, options)
  192. 168 key = Faraday::Adapter::TIMEOUT_KEYS[type]
  193. 168 options[key] || options[:timeout]
  194. end
  195. end
  196. 15 self.supports_parallel = true
  197. 15 class << self
  198. 15 def setup_parallel_manager(options = {})
  199. 32 ParallelManager.new(options)
  200. end
  201. end
  202. 15 def call(env)
  203. 295 super
  204. 295 if parallel?(env)
  205. 32 handler = env[:parallel_manager].enqueue(env)
  206. 32 handler.on_response do |response|
  207. 32 if response.is_a?(::HTTPX::Response)
  208. 24 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  209. 24 response_headers.merge!(response.headers)
  210. end
  211. else
  212. 7 env[:error] = response.error
  213. 8 save_response(env, 0, "", {}, nil)
  214. end
  215. end
  216. 28 return handler
  217. end
  218. 263 response = connect_and_request(env)
  219. 217 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  220. 217 response_headers.merge!(response.headers)
  221. end
  222. 217 @app.call(env)
  223. end
  224. 15 private
  225. 15 def connect_and_request(env)
  226. 263 connect(env) do |session|
  227. 263 request = session.build_request(*build_request(env))
  228. 263 request.response_on_data = env.request.on_data if env.request.stream_response?
  229. 263 response = session.request(request)
  230. # do not call #raise_for_status for HTTP 4xx or 5xx, as faraday has a middleware for that.
  231. 263 response.raise_for_status unless response.is_a?(::HTTPX::Response)
  232. 217 response
  233. end
  234. rescue ::HTTPX::TimeoutError => e
  235. 24 raise Faraday::TimeoutError, e
  236. end
  237. 15 def parallel?(env)
  238. 295 env[:parallel_manager]
  239. end
  240. end
  241. 15 register_middleware httpx: HTTPX
  242. end
  243. end

lib/httpx/adapters/sentry.rb

100.0% lines covered

62 relevant lines. 62 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "sentry-ruby"
  3. 6 module HTTPX::Plugins
  4. 6 module Sentry
  5. 6 module Tracer
  6. 6 module_function
  7. 6 def call(request)
  8. 115 sentry_span = start_sentry_span
  9. 115 return unless sentry_span
  10. 115 set_sentry_trace_header(request, sentry_span)
  11. 115 request.on(:response, &method(:finish_sentry_span).curry(3)[sentry_span, request])
  12. end
  13. 6 def start_sentry_span
  14. 115 return unless ::Sentry.initialized? && (span = ::Sentry.get_current_scope.get_span)
  15. 115 return if span.sampled == false
  16. 115 span.start_child(op: "httpx.client", start_timestamp: ::Sentry.utc_now.to_f)
  17. end
  18. 6 def set_sentry_trace_header(request, sentry_span)
  19. 115 return unless sentry_span
  20. 115 config = ::Sentry.configuration
  21. 115 url = request.uri.to_s
  22. 230 return unless config.propagate_traces && config.trace_propagation_targets.any? { |target| url.match?(target) }
  23. 115 trace = ::Sentry.get_current_client.generate_sentry_trace(sentry_span)
  24. 115 request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
  25. end
  26. 6 def finish_sentry_span(span, request, response)
  27. 121 return unless ::Sentry.initialized?
  28. 121 record_sentry_breadcrumb(request, response)
  29. 121 record_sentry_span(request, response, span)
  30. end
  31. 6 def record_sentry_breadcrumb(req, res)
  32. 121 return unless ::Sentry.configuration.breadcrumbs_logger.include?(:http_logger)
  33. 121 request_info = extract_request_info(req)
  34. 121 data = if res.is_a?(HTTPX::ErrorResponse)
  35. 12 { error: res.error.message, **request_info }
  36. else
  37. 109 { status: res.status, **request_info }
  38. end
  39. 121 crumb = ::Sentry::Breadcrumb.new(
  40. level: :info,
  41. category: "httpx",
  42. type: :info,
  43. data: data
  44. )
  45. 121 ::Sentry.add_breadcrumb(crumb)
  46. end
  47. 6 def record_sentry_span(req, res, sentry_span)
  48. 121 return unless sentry_span
  49. 121 request_info = extract_request_info(req)
  50. 121 sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
  51. 121 if res.is_a?(HTTPX::ErrorResponse)
  52. 12 sentry_span.set_data(:error, res.error.message)
  53. else
  54. 109 sentry_span.set_data(:status, res.status)
  55. end
  56. 121 sentry_span.set_timestamp(::Sentry.utc_now.to_f)
  57. end
  58. 6 def extract_request_info(req)
  59. 242 uri = req.uri
  60. result = {
  61. 242 method: req.verb,
  62. }
  63. 242 if ::Sentry.configuration.send_default_pii
  64. 24 uri += "?#{req.query}" unless req.query.empty?
  65. 24 result[:body] = req.body.to_s unless req.body.empty? || req.body.unbounded_body?
  66. end
  67. 242 result[:url] = uri.to_s
  68. 242 result
  69. end
  70. end
  71. 6 module RequestMethods
  72. 6 def __sentry_enable_trace!
  73. 121 return if @__sentry_enable_trace
  74. 115 Tracer.call(self)
  75. 115 @__sentry_enable_trace = true
  76. end
  77. end
  78. 6 module ConnectionMethods
  79. 6 def send(request)
  80. 121 request.__sentry_enable_trace!
  81. 121 super
  82. end
  83. end
  84. end
  85. end
  86. 6 Sentry.register_patch(:httpx) do
  87. 30 sentry_session = HTTPX.plugin(HTTPX::Plugins::Sentry)
  88. 30 HTTPX.send(:remove_const, :Session)
  89. 30 HTTPX.send(:const_set, :Session, sentry_session.class)
  90. end

lib/httpx/adapters/webmock.rb

100.0% lines covered

91 relevant lines. 91 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module WebMock
  3. 8 module HttpLibAdapters
  4. 8 require "net/http/status"
  5. 8 HTTP_REASONS = Net::HTTP::STATUS_CODES
  6. #
  7. # HTTPX plugin for webmock.
  8. #
  9. # Requests are "hijacked" at the session, before they're distributed to a connection.
  10. #
  11. 8 module Plugin
  12. 8 class << self
  13. 8 def build_webmock_request_signature(request)
  14. 188 uri = WebMock::Util::URI.heuristic_parse(request.uri)
  15. 188 uri.query = request.query
  16. 188 uri.path = uri.normalized_path.gsub("[^:]//", "/")
  17. 188 WebMock::RequestSignature.new(
  18. request.verb.downcase.to_sym,
  19. uri.to_s,
  20. body: request.body.to_s,
  21. headers: request.headers.to_h
  22. )
  23. end
  24. 8 def build_webmock_response(_request, response)
  25. 6 webmock_response = WebMock::Response.new
  26. 6 webmock_response.status = [response.status, HTTP_REASONS[response.status]]
  27. 6 webmock_response.body = response.body.to_s
  28. 6 webmock_response.headers = response.headers.to_h
  29. 6 webmock_response
  30. end
  31. 8 def build_from_webmock_response(request, webmock_response)
  32. 158 return build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
  33. 140 return build_error_response(request, webmock_response.exception) if webmock_response.exception
  34. 133 request.options.response_class.new(request,
  35. webmock_response.status[0],
  36. "2.0",
  37. webmock_response.headers).tap do |res|
  38. 133 res.mocked = true
  39. end
  40. end
  41. 8 def build_error_response(request, exception)
  42. 25 HTTPX::ErrorResponse.new(request, exception)
  43. end
  44. end
  45. 8 module InstanceMethods
  46. 8 private
  47. 8 def do_init_connection(connection, selector)
  48. 170 super
  49. 170 connection.once(:unmock_connection) do
  50. 24 unless connection.addresses
  51. # reset Happy Eyeballs, fail early
  52. 24 connection.sibling = nil
  53. 24 deselect_connection(connection, selector)
  54. end
  55. 24 resolve_connection(connection, selector)
  56. end
  57. end
  58. end
  59. 8 module ResponseMethods
  60. 8 attr_accessor :mocked
  61. 8 def initialize(*)
  62. 157 super
  63. 157 @mocked = false
  64. end
  65. end
  66. 8 module ResponseBodyMethods
  67. 8 def decode_chunk(chunk)
  68. 96 return chunk if @response.mocked
  69. 42 super
  70. end
  71. end
  72. 8 module ConnectionMethods
  73. 8 def initialize(*)
  74. 170 super
  75. 170 @mocked = true
  76. end
  77. 8 def open?
  78. 194 return true if @mocked
  79. 24 super
  80. end
  81. 8 def interests
  82. 276 return if @mocked
  83. 246 super
  84. end
  85. 8 def terminate
  86. 145 force_reset
  87. end
  88. 8 def send(request)
  89. 188 request_signature = Plugin.build_webmock_request_signature(request)
  90. 188 WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
  91. 188 if (mock_response = WebMock::StubRegistry.instance.response_for_request(request_signature))
  92. 158 response = Plugin.build_from_webmock_response(request, mock_response)
  93. 158 WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, request_signature, mock_response)
  94. 158 log { "mocking #{request.uri} with #{mock_response.inspect}" }
  95. 158 request.transition(:headers)
  96. 158 request.transition(:body)
  97. 158 request.transition(:trailers)
  98. 158 request.transition(:done)
  99. 158 request.response = response
  100. 158 request.emit(:response, response)
  101. 158 request_signature.headers = request.headers.to_h
  102. 158 response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)
  103. 30 elsif WebMock.net_connect_allowed?(request_signature.uri)
  104. 24 if WebMock::CallbackRegistry.any_callbacks?
  105. 6 request.on(:response) do |resp|
  106. 6 unless resp.is_a?(HTTPX::ErrorResponse)
  107. 6 webmock_response = Plugin.build_webmock_response(request, resp)
  108. 6 WebMock::CallbackRegistry.invoke_callbacks(
  109. { lib: :httpx, real_request: true }, request_signature,
  110. webmock_response
  111. )
  112. end
  113. end
  114. end
  115. 24 @mocked = false
  116. 24 emit(:unmock_connection, self)
  117. 24 super
  118. else
  119. 6 raise WebMock::NetConnectNotAllowedError, request_signature
  120. end
  121. end
  122. end
  123. end
  124. 8 class HttpxAdapter < HttpLibAdapter
  125. 8 adapter_for :httpx
  126. 8 class << self
  127. 8 def enable!
  128. 370 @original_session ||= HTTPX::Session
  129. 370 webmock_session = HTTPX.plugin(Plugin)
  130. 370 HTTPX.send(:remove_const, :Session)
  131. 370 HTTPX.send(:const_set, :Session, webmock_session.class)
  132. end
  133. 8 def disable!
  134. 370 return unless @original_session
  135. 362 HTTPX.send(:remove_const, :Session)
  136. 362 HTTPX.send(:const_set, :Session, @original_session)
  137. end
  138. end
  139. end
  140. end
  141. end

lib/httpx/altsvc.rb

96.39% lines covered

83 relevant lines. 80 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "strscan"
  3. 26 module HTTPX
  4. 26 module AltSvc
  5. # makes connections able to accept requests destined to primary service.
  6. 26 module ConnectionMixin
  7. 26 using URIExtensions
  8. 26 def send(request)
  9. 8 request.headers["alt-used"] = @origin.authority if @parser && !@write_buffer.full? && match_altsvcs?(request.uri)
  10. 8 super
  11. end
  12. 26 def match?(uri, options)
  13. 8 return false if !used? && (@state == :closing || @state == :closed)
  14. 8 match_altsvcs?(uri) && match_altsvc_options?(uri, options)
  15. end
  16. 26 private
  17. # checks if this is connection is an alternative service of
  18. # +uri+
  19. 26 def match_altsvcs?(uri)
  20. 24 @origins.any? { |origin| altsvc_match?(uri, origin) } ||
  21. AltSvc.cached_altsvc(@origin).any? do |altsvc|
  22. origin = altsvc["origin"]
  23. altsvc_match?(origin, uri.origin)
  24. end
  25. end
  26. 26 def match_altsvc_options?(uri, options)
  27. 8 return @options == options unless @options.ssl.all? do |k, v|
  28. 8 v == (k == :hostname ? uri.host : options.ssl[k])
  29. end
  30. 8 @options.options_equals?(options, Options::REQUEST_BODY_IVARS + %i[@ssl])
  31. end
  32. 26 def altsvc_match?(uri, other_uri)
  33. 16 other_uri = URI(other_uri)
  34. 16 uri.origin == other_uri.origin || begin
  35. 7 case uri.scheme
  36. when "h2"
  37. (other_uri.scheme == "https" || other_uri.scheme == "h2") &&
  38. uri.host == other_uri.host &&
  39. uri.port == other_uri.port
  40. else
  41. 8 false
  42. end
  43. end
  44. end
  45. end
  46. 26 @altsvc_mutex = Thread::Mutex.new
  47. 47 @altsvcs = Hash.new { |h, k| h[k] = [] }
  48. 26 module_function
  49. 26 def cached_altsvc(origin)
  50. 40 now = Utils.now
  51. 40 @altsvc_mutex.synchronize do
  52. 40 lookup(origin, now)
  53. end
  54. end
  55. 26 def cached_altsvc_set(origin, entry)
  56. 24 now = Utils.now
  57. 24 @altsvc_mutex.synchronize do
  58. 24 return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
  59. 24 entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
  60. 24 @altsvcs[origin] << entry
  61. 24 entry
  62. end
  63. end
  64. 26 def lookup(origin, ttl)
  65. 40 return [] unless @altsvcs.key?(origin)
  66. 28 @altsvcs[origin] = @altsvcs[origin].select do |entry|
  67. 24 !entry.key?("TTL") || entry["TTL"] > ttl
  68. end
  69. 48 @altsvcs[origin].reject { |entry| entry["noop"] }
  70. end
  71. 26 def emit(request, response)
  72. 7728 return unless response.respond_to?(:headers)
  73. # Alt-Svc
  74. 7702 return unless response.headers.key?("alt-svc")
  75. 80 origin = request.origin
  76. 80 host = request.uri.host
  77. 80 altsvc = response.headers["alt-svc"]
  78. # https://datatracker.ietf.org/doc/html/rfc7838#section-3
  79. # A field value containing the special value "clear" indicates that the
  80. # origin requests all alternatives for that origin to be invalidated
  81. # (including those specified in the same response, in case of an
  82. # invalid reply containing both "clear" and alternative services).
  83. 80 if altsvc == "clear"
  84. 8 @altsvc_mutex.synchronize do
  85. 8 @altsvcs[origin].clear
  86. end
  87. 7 return
  88. end
  89. 72 parse(altsvc) do |alt_origin, alt_params|
  90. 8 alt_origin.host ||= host
  91. 8 yield(alt_origin, origin, alt_params)
  92. end
  93. end
  94. 26 def parse(altsvc)
  95. 184 return enum_for(__method__, altsvc) unless block_given?
  96. 128 scanner = StringScanner.new(altsvc)
  97. 134 until scanner.eos?
  98. 128 alt_service = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  99. 128 alt_params = []
  100. 128 loop do
  101. 152 alt_param = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  102. 152 alt_params << alt_param.strip if alt_param
  103. 152 scanner.skip(/;/)
  104. 152 break if scanner.eos? || scanner.scan(/ *, */)
  105. end
  106. 256 alt_params = Hash[alt_params.map { |field| field.split("=", 2) }]
  107. 128 alt_proto, alt_authority = alt_service.split("=", 2)
  108. 128 alt_origin = parse_altsvc_origin(alt_proto, alt_authority)
  109. 128 return unless alt_origin
  110. 48 yield(alt_origin, alt_params.merge("proto" => alt_proto))
  111. end
  112. end
  113. 26 def parse_altsvc_scheme(alt_proto)
  114. 135 case alt_proto
  115. when "h2c"
  116. 8 "http"
  117. when "h2"
  118. 56 "https"
  119. end
  120. end
  121. 26 def parse_altsvc_origin(alt_proto, alt_origin)
  122. 128 alt_scheme = parse_altsvc_scheme(alt_proto)
  123. 128 return unless alt_scheme
  124. 48 alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
  125. 48 URI.parse("#{alt_scheme}://#{alt_origin}")
  126. end
  127. end
  128. end

lib/httpx/buffer.rb

100.0% lines covered

21 relevant lines. 21 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "forwardable"
  3. 26 module HTTPX
  4. # Internal class to abstract a string buffer, by wrapping a string and providing the
  5. # minimum possible API and functionality required.
  6. #
  7. # buffer = Buffer.new(640)
  8. # buffer.full? #=> false
  9. # buffer << "aa"
  10. # buffer.capacity #=> 638
  11. #
  12. 26 class Buffer
  13. 26 extend Forwardable
  14. 26 def_delegator :@buffer, :<<
  15. 26 def_delegator :@buffer, :to_s
  16. 26 def_delegator :@buffer, :to_str
  17. 26 def_delegator :@buffer, :empty?
  18. 26 def_delegator :@buffer, :bytesize
  19. 26 def_delegator :@buffer, :clear
  20. 26 def_delegator :@buffer, :replace
  21. 26 attr_reader :limit
  22. 26 def initialize(limit)
  23. 21109 @buffer = "".b
  24. 21109 @limit = limit
  25. end
  26. 26 def full?
  27. 11486772 @buffer.bytesize >= @limit
  28. end
  29. 26 def capacity
  30. 12 @limit - @buffer.bytesize
  31. end
  32. 26 def shift!(fin)
  33. 21283 @buffer = @buffer.byteslice(fin..-1) || "".b
  34. end
  35. end
  36. end

lib/httpx/callbacks.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Callbacks
  4. 26 def on(type, &action)
  5. 350963 callbacks(type) << action
  6. 350963 action
  7. end
  8. 26 def once(type, &block)
  9. 187447 on(type) do |*args, &callback|
  10. 95739 block.call(*args, &callback)
  11. 95675 :delete
  12. end
  13. end
  14. 26 def emit(type, *args)
  15. 124409 log { "emit #{type.inspect} callbacks" } if respond_to?(:log)
  16. 269246 callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
  17. end
  18. 26 def callbacks_for?(type)
  19. 2998 @callbacks.key?(type) && @callbacks[type].any?
  20. end
  21. 26 protected
  22. 26 def callbacks(type = nil)
  23. 478495 return @callbacks unless type
  24. 685880 @callbacks ||= Hash.new { |h, k| h[k] = [] }
  25. 478408 @callbacks[type]
  26. end
  27. end
  28. end

lib/httpx/chainable.rb

95.65% lines covered

46 relevant lines. 44 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. # Session mixin, implements most of the APIs that the users call.
  4. # delegates to a default session when extended.
  5. 26 module Chainable
  6. 26 %w[head get post put delete trace options connect patch].each do |meth|
  7. 225 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  8. 9 def #{meth}(*uri, **options) # def get(*uri, **options)
  9. 18 request("#{meth.upcase}", uri, **options) # request("GET", uri, **options)
  10. end # end
  11. MOD
  12. end
  13. # delegates to the default session (see HTTPX::Session#request).
  14. 26 def request(*args, **options)
  15. 2606 branch(default_options).request(*args, **options)
  16. end
  17. 26 def accept(type)
  18. 16 with(headers: { "accept" => String(type) })
  19. end
  20. # delegates to the default session (see HTTPX::Session#wrap).
  21. 26 def wrap(&blk)
  22. 89 branch(default_options).wrap(&blk)
  23. end
  24. # returns a new instance loaded with the +pl+ plugin and +options+.
  25. 26 def plugin(pl, options = nil, &blk)
  26. 5059 klass = is_a?(S) ? self.class : Session
  27. 5059 klass = Class.new(klass)
  28. 5059 klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
  29. 5059 klass.plugin(pl, options, &blk).new
  30. end
  31. # returns a new instance loaded with +options+.
  32. 26 def with(options, &blk)
  33. 2661 branch(default_options.merge(options), &blk)
  34. end
  35. 26 private
  36. # returns default instance of HTTPX::Options.
  37. 26 def default_options
  38. 10471 @options || Session.default_options
  39. end
  40. # returns a default instance of HTTPX::Session.
  41. 26 def branch(options, &blk)
  42. 5340 return self.class.new(options, &blk) if is_a?(S)
  43. 3099 Session.new(options, &blk)
  44. end
  45. 26 def method_missing(meth, *args, **options, &blk)
  46. 652 case meth
  47. when /\Awith_(.+)/
  48. 729 option = Regexp.last_match(1)
  49. 729 return super unless option
  50. 729 with(option.to_sym => args.first || options)
  51. when /\Aon_(.+)/
  52. 9 callback = Regexp.last_match(1)
  53. 6 return super unless %w[
  54. connection_opened connection_closed
  55. request_error
  56. request_started request_body_chunk request_completed
  57. response_started response_body_chunk response_completed
  58. 2 ].include?(callback)
  59. 9 warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
  60. 1 "Use `HTTPX.plugin(:callbacks).#{meth}` instead."
  61. 9 plugin(:callbacks).__send__(meth, *args, **options, &blk)
  62. else
  63. super
  64. end
  65. end
  66. 26 def respond_to_missing?(meth, *)
  67. 49 case meth
  68. when /\Awith_(.+)/
  69. 40 option = Regexp.last_match(1)
  70. 40 default_options.respond_to?(option) || super
  71. when /\Aon_(.+)/
  72. 16 callback = Regexp.last_match(1)
  73. 12 %w[
  74. connection_opened connection_closed
  75. request_error
  76. request_started request_body_chunk request_completed
  77. response_started response_body_chunk response_completed
  78. 3 ].include?(callback) || super
  79. else
  80. super
  81. end
  82. end
  83. end
  84. 26 extend Chainable
  85. end

lib/httpx/connection.rb

92.47% lines covered

478 relevant lines. 442 lines covered and 36 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "resolv"
  3. 26 require "forwardable"
  4. 26 require "httpx/io"
  5. 26 require "httpx/buffer"
  6. 26 module HTTPX
  7. # The Connection can be watched for IO events.
  8. #
  9. # It contains the +io+ object to read/write from, and knows what to do when it can.
  10. #
  11. # It defers connecting until absolutely necessary. Connection should be triggered from
  12. # the IO selector (until then, any request will be queued).
  13. #
  14. # A connection boots up its parser after connection is established. All pending requests
  15. # will be redirected there after connection.
  16. #
  17. # A connection can be prevented from closing by the parser, that is, if there are pending
  18. # requests. This will signal that the connection was prematurely closed, due to a possible
  19. # number of conditions:
  20. #
  21. # * Remote peer closed the connection ("Connection: close");
  22. # * Remote peer doesn't support pipelining;
  23. #
  24. # A connection may also route requests for a different host for which the +io+ was connected
  25. # to, provided that the IP is the same and the port and scheme as well. This will allow to
  26. # share the same socket to send HTTP/2 requests to different hosts.
  27. #
  28. 26 class Connection
  29. 26 extend Forwardable
  30. 26 include Loggable
  31. 26 include Callbacks
  32. 26 using URIExtensions
  33. 26 require "httpx/connection/http2"
  34. 26 require "httpx/connection/http1"
  35. 26 def_delegator :@io, :closed?
  36. 26 def_delegator :@write_buffer, :empty?
  37. 26 attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
  38. 26 attr_writer :current_selector
  39. 26 attr_accessor :current_session, :family
  40. 26 protected :sibling
  41. 26 def initialize(uri, options)
  42. 7164 @current_session = @current_selector = @sibling = @coalesced_connection = nil
  43. 7164 @exhausted = @cloned = @main_sibling = false
  44. 7164 @options = Options.new(options)
  45. 7164 @type = initialize_type(uri, @options)
  46. 7164 @origins = [uri.origin]
  47. 7164 @origin = Utils.to_uri(uri.origin)
  48. 7164 @window_size = @options.window_size
  49. 7164 @read_buffer = Buffer.new(@options.buffer_size)
  50. 7164 @write_buffer = Buffer.new(@options.buffer_size)
  51. 7164 @pending = []
  52. 7164 on(:error, &method(:on_error))
  53. 7164 if @options.io
  54. # if there's an already open IO, get its
  55. # peer address, and force-initiate the parser
  56. 66 transition(:already_open)
  57. 66 @io = build_socket
  58. 66 parser
  59. else
  60. 7098 transition(:idle)
  61. end
  62. 7164 on(:close) do
  63. 7337 next if @exhausted # it'll reset
  64. # may be called after ":close" above, so after the connection has been checked back in.
  65. # next unless @current_session
  66. 7329 next unless @current_session
  67. 7329 @current_session.deselect_connection(self, @current_selector, @cloned)
  68. end
  69. 7164 on(:terminate) do
  70. 2693 next if @exhausted # it'll reset
  71. 2685 current_session = @current_session
  72. 2685 current_selector = @current_selector
  73. # may be called after ":close" above, so after the connection has been checked back in.
  74. 2685 next unless current_session && current_selector
  75. 16 current_session.deselect_connection(self, current_selector)
  76. end
  77. 7164 on(:altsvc) do |alt_origin, origin, alt_params|
  78. 8 build_altsvc_connection(alt_origin, origin, alt_params)
  79. end
  80. 7164 @inflight = 0
  81. 7164 @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
  82. 7164 self.addresses = @options.addresses if @options.addresses
  83. end
  84. 26 def peer
  85. 14478 @origin
  86. end
  87. # this is a semi-private method, to be used by the resolver
  88. # to initiate the io object.
  89. 26 def addresses=(addrs)
  90. 6932 if @io
  91. 223 @io.add_addresses(addrs)
  92. else
  93. 6709 @io = build_socket(addrs)
  94. end
  95. end
  96. 26 def addresses
  97. 14300 @io && @io.addresses
  98. end
  99. 26 def match?(uri, options)
  100. 2023 return false if !used? && (@state == :closing || @state == :closed)
  101. 300 (
  102. 1755 @origins.include?(uri.origin) &&
  103. # if there is more than one origin to match, it means that this connection
  104. # was the result of coalescing. To prevent blind trust in the case where the
  105. # origin came from an ORIGIN frame, we're going to verify the hostname with the
  106. # SSL certificate
  107. 1821 (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
  108. ) && @options == options
  109. end
  110. 26 def expired?
  111. return false unless @io
  112. @io.expired?
  113. end
  114. 26 def mergeable?(connection)
  115. 314 return false if @state == :closing || @state == :closed || !@io
  116. 64 return false unless connection.addresses
  117. 2 (
  118. 64 (open? && @origin == connection.origin) ||
  119. 64 !(@io.addresses & (connection.addresses || [])).empty?
  120. ) && @options == connection.options
  121. end
  122. # coalescable connections need to be mergeable!
  123. # but internally, #mergeable? is called before #coalescable?
  124. 26 def coalescable?(connection)
  125. 27 if @io.protocol == "h2" &&
  126. @origin.scheme == "https" &&
  127. connection.origin.scheme == "https" &&
  128. @io.can_verify_peer?
  129. 13 @io.verify_hostname(connection.origin.host)
  130. else
  131. 14 @origin == connection.origin
  132. end
  133. end
  134. 26 def create_idle(options = {})
  135. self.class.new(@origin, @options.merge(options))
  136. end
  137. 26 def merge(connection)
  138. 34 @origins |= connection.instance_variable_get(:@origins)
  139. 37 if connection.ssl_session
  140. 14 @ssl_session = connection.ssl_session
  141. 2 @io.session_new_cb do |sess|
  142. 21 @ssl_session = sess
  143. 14 end if @io
  144. end
  145. 37 connection.purge_pending do |req|
  146. 7 send(req)
  147. end
  148. end
  149. 26 def purge_pending(&block)
  150. 37 pendings = []
  151. 37 if @parser
  152. 21 @inflight -= @parser.pending.size
  153. 24 pendings << @parser.pending
  154. end
  155. 37 pendings << @pending
  156. 37 pendings.each do |pending|
  157. 61 pending.reject!(&block)
  158. end
  159. end
  160. 26 def io_connected?
  161. return @coalesced_connection.io_connected? if @coalesced_connection
  162. @io && @io.state == :connected
  163. end
  164. 26 def connecting?
  165. 11520717 @state == :idle
  166. end
  167. 26 def inflight?
  168. 2807 @parser && (
  169. # parser may be dealing with other requests (possibly started from a different fiber)
  170. 2350 !@parser.empty? ||
  171. # connection may be doing connection termination handshake
  172. !@write_buffer.empty?
  173. )
  174. end
  175. 26 def interests
  176. # connecting
  177. 11509447 if connecting?
  178. 10969 connect
  179. 10969 return @io.interests if connecting?
  180. end
  181. # if the write buffer is full, we drain it
  182. 11499048 return :w unless @write_buffer.empty?
  183. 11458775 return @parser.interests if @parser
  184. 22 nil
  185. rescue StandardError => e
  186. emit(:error, e)
  187. nil
  188. end
  189. 26 def to_io
  190. 22031 @io.to_io
  191. end
  192. 26 def call
  193. 19527 case @state
  194. when :idle
  195. 10252 connect
  196. 10236 consume
  197. when :closed
  198. return
  199. when :closing
  200. consume
  201. transition(:closed)
  202. when :open
  203. 10817 consume
  204. end
  205. 8251 nil
  206. rescue StandardError => e
  207. 22 emit(:error, e)
  208. 22 raise e
  209. end
  210. 26 def close
  211. 2645 transition(:active) if @state == :inactive
  212. 2645 @parser.close if @parser
  213. end
  214. 26 def terminate
  215. 2645 @connected_at = nil if @state == :closed
  216. 2645 close
  217. end
  218. # bypasses the state machine to force closing of connections still connecting.
  219. # **only** used for Happy Eyeballs v2.
  220. 26 def force_reset(cloned = false)
  221. 292 @state = :closing
  222. 292 @cloned = cloned
  223. 292 transition(:closed)
  224. end
  225. 26 def reset
  226. 7264 return if @state == :closing || @state == :closed
  227. 7186 transition(:closing)
  228. 7186 transition(:closed)
  229. end
  230. 26 def send(request)
  231. 8698 return @coalesced_connection.send(request) if @coalesced_connection
  232. 8692 if @parser && !@write_buffer.full?
  233. 419 if @response_received_at && @keep_alive_timeout &&
  234. Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
  235. # when pushing a request into an existing connection, we have to check whether there
  236. # is the possibility that the connection might have extended the keep alive timeout.
  237. # for such cases, we want to ping for availability before deciding to shovel requests.
  238. 10 log(level: 3) { "keep alive timeout expired, pinging connection..." }
  239. 10 @pending << request
  240. 10 transition(:active) if @state == :inactive
  241. 10 parser.ping
  242. 9 return
  243. end
  244. 409 send_request_to_parser(request)
  245. else
  246. 8273 @pending << request
  247. end
  248. end
  249. 26 def timeout
  250. 11387876 return if @state == :closed || @state == :inactive
  251. 11387876 return @timeout if @timeout
  252. 11375671 return @options.timeout[:connect_timeout] if @state == :idle
  253. 11375671 @options.timeout[:operation_timeout]
  254. end
  255. 26 def idling
  256. 783 purge_after_closed
  257. 783 @write_buffer.clear
  258. 783 transition(:idle)
  259. 783 @parser = nil if @parser
  260. end
  261. 26 def used?
  262. 2242 @connected_at
  263. end
  264. 26 def deactivate
  265. 344 transition(:inactive)
  266. end
  267. 26 def open?
  268. 7007 @state == :open || @state == :inactive
  269. end
  270. 26 def handle_socket_timeout(interval)
  271. 32 error = OperationTimeoutError.new(interval, "timed out while waiting on select")
  272. 32 error.set_backtrace(caller)
  273. 32 on_error(error)
  274. end
  275. 26 def coalesced_connection=(connection)
  276. 13 @coalesced_connection = connection
  277. 13 close_sibling
  278. 13 connection.merge(self)
  279. end
  280. 26 def sibling=(connection)
  281. 24 @sibling = connection
  282. 24 return unless connection
  283. @main_sibling = connection.sibling.nil?
  284. return unless @main_sibling
  285. connection.sibling = self
  286. end
  287. 26 def handle_connect_error(error)
  288. 268 @connect_error = error
  289. 268 return handle_error(error) unless @sibling && @sibling.connecting?
  290. @sibling.merge(self)
  291. force_reset(true)
  292. end
  293. 26 def disconnect
  294. 7459 return unless @current_session && @current_selector
  295. 7337 emit(:close)
  296. 7321 @current_session = nil
  297. 7321 @current_selector = nil
  298. end
  299. 26 private
  300. 26 def connect
  301. 20212 transition(:open)
  302. end
  303. 26 def consume
  304. 24189 return unless @io
  305. 24189 catch(:called) do
  306. 24189 epiped = false
  307. 24189 loop do
  308. # connection may have
  309. 43858 return if @state == :idle
  310. 40708 parser.consume
  311. # we exit if there's no more requests to process
  312. #
  313. # this condition takes into account:
  314. #
  315. # * the number of inflight requests
  316. # * the number of pending requests
  317. # * whether the write buffer has bytes (i.e. for close handshake)
  318. 40692 if @pending.empty? && @inflight.zero? && @write_buffer.empty?
  319. 2754 log(level: 3) { "NO MORE REQUESTS..." }
  320. 2738 return
  321. end
  322. 37954 @timeout = @current_timeout
  323. 37954 read_drained = false
  324. 37954 write_drained = nil
  325. #
  326. # tight read loop.
  327. #
  328. # read as much of the socket as possible.
  329. #
  330. # this tight loop reads all the data it can from the socket and pipes it to
  331. # its parser.
  332. #
  333. 7252 loop do
  334. 49572 siz = @io.read(@window_size, @read_buffer)
  335. 49687 log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
  336. 49572 unless siz
  337. 18 @write_buffer.clear
  338. 18 ex = EOFError.new("descriptor closed")
  339. 18 ex.set_backtrace(caller)
  340. 18 on_error(ex)
  341. 18 return
  342. end
  343. # socket has been drained. mark and exit the read loop.
  344. 49554 if siz.zero?
  345. 10311 read_drained = @read_buffer.empty?
  346. 10311 epiped = false
  347. 10311 break
  348. end
  349. 39243 parser << @read_buffer.to_s
  350. # continue reading if possible.
  351. 34960 break if interests == :w && !epiped
  352. # exit the read loop if connection is preparing to be closed
  353. 27780 break if @state == :closing || @state == :closed
  354. # exit #consume altogether if all outstanding requests have been dealt with
  355. 27765 return if @pending.empty? && @inflight.zero?
  356. 37954 end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
  357. #
  358. # tight write loop.
  359. #
  360. # flush as many bytes as the sockets allow.
  361. #
  362. 5963 loop do
  363. # buffer has been drainned, mark and exit the write loop.
  364. 23487 if @write_buffer.empty?
  365. # we only mark as drained on the first loop
  366. 2887 write_drained = write_drained.nil? && @inflight.positive?
  367. 2887 break
  368. end
  369. 2017 begin
  370. 20600 siz = @io.write(@write_buffer)
  371. rescue Errno::EPIPE
  372. # this can happen if we still have bytes in the buffer to send to the server, but
  373. # the server wants to respond immediately with some message, or an error. An example is
  374. # when one's uploading a big file to an unintended endpoint, and the server stops the
  375. # consumption, and responds immediately with an authorization of even method not allowed error.
  376. # at this point, we have to let the connection switch to read-mode.
  377. 11 log(level: 2) { "pipe broken, could not flush buffer..." }
  378. 11 epiped = true
  379. 11 read_drained = false
  380. 11 break
  381. end
  382. 20666 log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
  383. 20580 unless siz
  384. @write_buffer.clear
  385. ex = EOFError.new("descriptor closed")
  386. ex.set_backtrace(caller)
  387. on_error(ex)
  388. return
  389. end
  390. # socket closed for writing. mark and exit the write loop.
  391. 20580 if siz.zero?
  392. 24 write_drained = !@write_buffer.empty?
  393. 24 break
  394. end
  395. # exit write loop if marked to consume from peer, or is closing.
  396. 20556 break if interests == :r || @state == :closing || @state == :closed
  397. 3092 write_drained = false
  398. 30697 end unless (ints = interests) == :r
  399. 30688 send_pending if @state == :open
  400. # return if socket is drained
  401. 30688 next unless (ints != :r || read_drained) && (ints != :w || write_drained)
  402. # gotta go back to the event loop. It happens when:
  403. #
  404. # * the socket is drained of bytes or it's not the interest of the conn to read;
  405. # * theres nothing more to write, or it's not in the interest of the conn to write;
  406. 11058 log(level: 3) { "(#{ints}): WAITING FOR EVENTS..." }
  407. 11019 return
  408. end
  409. end
  410. end
  411. 26 def send_pending
  412. 79539 while !@write_buffer.full? && (request = @pending.shift)
  413. 18425 send_request_to_parser(request)
  414. end
  415. end
  416. 26 def parser
  417. 107203 @parser ||= build_parser
  418. end
  419. 26 def send_request_to_parser(request)
  420. 17942 @inflight += 1
  421. 18834 request.peer_address = @io.ip
  422. 18834 parser.send(request)
  423. 18834 set_request_timeouts(request)
  424. 18834 return unless @state == :inactive
  425. 7 transition(:active)
  426. end
  427. 26 def build_parser(protocol = @io.protocol)
  428. 7216 parser = self.class.parser_type(protocol).new(@write_buffer, @options)
  429. 7216 set_parser_callbacks(parser)
  430. 7216 parser
  431. end
  432. 26 def set_parser_callbacks(parser)
  433. 7327 parser.on(:response) do |request, response|
  434. 7720 AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
  435. 8 emit(:altsvc, alt_origin, origin, alt_params)
  436. end
  437. 7720 @response_received_at = Utils.now
  438. 6904 @inflight -= 1
  439. 7720 request.emit(:response, response)
  440. end
  441. 7327 parser.on(:altsvc) do |alt_origin, origin, alt_params|
  442. emit(:altsvc, alt_origin, origin, alt_params)
  443. end
  444. 7327 parser.on(:pong, &method(:send_pending))
  445. 7327 parser.on(:promise) do |request, stream|
  446. 24 request.emit(:promise, parser, stream)
  447. end
  448. 7327 parser.on(:exhausted) do
  449. 8 @exhausted = true
  450. 8 current_session = @current_session
  451. 8 current_selector = @current_selector
  452. begin
  453. 8 parser.close
  454. 8 @pending.concat(parser.pending)
  455. ensure
  456. 8 @current_session = current_session
  457. 8 @current_selector = current_selector
  458. end
  459. 7 case @state
  460. when :closed
  461. 8 idling
  462. 8 @exhausted = false
  463. when :closing
  464. once(:closed) do
  465. idling
  466. @exhausted = false
  467. end
  468. end
  469. end
  470. 7327 parser.on(:origin) do |origin|
  471. @origins |= [origin]
  472. end
  473. 7327 parser.on(:close) do |force|
  474. 2693 if force
  475. 2693 reset
  476. 2685 emit(:terminate)
  477. end
  478. end
  479. 7327 parser.on(:close_handshake) do
  480. 8 consume
  481. end
  482. 7327 parser.on(:reset) do
  483. 3916 @pending.concat(parser.pending) unless parser.empty?
  484. 3916 current_session = @current_session
  485. 3916 current_selector = @current_selector
  486. 3916 reset
  487. 3908 unless @pending.empty?
  488. 171 idling
  489. 171 @current_session = current_session
  490. 171 @current_selector = current_selector
  491. end
  492. end
  493. 7327 parser.on(:current_timeout) do
  494. 3066 @current_timeout = @timeout = parser.timeout
  495. end
  496. 7327 parser.on(:timeout) do |tout|
  497. 2629 @timeout = tout
  498. end
  499. 7327 parser.on(:error) do |request, error|
  500. 56 case error
  501. when :http_1_1_required
  502. 16 current_session = @current_session
  503. 16 current_selector = @current_selector
  504. 16 parser.close
  505. 16 other_connection = current_session.find_connection(@origin, current_selector,
  506. @options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
  507. 16 other_connection.merge(self)
  508. 16 request.transition(:idle)
  509. 16 other_connection.send(request)
  510. 16 next
  511. when OperationTimeoutError
  512. # request level timeouts should take precedence
  513. next unless request.active_timeouts.empty?
  514. end
  515. 47 response = ErrorResponse.new(request, error)
  516. 47 request.response = response
  517. 47 request.emit(:response, response)
  518. end
  519. end
  520. 26 def transition(nextstate)
  521. 44762 handle_transition(nextstate)
  522. rescue Errno::ECONNABORTED,
  523. Errno::ECONNREFUSED,
  524. Errno::ECONNRESET,
  525. Errno::EADDRNOTAVAIL,
  526. Errno::EHOSTUNREACH,
  527. Errno::EINVAL,
  528. Errno::ENETUNREACH,
  529. Errno::EPIPE,
  530. Errno::ENOENT,
  531. SocketError,
  532. IOError => e
  533. # connect errors, exit gracefully
  534. 81 error = ConnectionError.new(e.message)
  535. 81 error.set_backtrace(e.backtrace)
  536. 81 handle_connect_error(error) if connecting?
  537. 81 @state = :closed
  538. 81 purge_after_closed
  539. 81 disconnect
  540. rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
  541. # connect errors, exit gracefully
  542. 25 handle_error(e)
  543. 25 handle_connect_error(e) if connecting?
  544. 25 @state = :closed
  545. 25 purge_after_closed
  546. 25 disconnect
  547. end
  548. 26 def handle_transition(nextstate)
  549. 39589 case nextstate
  550. when :idle
  551. 7897 @timeout = @current_timeout = @options.timeout[:connect_timeout]
  552. 7897 @connected_at = nil
  553. when :open
  554. 20509 return if @state == :closed
  555. 20509 @io.connect
  556. 20404 close_sibling if @io.state == :connected
  557. 20404 return unless @io.connected?
  558. 7222 @connected_at = Utils.now
  559. 7222 send_pending
  560. 7222 @timeout = @current_timeout = parser.timeout
  561. 7222 emit(:open)
  562. when :inactive
  563. 344 return unless @state == :open
  564. # do not deactivate connection in use
  565. 343 return if @inflight.positive?
  566. when :closing
  567. 7194 return unless @state == :idle || @state == :open
  568. 7194 unless @write_buffer.empty?
  569. # preset state before handshake, as error callbacks
  570. # may take it back here.
  571. 2650 @state = nextstate
  572. # handshakes, try sending
  573. 2650 consume
  574. 2649 @write_buffer.clear
  575. 2649 return
  576. end
  577. when :closed
  578. 7486 return unless @state == :closing
  579. 7485 return unless @write_buffer.empty?
  580. 7460 purge_after_closed
  581. 7460 disconnect if @pending.empty?
  582. when :already_open
  583. 66 nextstate = :open
  584. # the first check for given io readiness must still use a timeout.
  585. # connect is the reasonable choice in such a case.
  586. 66 @timeout = @options.timeout[:connect_timeout]
  587. 66 send_pending
  588. when :active
  589. 177 return unless @state == :inactive
  590. 177 nextstate = :open
  591. # activate
  592. 177 @current_session.select_connection(self, @current_selector)
  593. end
  594. 28255 @state = nextstate
  595. end
  596. 26 def close_sibling
  597. 9473 return unless @sibling
  598. if @sibling.io_connected?
  599. reset
  600. # TODO: transition connection to closed
  601. end
  602. unless @sibling.state == :closed
  603. merge(@sibling) unless @main_sibling
  604. @sibling.force_reset(true)
  605. end
  606. @sibling = nil
  607. end
  608. 26 def purge_after_closed
  609. 8357 @io.close if @io
  610. 8357 @read_buffer.clear
  611. 8357 @timeout = nil
  612. end
  613. 26 def initialize_type(uri, options)
  614. 6798 options.transport || begin
  615. 6026 case uri.scheme
  616. when "http"
  617. 3893 "tcp"
  618. when "https"
  619. 2877 "ssl"
  620. else
  621. raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
  622. end
  623. end
  624. end
  625. # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
  626. 26 def build_altsvc_connection(alt_origin, origin, alt_params)
  627. # do not allow security downgrades on altsvc negotiation
  628. 8 return if @origin.scheme == "https" && alt_origin.scheme != "https"
  629. 8 altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
  630. # altsvc already exists, somehow it wasn't advertised, probably noop
  631. 8 return unless altsvc
  632. 8 alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
  633. 8 connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
  634. # advertised altsvc is the same origin being used, ignore
  635. 8 return if connection == self
  636. 8 connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
  637. 8 log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
  638. 8 connection.merge(self)
  639. 8 terminate
  640. rescue UnsupportedSchemeError
  641. altsvc["noop"] = true
  642. nil
  643. end
  644. 26 def build_socket(addrs = nil)
  645. 6001 case @type
  646. when "tcp"
  647. 3965 TCP.new(peer, addrs, @options)
  648. when "ssl"
  649. 2782 SSL.new(peer, addrs, @options) do |sock|
  650. 2760 sock.ssl_session = @ssl_session
  651. 2760 sock.session_new_cb do |sess|
  652. 4809 @ssl_session = sess
  653. 4809 sock.ssl_session = sess
  654. end
  655. end
  656. when "unix"
  657. 28 path = Array(addrs).first
  658. 28 path = String(path) if path
  659. 28 UNIX.new(peer, path, @options)
  660. else
  661. raise Error, "unsupported transport (#{@type})"
  662. end
  663. end
  664. 26 def on_error(error, request = nil)
  665. 631 if error.is_a?(OperationTimeoutError)
  666. # inactive connections do not contribute to the select loop, therefore
  667. # they should not fail due to such errors.
  668. 32 return if @state == :inactive
  669. 32 if @timeout
  670. 28 @timeout -= error.timeout
  671. 32 return unless @timeout <= 0
  672. end
  673. 32 error = error.to_connection_error if connecting?
  674. end
  675. 631 handle_error(error, request)
  676. 631 reset
  677. end
  678. 26 def handle_error(error, request = nil)
  679. 924 parser.handle_error(error, request) if @parser && parser.respond_to?(:handle_error)
  680. 2038 while (req = @pending.shift)
  681. 420 next if request && req == request
  682. 420 response = ErrorResponse.new(req, error)
  683. 420 req.response = response
  684. 420 req.emit(:response, response)
  685. end
  686. 924 return unless request
  687. 431 response = ErrorResponse.new(request, error)
  688. 431 request.response = response
  689. 431 request.emit(:response, response)
  690. end
  691. 26 def set_request_timeouts(request)
  692. 18834 set_request_write_timeout(request)
  693. 18834 set_request_read_timeout(request)
  694. 18834 set_request_request_timeout(request)
  695. end
  696. 26 def set_request_read_timeout(request)
  697. 18834 read_timeout = request.read_timeout
  698. 18834 return if read_timeout.nil? || read_timeout.infinite?
  699. 18548 set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
  700. 24 read_timeout_callback(request, read_timeout)
  701. end
  702. end
  703. 26 def set_request_write_timeout(request)
  704. 18834 write_timeout = request.write_timeout
  705. 18834 return if write_timeout.nil? || write_timeout.infinite?
  706. 18834 set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
  707. 24 write_timeout_callback(request, write_timeout)
  708. end
  709. end
  710. 26 def set_request_request_timeout(request)
  711. 18550 request_timeout = request.request_timeout
  712. 18550 return if request_timeout.nil? || request_timeout.infinite?
  713. 516 set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
  714. 383 read_timeout_callback(request, request_timeout, RequestTimeoutError)
  715. end
  716. end
  717. 26 def write_timeout_callback(request, write_timeout)
  718. 24 return if request.state == :done
  719. 24 @write_buffer.clear
  720. 24 error = WriteTimeoutError.new(request, nil, write_timeout)
  721. 24 on_error(error, request)
  722. end
  723. 26 def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
  724. 407 response = request.response
  725. 407 return if response && response.finished?
  726. 407 @write_buffer.clear
  727. 407 error = error_type.new(request, request.response, read_timeout)
  728. 407 on_error(error, request)
  729. end
  730. 26 def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
  731. 37978 request.set_timeout_callback(start_event) do
  732. 37116 timer = @current_selector.after(timeout, callback)
  733. 37116 request.active_timeouts << label
  734. 37116 Array(finish_events).each do |event|
  735. # clean up request timeouts if the connection errors out
  736. 55624 request.set_timeout_callback(event) do
  737. 55206 timer.cancel
  738. 55206 request.active_timeouts.delete(label)
  739. end
  740. end
  741. end
  742. end
  743. 26 class << self
  744. 26 def parser_type(protocol)
  745. 6523 case protocol
  746. 3081 when "h2" then HTTP2
  747. 4279 when "http/1.1" then HTTP1
  748. else
  749. raise Error, "unsupported protocol (##{protocol})"
  750. end
  751. end
  752. end
  753. end
  754. end

lib/httpx/connection/http1.rb

89.19% lines covered

222 relevant lines. 198 lines covered and 24 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "httpx/parser/http1"
  3. 26 module HTTPX
  4. 26 class Connection::HTTP1
  5. 26 include Callbacks
  6. 26 include Loggable
  7. 26 MAX_REQUESTS = 200
  8. 26 CRLF = "\r\n"
  9. 26 attr_reader :pending, :requests
  10. 26 attr_accessor :max_concurrent_requests
  11. 26 def initialize(buffer, options)
  12. 4279 @options = options
  13. 4279 @max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
  14. 4279 @max_requests = @options.max_requests
  15. 4279 @parser = Parser::HTTP1.new(self)
  16. 4279 @buffer = buffer
  17. 4279 @version = [1, 1]
  18. 4279 @pending = []
  19. 4279 @requests = []
  20. 4279 @handshake_completed = false
  21. end
  22. 26 def timeout
  23. 4162 @options.timeout[:operation_timeout]
  24. end
  25. 26 def interests
  26. # this means we're processing incoming response already
  27. 29721 return :r if @request
  28. 24630 return if @requests.empty?
  29. 24610 request = @requests.first
  30. 24610 return unless request
  31. 24610 return :w if request.interests == :w || !@buffer.empty?
  32. 21684 :r
  33. end
  34. 26 def reset
  35. 4045 @max_requests = @options.max_requests || MAX_REQUESTS
  36. 4045 @parser.reset!
  37. 4045 @handshake_completed = false
  38. 4045 @pending.concat(@requests) unless @requests.empty?
  39. end
  40. 26 def close
  41. 81 reset
  42. 81 emit(:close, true)
  43. end
  44. 26 def exhausted?
  45. 520 !@max_requests.positive?
  46. end
  47. 26 def empty?
  48. # this means that for every request there's an available
  49. # partial response, so there are no in-flight requests waiting.
  50. 3964 @requests.empty? || (
  51. # checking all responses can be time-consuming. Alas, as in HTTP/1, responses
  52. # do not come out of order, we can get away with checking first and last.
  53. 178 !@requests.first.response.nil? &&
  54. (@requests.size == 1 || !@requests.last.response.nil?)
  55. )
  56. end
  57. 26 def <<(data)
  58. 6994 @parser << data
  59. end
  60. 26 def send(request)
  61. 15360 unless @max_requests.positive?
  62. @pending << request
  63. return
  64. end
  65. 15360 return if @requests.include?(request)
  66. 15360 @requests << request
  67. 15360 @pipelining = true if @requests.size > 1
  68. end
  69. 26 def consume
  70. 15701 requests_limit = [@max_requests, @requests.size].min
  71. 15701 concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
  72. 15701 @requests.each_with_index do |request, idx|
  73. 18230 break if idx >= concurrent_requests_limit
  74. 15657 next if request.state == :done
  75. 6007 handle(request)
  76. end
  77. end
  78. # HTTP Parser callbacks
  79. #
  80. # must be public methods, or else they won't be reachable
  81. 26 def on_start
  82. 4653 log(level: 2) { "parsing begins" }
  83. end
  84. 26 def on_headers(h)
  85. 4629 @request = @requests.first
  86. 4629 return if @request.response
  87. 4653 log(level: 2) { "headers received" }
  88. 4629 headers = @request.options.headers_class.new(h)
  89. 4629 response = @request.options.response_class.new(@request,
  90. @parser.status_code,
  91. @parser.http_version.join("."),
  92. headers)
  93. 4653 log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
  94. 4845 log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
  95. 4629 @request.response = response
  96. 4621 on_complete if response.finished?
  97. end
  98. 26 def on_trailers(h)
  99. 8 return unless @request
  100. 8 response = @request.response
  101. 8 log(level: 2) { "trailer headers received" }
  102. 8 log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v.join(", ")}" }.join("\n") }
  103. 8 response.merge_headers(h)
  104. end
  105. 26 def on_data(chunk)
  106. 5427 request = @request
  107. 5427 return unless request
  108. 5451 log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
  109. 5451 log(level: 2, color: :green) { "-> #{chunk.inspect}" }
  110. 5427 response = request.response
  111. 5427 response << chunk
  112. rescue StandardError => e
  113. 14 error_response = ErrorResponse.new(request, e)
  114. 14 request.response = error_response
  115. 14 dispatch
  116. end
  117. 26 def on_complete
  118. 4599 request = @request
  119. 4599 return unless request
  120. 4623 log(level: 2) { "parsing complete" }
  121. 4599 dispatch
  122. end
  123. 26 def dispatch
  124. 4613 request = @request
  125. 4613 if request.expects?
  126. 72 @parser.reset!
  127. 63 return handle(request)
  128. end
  129. 4541 @request = nil
  130. 4541 @requests.shift
  131. 4541 response = request.response
  132. 4541 response.finish! unless response.is_a?(ErrorResponse)
  133. 4541 emit(:response, request, response)
  134. 4484 if @parser.upgrade?
  135. 32 response << @parser.upgrade_data
  136. 32 throw(:called)
  137. end
  138. 4452 @parser.reset!
  139. 4001 @max_requests -= 1
  140. 4452 if response.is_a?(ErrorResponse)
  141. 14 disable
  142. else
  143. 4438 manage_connection(request, response)
  144. end
  145. 520 if exhausted?
  146. @pending.concat(@requests)
  147. @requests.clear
  148. emit(:exhausted)
  149. else
  150. 520 send(@pending.shift) unless @pending.empty?
  151. end
  152. end
  153. 26 def handle_error(ex, request = nil)
  154. 219 if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request && @request.response &&
  155. !@request.response.headers.key?("content-length") &&
  156. !@request.response.headers.key?("transfer-encoding")
  157. # if the response does not contain a content-length header, the server closing the
  158. # connnection is the indicator of response consumed.
  159. # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
  160. 32 catch(:called) { on_complete }
  161. 14 return
  162. end
  163. 203 if @pipelining
  164. catch(:called) { disable }
  165. else
  166. 203 @requests.each do |req|
  167. 194 next if request && request == req
  168. emit(:error, req, ex)
  169. end
  170. 203 @pending.each do |req|
  171. next if request && request == req
  172. emit(:error, req, ex)
  173. end
  174. end
  175. end
  176. 26 def ping
  177. reset
  178. emit(:reset)
  179. emit(:exhausted)
  180. end
  181. 26 private
  182. 26 def manage_connection(request, response)
  183. 4438 connection = response.headers["connection"]
  184. 3988 case connection
  185. when /keep-alive/i
  186. 520 if @handshake_completed
  187. if @max_requests.zero?
  188. @pending.concat(@requests)
  189. @requests.clear
  190. emit(:exhausted)
  191. end
  192. return
  193. end
  194. 520 keep_alive = response.headers["keep-alive"]
  195. 520 return unless keep_alive
  196. 108 parameters = Hash[keep_alive.split(/ *, */).map do |pair|
  197. 108 pair.split(/ *= */, 2)
  198. end]
  199. 108 @max_requests = parameters["max"].to_i - 1 if parameters.key?("max")
  200. 108 if parameters.key?("timeout")
  201. keep_alive_timeout = parameters["timeout"].to_i
  202. emit(:timeout, keep_alive_timeout)
  203. end
  204. 108 @handshake_completed = true
  205. when /close/i
  206. 3918 disable
  207. when nil
  208. # In HTTP/1.1, it's keep alive by default
  209. return if response.version == "1.1" && request.headers["connection"] != "close"
  210. disable
  211. end
  212. end
  213. 26 def disable
  214. 3932 disable_pipelining
  215. 3932 reset
  216. 3932 emit(:reset)
  217. 3924 throw(:called)
  218. end
  219. 26 def disable_pipelining
  220. 3932 return if @requests.empty?
  221. # do not disable pipelining if already set to 1 request at a time
  222. 179 return if @max_concurrent_requests == 1
  223. 21 @requests.each do |r|
  224. 21 r.transition(:idle)
  225. # when we disable pipelining, we still want to try keep-alive.
  226. # only when keep-alive with one request fails, do we fallback to
  227. # connection: close.
  228. 21 r.headers["connection"] = "close" if @max_concurrent_requests == 1
  229. end
  230. # server doesn't handle pipelining, and probably
  231. # doesn't support keep-alive. Fallback to send only
  232. # 1 keep alive request.
  233. 21 @max_concurrent_requests = 1
  234. 21 @pipelining = false
  235. end
  236. 26 def set_protocol_headers(request)
  237. 4780 if !request.headers.key?("content-length") &&
  238. request.body.bytesize == Float::INFINITY
  239. 32 request.body.chunk!
  240. end
  241. 4780 extra_headers = {}
  242. 4780 unless request.headers.key?("connection")
  243. 4756 connection_value = if request.persistent?
  244. # when in a persistent connection, the request can't be at
  245. # the edge of a renegotiation
  246. 105 if @requests.index(request) + 1 < @max_requests
  247. 105 "keep-alive"
  248. else
  249. "close"
  250. end
  251. else
  252. # when it's not a persistent connection, it sets "Connection: close" always
  253. # on the last request of the possible batch (either allowed max requests,
  254. # or if smaller, the size of the batch itself)
  255. 4651 requests_limit = [@max_requests, @requests.size].min
  256. 4651 if request == @requests[requests_limit - 1]
  257. 4094 "close"
  258. else
  259. 557 "keep-alive"
  260. end
  261. end
  262. 4268 extra_headers["connection"] = connection_value
  263. end
  264. 4780 extra_headers["host"] = request.authority unless request.headers.key?("host")
  265. 4780 extra_headers
  266. end
  267. 26 def handle(request)
  268. 6079 catch(:buffer_full) do
  269. 6079 request.transition(:headers)
  270. 6071 join_headers(request) if request.state == :headers
  271. 6071 request.transition(:body)
  272. 6071 join_body(request) if request.state == :body
  273. 4971 request.transition(:trailers)
  274. # HTTP/1.1 trailers should only work for chunked encoding
  275. 4971 join_trailers(request) if request.body.chunked? && request.state == :trailers
  276. 4971 request.transition(:done)
  277. end
  278. end
  279. 26 def join_headline(request)
  280. 4219 "#{request.verb} #{request.path} HTTP/#{@version.join(".")}"
  281. end
  282. 26 def join_headers(request)
  283. 4780 headline = join_headline(request)
  284. 4780 @buffer << headline << CRLF
  285. 4804 log(color: :yellow) { "<- HEADLINE: #{headline.chomp.inspect}" }
  286. 4780 extra_headers = set_protocol_headers(request)
  287. 4780 join_headers2(request.headers.each(extra_headers))
  288. 4804 log { "<- " }
  289. 4780 @buffer << CRLF
  290. end
  291. 26 def join_body(request)
  292. 5856 return if request.body.empty?
  293. 6579 while (chunk = request.drain_body)
  294. 3288 log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
  295. 3288 log(level: 2, color: :green) { "<- #{chunk.inspect}" }
  296. 3288 @buffer << chunk
  297. 3288 throw(:buffer_full, request) if @buffer.full?
  298. end
  299. 1556 return unless (error = request.drain_error)
  300. raise error
  301. end
  302. 26 def join_trailers(request)
  303. 96 return unless request.trailers? && request.callbacks_for?(:trailers)
  304. 32 join_headers2(request.trailers)
  305. 32 log { "<- " }
  306. 32 @buffer << CRLF
  307. end
  308. 26 def join_headers2(headers)
  309. 4812 headers.each do |field, value|
  310. 29690 buffer = "#{capitalized(field)}: #{value}#{CRLF}"
  311. 29810 log(color: :yellow) { "<- HEADER: #{buffer.chomp}" }
  312. 29690 @buffer << buffer
  313. end
  314. end
  315. 26 UPCASED = {
  316. "www-authenticate" => "WWW-Authenticate",
  317. "http2-settings" => "HTTP2-Settings",
  318. "content-md5" => "Content-MD5",
  319. }.freeze
  320. 26 def capitalized(field)
  321. 29690 UPCASED[field] || field.split("-").map(&:capitalize).join("-")
  322. end
  323. end
  324. end

lib/httpx/connection/http2.rb

94.9% lines covered

255 relevant lines. 242 lines covered and 13 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "securerandom"
  3. 26 require "http/2"
  4. 26 module HTTPX
  5. 26 class Connection::HTTP2
  6. 26 include Callbacks
  7. 26 include Loggable
  8. 26 MAX_CONCURRENT_REQUESTS = ::HTTP2::DEFAULT_MAX_CONCURRENT_STREAMS
  9. 26 class Error < Error
  10. 26 def initialize(id, code)
  11. 36 super("stream #{id} closed with error: #{code}")
  12. end
  13. end
  14. 26 class PingError < Error
  15. 26 def initialize
  16. super(0, :ping_error)
  17. end
  18. end
  19. 26 class GoawayError < Error
  20. 26 def initialize
  21. 17 super(0, :no_error)
  22. end
  23. end
  24. 26 attr_reader :streams, :pending
  25. 26 def initialize(buffer, options)
  26. 3105 @options = options
  27. 3105 @settings = @options.http2_settings
  28. 3105 @pending = []
  29. 3105 @streams = {}
  30. 3105 @drains = {}
  31. 3105 @pings = []
  32. 3105 @buffer = buffer
  33. 3105 @handshake_completed = false
  34. 3105 @wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
  35. 3105 @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
  36. 3105 @max_requests = @options.max_requests
  37. 3105 init_connection
  38. end
  39. 26 def timeout
  40. 6126 return @options.timeout[:operation_timeout] if @handshake_completed
  41. 3060 @options.timeout[:settings_timeout]
  42. end
  43. 26 def interests
  44. # waiting for WINDOW_UPDATE frames
  45. 11428972 return :r if @buffer.full?
  46. 11428972 if @connection.state == :closed
  47. 2839 return unless @handshake_completed
  48. 2461 return :w
  49. end
  50. 11426133 unless @connection.state == :connected && @handshake_completed
  51. 11760 return @buffer.empty? ? :r : :rw
  52. end
  53. 11412955 return :w if !@pending.empty? && can_buffer_more_requests?
  54. 11412955 return :w unless @drains.empty?
  55. 11411977 if @buffer.empty?
  56. 11411977 return if @streams.empty? && @pings.empty?
  57. 38028 return :r
  58. end
  59. :rw
  60. end
  61. 26 def close
  62. 2637 unless @connection.state == :closed
  63. 2629 @connection.goaway
  64. 2629 emit(:timeout, @options.timeout[:close_handshake_timeout])
  65. end
  66. 2637 emit(:close, true)
  67. end
  68. 26 def empty?
  69. 2630 @connection.state == :closed || @streams.empty?
  70. end
  71. 26 def exhausted?
  72. 3105 !@max_requests.positive?
  73. end
  74. 26 def <<(data)
  75. 31945 @connection << data
  76. end
  77. 26 def can_buffer_more_requests?
  78. 7285 (@handshake_completed || !@wait_for_handshake) &&
  79. @streams.size < @max_concurrent_requests &&
  80. @streams.size < @max_requests
  81. end
  82. 26 def send(request, head = false)
  83. 6788 unless can_buffer_more_requests?
  84. 3297 head ? @pending.unshift(request) : @pending << request
  85. 3297 return false
  86. end
  87. 3491 unless (stream = @streams[request])
  88. 3491 stream = @connection.new_stream
  89. 3491 handle_stream(stream, request)
  90. 3099 @streams[request] = stream
  91. 3099 @max_requests -= 1
  92. end
  93. 3491 handle(request, stream)
  94. 3475 true
  95. rescue ::HTTP2::Error::StreamLimitExceeded
  96. @pending.unshift(request)
  97. false
  98. end
  99. 26 def consume
  100. 24135 @streams.each do |request, stream|
  101. 9527 next if request.state == :done
  102. 1114 handle(request, stream)
  103. end
  104. end
  105. 26 def handle_error(ex, request = nil)
  106. 285 if ex.is_a?(OperationTimeoutError) && !@handshake_completed && @connection.state != :closed
  107. 8 @connection.goaway(:settings_timeout, "closing due to settings timeout")
  108. 8 emit(:close_handshake)
  109. 8 settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
  110. 8 settings_ex.set_backtrace(ex.backtrace)
  111. 8 ex = settings_ex
  112. end
  113. 285 @streams.each_key do |req|
  114. 214 next if request && request == req
  115. 14 emit(:error, req, ex)
  116. end
  117. 536 while (req = @pending.shift)
  118. 33 next if request && request == req
  119. 33 emit(:error, req, ex)
  120. end
  121. end
  122. 26 def ping
  123. 10 ping = SecureRandom.gen_random(8)
  124. 10 @connection.ping(ping)
  125. ensure
  126. 10 @pings << ping
  127. end
  128. 26 private
  129. 26 def send_pending
  130. 8180 while (request = @pending.shift)
  131. 3170 break unless send(request, true)
  132. end
  133. end
  134. 26 def handle(request, stream)
  135. 4669 catch(:buffer_full) do
  136. 4669 request.transition(:headers)
  137. 4661 join_headers(stream, request) if request.state == :headers
  138. 4661 request.transition(:body)
  139. 4661 join_body(stream, request) if request.state == :body
  140. 3667 request.transition(:trailers)
  141. 3667 join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
  142. 3667 request.transition(:done)
  143. end
  144. end
  145. 26 def init_connection
  146. 3105 @connection = ::HTTP2::Client.new(@settings)
  147. 3105 @connection.on(:frame, &method(:on_frame))
  148. 3105 @connection.on(:frame_sent, &method(:on_frame_sent))
  149. 3105 @connection.on(:frame_received, &method(:on_frame_received))
  150. 3105 @connection.on(:origin, &method(:on_origin))
  151. 3105 @connection.on(:promise, &method(:on_promise))
  152. 3105 @connection.on(:altsvc) { |frame| on_altsvc(frame[:origin], frame) }
  153. 3105 @connection.on(:settings_ack, &method(:on_settings))
  154. 3105 @connection.on(:ack, &method(:on_pong))
  155. 3105 @connection.on(:goaway, &method(:on_close))
  156. #
  157. # Some servers initiate HTTP/2 negotiation right away, some don't.
  158. # As such, we have to check the socket buffer. If there is something
  159. # to read, the server initiated the negotiation. If not, we have to
  160. # initiate it.
  161. #
  162. 3105 @connection.send_connection_preface
  163. end
  164. 26 alias_method :reset, :init_connection
  165. 26 public :reset
  166. 26 def handle_stream(stream, request)
  167. 3507 request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
  168. 3507 stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
  169. 3507 stream.on(:half_close) do
  170. 3485 log(level: 2) { "#{stream.id}: waiting for response..." }
  171. end
  172. 3507 stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
  173. 3507 stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
  174. 3507 stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
  175. end
  176. 26 def set_protocol_headers(request)
  177. 391 {
  178. 3091 ":scheme" => request.scheme,
  179. ":method" => request.verb,
  180. ":path" => request.path,
  181. ":authority" => request.authority,
  182. }
  183. end
  184. 26 def join_headers(stream, request)
  185. 3483 extra_headers = set_protocol_headers(request)
  186. 3483 if request.headers.key?("host")
  187. 8 log { "forbidden \"host\" header found (#{request.headers["host"]}), will use it as authority..." }
  188. 7 extra_headers[":authority"] = request.headers["host"]
  189. end
  190. 3483 log(level: 1, color: :yellow) do
  191. 128 request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
  192. end
  193. 3483 stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
  194. end
  195. 26 def join_trailers(stream, request)
  196. 1493 unless request.trailers?
  197. 1485 stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
  198. 1325 return
  199. end
  200. 8 log(level: 1, color: :yellow) do
  201. 15 request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
  202. end
  203. 8 stream.headers(request.trailers.each, end_stream: true)
  204. end
  205. 26 def join_body(stream, request)
  206. 4477 return if request.body.empty?
  207. 2487 chunk = @drains.delete(request) || request.drain_body
  208. 2678 while chunk
  209. 2715 next_chunk = request.drain_body
  210. 2736 log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
  211. 2736 log(level: 2, color: :green) { "#{stream.id}: -> #{chunk.inspect}" }
  212. 2715 stream.data(chunk, end_stream: !(next_chunk || request.trailers? || request.callbacks_for?(:trailers)))
  213. 2715 if next_chunk && (@buffer.full? || request.body.unbounded_body?)
  214. 874 @drains[request] = next_chunk
  215. 994 throw(:buffer_full)
  216. end
  217. 1721 chunk = next_chunk
  218. end
  219. 1493 return unless (error = request.drain_error)
  220. 12 on_stream_refuse(stream, request, error)
  221. end
  222. ######
  223. # HTTP/2 Callbacks
  224. ######
  225. 26 def on_stream_headers(stream, request, h)
  226. 3464 response = request.response
  227. 3464 if response.is_a?(Response) && response.version == "2.0"
  228. 114 on_stream_trailers(stream, response, h)
  229. 114 return
  230. end
  231. 3350 log(color: :yellow) do
  232. 128 h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
  233. end
  234. 3350 _, status = h.shift
  235. 3350 headers = request.options.headers_class.new(h)
  236. 3350 response = request.options.response_class.new(request, status, "2.0", headers)
  237. 3350 request.response = response
  238. 2967 @streams[request] = stream
  239. 3342 handle(request, stream) if request.expects?
  240. end
  241. 26 def on_stream_trailers(stream, response, h)
  242. 114 log(color: :yellow) do
  243. h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
  244. end
  245. 114 response.merge_headers(h)
  246. end
  247. 26 def on_stream_data(stream, request, data)
  248. 6138 log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
  249. 6138 log(level: 2, color: :green) { "#{stream.id}: <- #{data.inspect}" }
  250. 6119 request.response << data
  251. end
  252. 26 def on_stream_refuse(stream, request, error)
  253. 12 on_stream_close(stream, request, error)
  254. 12 stream.close
  255. end
  256. 26 def on_stream_close(stream, request, error)
  257. 3264 return if error == :stream_closed && !@streams.key?(request)
  258. 3266 log(level: 2) { "#{stream.id}: closing stream" }
  259. 3252 @drains.delete(request)
  260. 3252 @streams.delete(request)
  261. 3252 if error
  262. 12 case error
  263. when :http_1_1_required
  264. emit(:error, request, error)
  265. else
  266. 12 ex = Error.new(stream.id, error)
  267. 12 ex.set_backtrace(caller)
  268. 12 response = ErrorResponse.new(request, ex)
  269. 12 request.response = response
  270. 12 emit(:response, request, response)
  271. end
  272. else
  273. 3240 response = request.response
  274. 3240 if response && response.is_a?(Response) && response.status == 421
  275. 8 emit(:error, request, :http_1_1_required)
  276. else
  277. 3232 emit(:response, request, response)
  278. end
  279. end
  280. 3244 send(@pending.shift) unless @pending.empty?
  281. 3244 return unless @streams.empty? && exhausted?
  282. 8 if @pending.empty?
  283. close
  284. else
  285. 8 emit(:exhausted)
  286. end
  287. end
  288. 26 def on_frame(bytes)
  289. 19292 @buffer << bytes
  290. end
  291. 26 def on_settings(*)
  292. 3066 @handshake_completed = true
  293. 3066 emit(:current_timeout)
  294. 3066 @max_concurrent_requests = [@max_concurrent_requests, @connection.remote_settings[:settings_max_concurrent_streams]].min
  295. 3066 send_pending
  296. end
  297. 26 def on_close(_last_frame, error, _payload)
  298. 32 is_connection_closed = @connection.state == :closed
  299. 32 if error
  300. 32 @buffer.clear if is_connection_closed
  301. 28 case error
  302. when :http_1_1_required
  303. 21 while (request = @pending.shift)
  304. 8 emit(:error, request, error)
  305. end
  306. when :no_error
  307. 17 ex = GoawayError.new
  308. 17 @pending.unshift(*@streams.keys)
  309. 17 @drains.clear
  310. 17 @streams.clear
  311. else
  312. 7 ex = Error.new(0, error)
  313. end
  314. 32 if ex
  315. 24 ex.set_backtrace(caller)
  316. 24 handle_error(ex)
  317. end
  318. end
  319. 32 return unless is_connection_closed && @streams.empty?
  320. 32 emit(:close, is_connection_closed)
  321. end
  322. 26 def on_frame_sent(frame)
  323. 16262 log(level: 2) { "#{frame[:stream]}: frame was sent!" }
  324. 16178 log(level: 2, color: :blue) do
  325. 96 payload = frame
  326. 96 payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
  327. 84 "#{frame[:stream]}: #{payload}"
  328. end
  329. end
  330. 26 def on_frame_received(frame)
  331. 17121 log(level: 2) { "#{frame[:stream]}: frame was received!" }
  332. 17060 log(level: 2, color: :magenta) do
  333. 69 payload = frame
  334. 69 payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
  335. 61 "#{frame[:stream]}: #{payload}"
  336. end
  337. end
  338. 26 def on_altsvc(origin, frame)
  339. log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
  340. log(level: 2) { "#{frame[:stream]}: #{frame.inspect}" }
  341. alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
  342. params = { "ma" => frame[:max_age] }
  343. emit(:altsvc, origin, alt_origin, origin, params)
  344. end
  345. 26 def on_promise(stream)
  346. 24 emit(:promise, @streams.key(stream.parent), stream)
  347. end
  348. 26 def on_origin(origin)
  349. emit(:origin, origin)
  350. end
  351. 26 def on_pong(ping)
  352. 8 raise PingError unless @pings.delete(ping.to_s)
  353. 8 emit(:pong)
  354. end
  355. end
  356. end

lib/httpx/domain_name.rb

95.56% lines covered

45 relevant lines. 43 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. #
  3. # domain_name.rb - Domain Name manipulation library for Ruby
  4. #
  5. # Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
  6. #
  7. # Redistribution and use in source and binary forms, with or without
  8. # modification, are permitted provided that the following conditions
  9. # are met:
  10. # 1. Redistributions of source code must retain the above copyright
  11. # notice, this list of conditions and the following disclaimer.
  12. # 2. Redistributions in binary form must reproduce the above copyright
  13. # notice, this list of conditions and the following disclaimer in the
  14. # documentation and/or other materials provided with the distribution.
  15. #
  16. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26. # SUCH DAMAGE.
  27. 26 require "ipaddr"
  28. 26 module HTTPX
  29. # Represents a domain name ready for extracting its registered domain
  30. # and TLD.
  31. 26 class DomainName
  32. 26 include Comparable
  33. # The full host name normalized, ASCII-ized and downcased using the
  34. # Unicode NFC rules and the Punycode algorithm. If initialized with
  35. # an IP address, the string representation of the IP address
  36. # suitable for opening a connection to.
  37. 26 attr_reader :hostname
  38. # The Unicode representation of the #hostname property.
  39. #
  40. # :attr_reader: hostname_idn
  41. # The least "universally original" domain part of this domain name.
  42. # For example, "example.co.uk" for "www.sub.example.co.uk". This
  43. # may be nil if the hostname does not have one, like when it is an
  44. # IP address, an effective TLD or higher itself, or of a
  45. # non-canonical domain.
  46. 26 attr_reader :domain
  47. 26 class << self
  48. 26 def new(domain)
  49. 856 return domain if domain.is_a?(self)
  50. 792 super(domain)
  51. end
  52. # Normalizes a _domain_ using the Punycode algorithm as necessary.
  53. # The result will be a downcased, ASCII-only string.
  54. 26 def normalize(domain)
  55. 760 unless domain.ascii_only?
  56. domain = domain.chomp(".").unicode_normalize(:nfc)
  57. domain = Punycode.encode_hostname(domain)
  58. end
  59. 760 domain.downcase
  60. end
  61. end
  62. # Parses _hostname_ into a DomainName object. An IP address is also
  63. # accepted. An IPv6 address may be enclosed in square brackets.
  64. 26 def initialize(hostname)
  65. 792 hostname = String(hostname)
  66. 792 raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
  67. 98 begin
  68. 792 @ipaddr = IPAddr.new(hostname)
  69. 32 @hostname = @ipaddr.to_s
  70. 32 return
  71. rescue IPAddr::Error
  72. 760 nil
  73. end
  74. 760 @hostname = DomainName.normalize(hostname)
  75. 760 tld = if (last_dot = @hostname.rindex("."))
  76. 184 @hostname[(last_dot + 1)..-1]
  77. else
  78. 576 @hostname
  79. end
  80. # unknown/local TLD
  81. 760 @domain = if last_dot
  82. # fallback - accept cookies down to second level
  83. # cf. http://www.dkim-reputation.org/regdom-libs/
  84. 184 if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
  85. 48 @hostname[(penultimate_dot + 1)..-1]
  86. else
  87. 136 @hostname
  88. end
  89. else
  90. # no domain part - must be a local hostname
  91. 576 tld
  92. end
  93. end
  94. # Checks if the server represented by this domain is qualified to
  95. # send and receive cookies with a domain attribute value of
  96. # _domain_. A true value given as the second argument represents
  97. # cookies without a domain attribute value, in which case only
  98. # hostname equality is checked.
  99. 26 def cookie_domain?(domain, host_only = false)
  100. # RFC 6265 #5.3
  101. # When the user agent "receives a cookie":
  102. 32 return self == @domain if host_only
  103. 32 domain = DomainName.new(domain)
  104. # RFC 6265 #5.1.3
  105. # Do not perform subdomain matching against IP addresses.
  106. 32 @hostname == domain.hostname if @ipaddr
  107. # RFC 6265 #4.1.1
  108. # Domain-value must be a subdomain.
  109. 32 @domain && self <= domain && domain <= @domain
  110. end
  111. 26 def <=>(other)
  112. 48 other = DomainName.new(other)
  113. 48 othername = other.hostname
  114. 48 if othername == @hostname
  115. 16 0
  116. 31 elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
  117. # The other is higher
  118. 16 -1
  119. else
  120. # The other is lower
  121. 16 1
  122. end
  123. end
  124. end
  125. end

lib/httpx/errors.rb

97.83% lines covered

46 relevant lines. 45 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. # the default exception class for exceptions raised by HTTPX.
  4. 26 class Error < StandardError; end
  5. 26 class UnsupportedSchemeError < Error; end
  6. 26 class ConnectionError < Error; end
  7. # Error raised when there was a timeout. Its subclasses allow for finer-grained
  8. # control of which timeout happened.
  9. 26 class TimeoutError < Error
  10. # The timeout value which caused this error to be raised.
  11. 26 attr_reader :timeout
  12. # initializes the timeout exception with the +timeout+ causing the error, and the
  13. # error +message+ for it.
  14. 26 def initialize(timeout, message)
  15. 534 @timeout = timeout
  16. 534 super(message)
  17. end
  18. # clones this error into a HTTPX::ConnectionTimeoutError.
  19. 26 def to_connection_error
  20. 24 ex = ConnectTimeoutError.new(@timeout, message)
  21. 24 ex.set_backtrace(backtrace)
  22. 24 ex
  23. end
  24. end
  25. # Raise when it can't acquire a connection for a given origin.
  26. 26 class PoolTimeoutError < TimeoutError
  27. 26 attr_reader :origin
  28. # initializes the +origin+ it refers to, and the
  29. # +timeout+ causing the error.
  30. 26 def initialize(origin, timeout)
  31. 8 @origin = origin
  32. 8 super(timeout, "Timed out after #{timeout} seconds while waiting for a connection to #{origin}")
  33. end
  34. end
  35. # Error raised when there was a timeout establishing the connection to a server.
  36. # This may be raised due to timeouts during TCP and TLS (when applicable) connection
  37. # establishment.
  38. 26 class ConnectTimeoutError < TimeoutError; end
  39. # Error raised when there was a timeout while sending a request, or receiving a response
  40. # from the server.
  41. 26 class RequestTimeoutError < TimeoutError
  42. # The HTTPX::Request request object this exception refers to.
  43. 26 attr_reader :request
  44. # initializes the exception with the +request+ and +response+ it refers to, and the
  45. # +timeout+ causing the error, and the
  46. 26 def initialize(request, response, timeout)
  47. 431 @request = request
  48. 431 @response = response
  49. 431 super(timeout, "Timed out after #{timeout} seconds")
  50. end
  51. 26 def marshal_dump
  52. [message]
  53. end
  54. end
  55. # Error raised when there was a timeout while receiving a response from the server.
  56. 26 class ReadTimeoutError < RequestTimeoutError; end
  57. # Error raised when there was a timeout while sending a request from the server.
  58. 26 class WriteTimeoutError < RequestTimeoutError; end
  59. # Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
  60. 26 class SettingsTimeoutError < TimeoutError; end
  61. # Error raised when there was a timeout while resolving a domain to an IP.
  62. 26 class ResolveTimeoutError < TimeoutError; end
  63. # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
  64. 26 class OperationTimeoutError < TimeoutError; end
  65. # Error raised when there was an error while resolving a domain to an IP.
  66. 26 class ResolveError < Error; end
  67. # Error raised when there was an error while resolving a domain to an IP
  68. # using a HTTPX::Resolver::Native resolver.
  69. 26 class NativeResolveError < ResolveError
  70. 26 attr_reader :connection, :host
  71. # initializes the exception with the +connection+ it refers to, the +host+ domain
  72. # which failed to resolve, and the error +message+.
  73. 26 def initialize(connection, host, message = "Can't resolve #{host}")
  74. 120 @connection = connection
  75. 120 @host = host
  76. 120 super(message)
  77. end
  78. end
  79. # The exception class for HTTP responses with 4xx or 5xx status.
  80. 26 class HTTPError < Error
  81. # The HTTPX::Response response object this exception refers to.
  82. 26 attr_reader :response
  83. # Creates the instance and assigns the HTTPX::Response +response+.
  84. 26 def initialize(response)
  85. 84 @response = response
  86. 84 super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
  87. end
  88. # The HTTP response status.
  89. #
  90. # error.status #=> 404
  91. 26 def status
  92. 16 @response.status
  93. end
  94. end
  95. end

lib/httpx/extensions.rb

67.86% lines covered

28 relevant lines. 19 lines covered and 9 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "uri"
  3. 26 module HTTPX
  4. 26 module ArrayExtensions
  5. 26 module FilterMap
  6. refine Array do
  7. # Ruby 2.7 backport
  8. def filter_map
  9. return to_enum(:filter_map) unless block_given?
  10. each_with_object([]) do |item, res|
  11. processed = yield(item)
  12. res << processed if processed
  13. end
  14. end
  15. 25 end unless Array.method_defined?(:filter_map)
  16. end
  17. 26 module Intersect
  18. refine Array do
  19. # Ruby 3.1 backport
  20. 4 def intersect?(arr)
  21. if size < arr.size
  22. smaller = self
  23. else
  24. smaller, arr = arr, self
  25. end
  26. (arr & smaller).size > 0
  27. end
  28. 25 end unless Array.method_defined?(:intersect?)
  29. end
  30. end
  31. 26 module URIExtensions
  32. # uri 0.11 backport, ships with ruby 3.1
  33. 26 refine URI::Generic do
  34. 26 def non_ascii_hostname
  35. 428 @non_ascii_hostname
  36. end
  37. 26 def non_ascii_hostname=(hostname)
  38. 32 @non_ascii_hostname = hostname
  39. end
  40. def authority
  41. 5500 return host if port == default_port
  42. 568 "#{host}:#{port}"
  43. 25 end unless URI::HTTP.method_defined?(:authority)
  44. def origin
  45. 4477 "#{scheme}://#{authority}"
  46. 25 end unless URI::HTTP.method_defined?(:origin)
  47. end
  48. end
  49. end

lib/httpx/headers.rb

100.0% lines covered

71 relevant lines. 71 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 class Headers
  4. 26 class << self
  5. 26 def new(headers = nil)
  6. 25043 return headers if headers.is_a?(self)
  7. 11472 super
  8. end
  9. end
  10. 26 def initialize(headers = nil)
  11. 11472 @headers = {}
  12. 11472 return unless headers
  13. 11287 headers.each do |field, value|
  14. 58964 array_value(value).each do |v|
  15. 59020 add(downcased(field), v)
  16. end
  17. end
  18. end
  19. # cloned initialization
  20. 26 def initialize_clone(orig)
  21. 8 super
  22. 8 @headers = orig.instance_variable_get(:@headers).clone
  23. end
  24. # dupped initialization
  25. 26 def initialize_dup(orig)
  26. 14490 super
  27. 14490 @headers = orig.instance_variable_get(:@headers).dup
  28. end
  29. # freezes the headers hash
  30. 26 def freeze
  31. 16178 @headers.freeze
  32. 16178 super
  33. end
  34. 26 def same_headers?(headers)
  35. 32 @headers.empty? || begin
  36. 32 headers.each do |k, v|
  37. 72 next unless key?(k)
  38. 72 return false unless v == self[k]
  39. end
  40. 16 true
  41. end
  42. end
  43. # merges headers with another header-quack.
  44. # the merge rule is, if the header already exists,
  45. # ignore what the +other+ headers has. Otherwise, set
  46. #
  47. 26 def merge(other)
  48. 4519 headers = dup
  49. 4519 other.each do |field, value|
  50. 3870 headers[downcased(field)] = value
  51. end
  52. 4519 headers
  53. end
  54. # returns the comma-separated values of the header field
  55. # identified by +field+, or nil otherwise.
  56. #
  57. 26 def [](field)
  58. 88104 a = @headers[downcased(field)] || return
  59. 26209 a.join(", ")
  60. end
  61. # sets +value+ (if not nil) as single value for the +field+ header.
  62. #
  63. 26 def []=(field, value)
  64. 39353 return unless value
  65. 35305 @headers[downcased(field)] = array_value(value)
  66. end
  67. # deletes all values associated with +field+ header.
  68. #
  69. 26 def delete(field)
  70. 259 canonical = downcased(field)
  71. 259 @headers.delete(canonical) if @headers.key?(canonical)
  72. end
  73. # adds additional +value+ to the existing, for header +field+.
  74. #
  75. 26 def add(field, value)
  76. 59492 (@headers[downcased(field)] ||= []) << String(value)
  77. end
  78. # helper to be used when adding an header field as a value to another field
  79. #
  80. # h2_headers.add_header("vary", "accept-encoding")
  81. # h2_headers["vary"] #=> "accept-encoding"
  82. # h1_headers.add_header("vary", "accept-encoding")
  83. # h1_headers["vary"] #=> "Accept-Encoding"
  84. #
  85. 26 alias_method :add_header, :add
  86. # returns the enumerable headers store in pairs of header field + the values in
  87. # the comma-separated string format
  88. #
  89. 26 def each(extra_headers = nil)
  90. 64357 return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
  91. 34329 @headers.each do |field, value|
  92. 44956 yield(field, value.join(", ")) unless value.empty?
  93. end
  94. 5777 extra_headers.each do |field, value|
  95. 23555 yield(field, value) unless value.empty?
  96. 34312 end if extra_headers
  97. end
  98. 26 def ==(other)
  99. 20076 other == to_hash
  100. end
  101. # the headers store in Hash format
  102. 26 def to_hash
  103. 21534 Hash[to_a]
  104. end
  105. 26 alias_method :to_h, :to_hash
  106. # the headers store in array of pairs format
  107. 26 def to_a
  108. 21557 Array(each)
  109. end
  110. # headers as string
  111. 26 def to_s
  112. 1889 @headers.to_s
  113. end
  114. skipped # :nocov:
  115. skipped def inspect
  116. skipped to_hash.inspect
  117. skipped end
  118. skipped # :nocov:
  119. # this is internal API and doesn't abide to other public API
  120. # guarantees, like downcasing strings.
  121. # Please do not use this outside of core!
  122. #
  123. 26 def key?(downcased_key)
  124. 61636 @headers.key?(downcased_key)
  125. end
  126. # returns the values for the +field+ header in array format.
  127. # This method is more internal, and for this reason doesn't try
  128. # to "correct" the user input, i.e. it doesn't downcase the key.
  129. #
  130. 26 def get(field)
  131. 287 @headers[field] || EMPTY
  132. end
  133. 26 private
  134. 26 def array_value(value)
  135. 87811 case value
  136. when Array
  137. 93871 value.map { |val| String(val).strip }
  138. else
  139. 56232 [String(value).strip]
  140. end
  141. end
  142. 26 def downcased(field)
  143. 250450 String(field).downcase
  144. end
  145. end
  146. end

lib/httpx/io.rb

100.0% lines covered

5 relevant lines. 5 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "socket"
  3. 26 require "httpx/io/udp"
  4. 26 require "httpx/io/tcp"
  5. 26 require "httpx/io/unix"
  6. begin
  7. 26 require "httpx/io/ssl"
  8. rescue LoadError
  9. end

lib/httpx/io/ssl.rb

96.47% lines covered

85 relevant lines. 82 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "openssl"
  3. 26 module HTTPX
  4. 26 TLSError = OpenSSL::SSL::SSLError
  5. 26 class SSL < TCP
  6. # rubocop:disable Style/MutableConstant
  7. 26 TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
  8. # https://github.com/jruby/jruby-openssl/issues/284
  9. 26 TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby"
  10. # rubocop:enable Style/MutableConstant
  11. 26 TLS_OPTIONS.freeze
  12. 26 attr_writer :ssl_session
  13. 26 def initialize(_, _, options)
  14. 2871 super
  15. 2871 ctx_options = TLS_OPTIONS.merge(options.ssl)
  16. 2871 @sni_hostname = ctx_options.delete(:hostname) || @hostname
  17. 2871 if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
  18. # externally initiated ssl socket
  19. 22 @ctx = @io.context
  20. 22 @state = :negotiated
  21. else
  22. 2849 @ctx = OpenSSL::SSL::SSLContext.new
  23. 2849 @ctx.set_params(ctx_options) unless ctx_options.empty?
  24. 2849 unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
  25. 2530 @ctx.session_cache_mode =
  26. OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
  27. end
  28. 2849 yield(self) if block_given?
  29. end
  30. 2871 @verify_hostname = @ctx.verify_hostname
  31. end
  32. 26 if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
  33. 25 def session_new_cb(&pr)
  34. 7295 @ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
  35. end
  36. else
  37. # session_new_cb not implemented under JRuby
  38. 1 def session_new_cb; end
  39. end
  40. 26 def protocol
  41. 2755 @io.alpn_protocol || super
  42. rescue StandardError
  43. 7 super
  44. end
  45. 26 if RUBY_ENGINE == "jruby"
  46. # in jruby, alpn_protocol may return ""
  47. # https://github.com/jruby/jruby-openssl/issues/287
  48. 1 def protocol
  49. 349 proto = @io.alpn_protocol
  50. 348 return super if proto.nil? || proto.empty?
  51. 347 proto
  52. rescue StandardError
  53. 1 super
  54. end
  55. end
  56. 26 def can_verify_peer?
  57. 13 @ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
  58. end
  59. 26 def verify_hostname(host)
  60. 15 return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
  61. 15 return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?
  62. 15 OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
  63. end
  64. 26 def connected?
  65. 12061 @state == :negotiated
  66. end
  67. 26 def expired?
  68. super || ssl_session_expired?
  69. end
  70. 26 def ssl_session_expired?
  71. 3064 @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
  72. end
  73. 26 def connect
  74. 12117 super
  75. 12085 return if @state == :negotiated ||
  76. @state != :connected
  77. 8294 unless @io.is_a?(OpenSSL::SSL::SSLSocket)
  78. 3064 if (hostname_is_ip = (@ip == @sni_hostname))
  79. # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
  80. 32 @sni_hostname = @ip.to_string
  81. # IP addresses in SNI is not valid per RFC 6066, section 3.
  82. 32 @ctx.verify_hostname = false
  83. end
  84. 3064 @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
  85. 3064 @io.hostname = @sni_hostname unless hostname_is_ip
  86. 3064 @io.session = @ssl_session unless ssl_session_expired?
  87. 3064 @io.sync_close = true
  88. end
  89. 8294 try_ssl_connect
  90. end
  91. 26 def try_ssl_connect
  92. 8294 ret = @io.connect_nonblock(exception: false)
  93. 8311 log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
  94. 7581 case ret
  95. when :wait_readable
  96. 5254 @interests = :r
  97. 5254 return
  98. when :wait_writable
  99. @interests = :w
  100. return
  101. end
  102. 3017 @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
  103. 3016 transition(:negotiated)
  104. 3016 @interests = :w
  105. end
  106. 26 private
  107. 26 def transition(nextstate)
  108. 10566 case nextstate
  109. when :negotiated
  110. 3016 return unless @state == :connected
  111. when :closed
  112. 2949 return unless @state == :negotiated ||
  113. @state == :connected
  114. end
  115. 11947 do_transition(nextstate)
  116. end
  117. 26 def log_transition_state(nextstate)
  118. 70 return super unless nextstate == :negotiated
  119. 16 server_cert = @io.peer_cert
  120. 14 "#{super}\n\n" \
  121. 2 "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
  122. 2 "ALPN, server accepted to use #{protocol}\n" \
  123. "Server certificate:\n " \
  124. 2 "subject: #{server_cert.subject}\n " \
  125. 2 "start date: #{server_cert.not_before}\n " \
  126. 2 "expire date: #{server_cert.not_after}\n " \
  127. 2 "issuer: #{server_cert.issuer}\n " \
  128. "SSL certificate verify ok."
  129. end
  130. end
  131. end

lib/httpx/io/tcp.rb

91.15% lines covered

113 relevant lines. 103 lines covered and 10 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "resolv"
  3. 26 require "ipaddr"
  4. 26 module HTTPX
  5. 26 class TCP
  6. 26 include Loggable
  7. 26 using URIExtensions
  8. 26 attr_reader :ip, :port, :addresses, :state, :interests
  9. 26 alias_method :host, :ip
  10. 26 def initialize(origin, addresses, options)
  11. 6854 @state = :idle
  12. 6854 @addresses = []
  13. 6854 @hostname = origin.host
  14. 6854 @options = options
  15. 6854 @fallback_protocol = @options.fallback_protocol
  16. 6854 @port = origin.port
  17. 6854 @interests = :w
  18. 6854 if @options.io
  19. 52 @io = case @options.io
  20. when Hash
  21. 16 @options.io[origin.authority]
  22. else
  23. 36 @options.io
  24. end
  25. 52 raise Error, "Given IO objects do not match the request authority" unless @io
  26. 52 _, _, _, @ip = @io.addr
  27. 52 @addresses << @ip
  28. 52 @keep_open = true
  29. 52 @state = :connected
  30. else
  31. 6802 add_addresses(addresses)
  32. end
  33. 6854 @ip_index = @addresses.size - 1
  34. end
  35. 26 def socket
  36. 195 @io
  37. end
  38. 26 def add_addresses(addrs)
  39. 7025 return if addrs.empty?
  40. 21668 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  41. 7025 ip_index = @ip_index || (@addresses.size - 1)
  42. 7025 if addrs.first.ipv6?
  43. # should be the next in line
  44. 233 @addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
  45. else
  46. 6792 @addresses.unshift(*addrs)
  47. 6792 @ip_index += addrs.size if @ip_index
  48. end
  49. end
  50. 26 def to_io
  51. 22160 @io.to_io
  52. end
  53. 26 def protocol
  54. 4329 @fallback_protocol
  55. end
  56. 26 def connect
  57. 38915 return unless closed?
  58. 33469 if !@io || @io.closed?
  59. 7492 transition(:idle)
  60. 7492 @io = build_socket
  61. end
  62. 33469 try_connect
  63. rescue Errno::ECONNREFUSED,
  64. Errno::EADDRNOTAVAIL,
  65. Errno::EHOSTUNREACH,
  66. SocketError,
  67. IOError => e
  68. 881 raise e if @ip_index <= 0
  69. 819 log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  70. 798 @ip_index -= 1
  71. 807 @io = build_socket
  72. 807 retry
  73. rescue Errno::ETIMEDOUT => e
  74. raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
  75. log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  76. @ip_index -= 1
  77. @io = build_socket
  78. retry
  79. end
  80. 26 def try_connect
  81. 33469 ret = @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
  82. 15809 log(level: 3, color: :cyan) { "TCP CONNECT: #{ret}..." }
  83. 14002 case ret
  84. when :wait_readable
  85. @interests = :r
  86. return
  87. when :wait_writable
  88. 8287 @interests = :w
  89. 8287 return
  90. end
  91. 7418 transition(:connected)
  92. 7418 @interests = :w
  93. rescue Errno::EALREADY
  94. 16883 @interests = :w
  95. end
  96. 26 private :try_connect
  97. 26 def read(size, buffer)
  98. 49618 ret = @io.read_nonblock(size, buffer, exception: false)
  99. 49618 if ret == :wait_readable
  100. 10327 buffer.clear
  101. 9558 return 0
  102. end
  103. 39291 return if ret.nil?
  104. 39349 log { "READ: #{buffer.bytesize} bytes..." }
  105. 39273 buffer.bytesize
  106. end
  107. 26 def write(buffer)
  108. 20616 siz = @io.write_nonblock(buffer, exception: false)
  109. 20598 return 0 if siz == :wait_writable
  110. 20574 return if siz.nil?
  111. 20660 log { "WRITE: #{siz} bytes..." }
  112. 20574 buffer.shift!(siz)
  113. 20574 siz
  114. end
  115. 26 def close
  116. 8191 return if @keep_open || closed?
  117. 831 begin
  118. 7241 @io.close
  119. ensure
  120. 7241 transition(:closed)
  121. end
  122. end
  123. 26 def connected?
  124. 25963 @state == :connected
  125. end
  126. 26 def closed?
  127. 47093 @state == :idle || @state == :closed
  128. end
  129. 26 def expired?
  130. # do not mess with external sockets
  131. return false if @options.io
  132. return true unless @addresses
  133. resolver_addresses = Resolver.nolookup_resolve(@hostname)
  134. (Array(resolver_addresses) & @addresses).empty?
  135. end
  136. skipped # :nocov:
  137. skipped def inspect
  138. skipped "#<#{self.class}: #{@ip}:#{@port} (state: #{@state})>"
  139. skipped end
  140. skipped # :nocov:
  141. 26 private
  142. 26 def build_socket
  143. 8299 @ip = @addresses[@ip_index]
  144. 8299 Socket.new(@ip.family, :STREAM, 0)
  145. end
  146. 26 def transition(nextstate)
  147. 11739 case nextstate
  148. # when :idle
  149. when :connected
  150. 4457 return unless @state == :idle
  151. when :closed
  152. 4292 return unless @state == :connected
  153. end
  154. 13241 do_transition(nextstate)
  155. end
  156. 26 def do_transition(nextstate)
  157. 25336 log(level: 1) { log_transition_state(nextstate) }
  158. 25188 @state = nextstate
  159. end
  160. 26 def log_transition_state(nextstate)
  161. 131 case nextstate
  162. when :connected
  163. 40 "Connected to #{host} (##{@io.fileno})"
  164. else
  165. 96 "#{host} #{@state} -> #{nextstate}"
  166. end
  167. end
  168. end
  169. end

lib/httpx/io/udp.rb

100.0% lines covered

35 relevant lines. 35 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "ipaddr"
  3. 26 module HTTPX
  4. 26 class UDP
  5. 26 include Loggable
  6. 26 def initialize(ip, port, options)
  7. 359 @host = ip
  8. 359 @port = port
  9. 359 @io = UDPSocket.new(IPAddr.new(ip).family)
  10. 359 @options = options
  11. end
  12. 26 def to_io
  13. 1024 @io.to_io
  14. end
  15. 26 def connect; end
  16. 26 def connected?
  17. 359 true
  18. end
  19. 26 def close
  20. 361 @io.close
  21. end
  22. 26 if RUBY_ENGINE == "jruby"
  23. # In JRuby, sendmsg_nonblock is not implemented
  24. 1 def write(buffer)
  25. 52 siz = @io.send(buffer.to_s, 0, @host, @port)
  26. 52 log { "WRITE: #{siz} bytes..." }
  27. 52 buffer.shift!(siz)
  28. 52 siz
  29. end
  30. else
  31. 25 def write(buffer)
  32. 657 siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
  33. 657 return 0 if siz == :wait_writable
  34. 657 return if siz.nil?
  35. 657 log { "WRITE: #{siz} bytes..." }
  36. 657 buffer.shift!(siz)
  37. 657 siz
  38. end
  39. end
  40. 26 def read(size, buffer)
  41. 1202 ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
  42. 1202 return 0 if ret == :wait_readable
  43. 637 return if ret.nil?
  44. 637 log { "READ: #{buffer.bytesize} bytes..." }
  45. 637 buffer.bytesize
  46. rescue IOError
  47. end
  48. end
  49. end

lib/httpx/io/unix.rb

94.29% lines covered

35 relevant lines. 33 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 class UNIX < TCP
  4. 26 using URIExtensions
  5. 26 attr_reader :path
  6. 26 alias_method :host, :path
  7. 26 def initialize(origin, path, options)
  8. 28 @addresses = []
  9. 28 @hostname = origin.host
  10. 28 @state = :idle
  11. 28 @options = options
  12. 28 @fallback_protocol = @options.fallback_protocol
  13. 28 if @options.io
  14. 14 @io = case @options.io
  15. when Hash
  16. 7 @options.io[origin.authority]
  17. else
  18. 7 @options.io
  19. end
  20. 14 raise Error, "Given IO objects do not match the request authority" unless @io
  21. 14 @path = @io.path
  22. 14 @keep_open = true
  23. 14 @state = :connected
  24. 14 elsif path
  25. 14 @path = path
  26. else
  27. raise Error, "No path given where to store the socket"
  28. end
  29. 28 @io ||= build_socket
  30. end
  31. 26 def connect
  32. 21 return unless closed?
  33. begin
  34. 21 if @io.closed?
  35. 7 transition(:idle)
  36. 7 @io = build_socket
  37. end
  38. 21 @io.connect_nonblock(Socket.sockaddr_un(@path))
  39. rescue Errno::EISCONN
  40. end
  41. 14 transition(:connected)
  42. rescue Errno::EINPROGRESS,
  43. Errno::EALREADY,
  44. ::IO::WaitReadable
  45. end
  46. 26 def expired?
  47. false
  48. end
  49. skipped # :nocov:
  50. skipped def inspect
  51. skipped "#<#{self.class}(path: #{@path}): (state: #{@state})>"
  52. skipped end
  53. skipped # :nocov:
  54. 26 private
  55. 26 def build_socket
  56. 21 Socket.new(Socket::PF_UNIX, :STREAM, 0)
  57. end
  58. end
  59. end

lib/httpx/loggable.rb

100.0% lines covered

17 relevant lines. 17 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Loggable
  4. 26 COLORS = {
  5. black: 30,
  6. red: 31,
  7. green: 32,
  8. yellow: 33,
  9. blue: 34,
  10. magenta: 35,
  11. cyan: 36,
  12. white: 37,
  13. }.freeze
  14. 26 USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
  15. 26 def log(level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug, &msg)
  16. 430303 return unless debug_level >= level
  17. 218679 debug_stream = debug || ($stderr if USE_DEBUG_LOG)
  18. 218679 return unless debug_stream
  19. 1910 klass = self.class
  20. 3714 until (class_name = klass.name)
  21. 364 klass = klass.superclass
  22. end
  23. 1910 message = +"(pid:#{Process.pid} tid:#{Thread.current.object_id}, self:#{class_name}##{object_id}) "
  24. 1910 message << msg.call << "\n"
  25. 1910 message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
  26. 1910 debug_stream << message
  27. end
  28. 26 def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
  29. 1199 log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
  30. end
  31. end
  32. end

lib/httpx/options.rb

98.73% lines covered

157 relevant lines. 155 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "socket"
  3. 26 module HTTPX
  4. # Contains a set of options which are passed and shared across from session to its requests or
  5. # responses.
  6. 26 class Options
  7. 26 BUFFER_SIZE = 1 << 14
  8. 26 WINDOW_SIZE = 1 << 14 # 16K
  9. 26 MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
  10. 26 KEEP_ALIVE_TIMEOUT = 20
  11. 26 SETTINGS_TIMEOUT = 10
  12. 26 CLOSE_HANDSHAKE_TIMEOUT = 10
  13. 26 CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
  14. 26 REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
  15. # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
  16. 2 ip_address_families = begin
  17. 26 list = Socket.ip_address_list
  18. 106 if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? && !a.ipv6_unique_local? }
  19. [Socket::AF_INET6, Socket::AF_INET]
  20. else
  21. 26 [Socket::AF_INET]
  22. end
  23. rescue NotImplementedError
  24. [Socket::AF_INET]
  25. end.freeze
  26. 2 DEFAULT_OPTIONS = {
  27. 24 :max_requests => Float::INFINITY,
  28. :debug => nil,
  29. 26 :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
  30. :ssl => EMPTY_HASH,
  31. :http2_settings => { settings_enable_push: 0 }.freeze,
  32. :fallback_protocol => "http/1.1",
  33. :supported_compression_formats => %w[gzip deflate],
  34. :decompress_response_body => true,
  35. :compress_request_body => true,
  36. :timeout => {
  37. connect_timeout: CONNECT_TIMEOUT,
  38. settings_timeout: SETTINGS_TIMEOUT,
  39. close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
  40. operation_timeout: OPERATION_TIMEOUT,
  41. keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
  42. read_timeout: READ_TIMEOUT,
  43. write_timeout: WRITE_TIMEOUT,
  44. request_timeout: REQUEST_TIMEOUT,
  45. },
  46. :headers_class => Class.new(Headers),
  47. :headers => {},
  48. :window_size => WINDOW_SIZE,
  49. :buffer_size => BUFFER_SIZE,
  50. :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
  51. :request_class => Class.new(Request),
  52. :response_class => Class.new(Response),
  53. :request_body_class => Class.new(Request::Body),
  54. :response_body_class => Class.new(Response::Body),
  55. :pool_class => Class.new(Pool),
  56. :connection_class => Class.new(Connection),
  57. :options_class => Class.new(self),
  58. :transport => nil,
  59. :addresses => nil,
  60. :persistent => false,
  61. 26 :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
  62. :resolver_options => { cache: true }.freeze,
  63. :pool_options => EMPTY_HASH,
  64. :ip_families => ip_address_families,
  65. }.freeze
  66. 26 class << self
  67. 26 def new(options = {})
  68. # let enhanced options go through
  69. 11719 return options if self == Options && options.class < self
  70. 9075 return options if options.is_a?(self)
  71. 4531 super
  72. end
  73. 26 def method_added(meth)
  74. 20012 super
  75. 20012 return unless meth =~ /^option_(.+)$/
  76. 9340 optname = Regexp.last_match(1).to_sym
  77. 9340 attr_reader(optname)
  78. end
  79. end
  80. # creates a new options instance from a given hash, which optionally define the following:
  81. #
  82. # :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
  83. # :debug_level :: the log level of messages (can be 1, 2, or 3).
  84. # :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::IO::SSL)
  85. # :http2_settings :: a hash of options to be passed to a HTTP2::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
  86. # :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
  87. # like ALPN (defaults to <tt>"http/1.1"</tt>)
  88. # :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
  89. # :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
  90. # :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
  91. # :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
  92. # <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>, <tt>:read_timeout</tt>, <tt>:write_timeout</tt>
  93. # and <tt>:request_timeout</tt>
  94. # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
  95. # :window_size :: number of bytes to read from a socket
  96. # :buffer_size :: internal read and write buffer size in bytes
  97. # :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
  98. # :request_class :: class used to instantiate a request
  99. # :response_class :: class used to instantiate a response
  100. # :headers_class :: class used to instantiate headers
  101. # :request_body_class :: class used to instantiate a request body
  102. # :response_body_class :: class used to instantiate a response body
  103. # :connection_class :: class used to instantiate connections
  104. # :pool_class :: class used to instantiate the session connection pool
  105. # :options_class :: class used to instantiate options
  106. # :transport :: type of transport to use (set to "unix" for UNIX sockets)
  107. # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
  108. # paths should be used for UNIX sockets instead)
  109. # :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
  110. # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
  111. # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
  112. # using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
  113. # :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
  114. # :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
  115. # :ip_families :: which socket families are supported (system-dependent)
  116. # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
  117. # :base_path :: path to prefix given relative paths with (ex: "/v2")
  118. # :max_concurrent_requests :: max number of requests which can be set concurrently
  119. # :max_requests :: max number of requests which can be made on socket before it reconnects.
  120. #
  121. # This list of options are enhanced with each loaded plugin, see the plugin docs for details.
  122. 26 def initialize(options = {})
  123. 4531 do_initialize(options)
  124. 4515 freeze
  125. end
  126. 26 def freeze
  127. 11651 super
  128. 11651 @origin.freeze
  129. 11651 @base_path.freeze
  130. 11651 @timeout.freeze
  131. 11651 @headers.freeze
  132. 11651 @addresses.freeze
  133. 11651 @supported_compression_formats.freeze
  134. end
  135. 26 def option_origin(value)
  136. 608 URI(value)
  137. end
  138. 26 def option_base_path(value)
  139. 32 String(value)
  140. end
  141. 26 def option_headers(value)
  142. 7934 headers_class.new(value)
  143. end
  144. 26 def option_timeout(value)
  145. 8519 Hash[value]
  146. end
  147. 26 def option_supported_compression_formats(value)
  148. 7342 Array(value).map(&:to_s)
  149. end
  150. 26 def option_max_concurrent_requests(value)
  151. 963 raise TypeError, ":max_concurrent_requests must be positive" unless value.positive?
  152. 963 value
  153. end
  154. 26 def option_max_requests(value)
  155. 7328 raise TypeError, ":max_requests must be positive" unless value.positive?
  156. 7328 value
  157. end
  158. 26 def option_window_size(value)
  159. 7334 value = Integer(value)
  160. 7334 raise TypeError, ":window_size must be positive" unless value.positive?
  161. 7334 value
  162. end
  163. 26 def option_buffer_size(value)
  164. 7334 value = Integer(value)
  165. 7334 raise TypeError, ":buffer_size must be positive" unless value.positive?
  166. 7334 value
  167. end
  168. 26 def option_body_threshold_size(value)
  169. 7318 bytes = Integer(value)
  170. 7318 raise TypeError, ":body_threshold_size must be positive" unless bytes.positive?
  171. 7318 bytes
  172. end
  173. 26 def option_transport(value)
  174. 49 transport = value.to_s
  175. 49 raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
  176. 49 transport
  177. end
  178. 26 def option_addresses(value)
  179. 43 Array(value)
  180. end
  181. 26 def option_ip_families(value)
  182. 7318 Array(value)
  183. end
  184. 26 %i[
  185. ssl http2_settings
  186. request_class response_class headers_class request_body_class
  187. response_body_class connection_class options_class
  188. pool_class pool_options
  189. io fallback_protocol debug debug_level resolver_class resolver_options
  190. compress_request_body decompress_response_body
  191. persistent
  192. ].each do |method_name|
  193. 520 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  194. 20 # sets +v+ as the value of #{method_name}
  195. 20 def option_#{method_name}(v); v; end # def option_smth(v); v; end
  196. OUT
  197. end
  198. 26 REQUEST_BODY_IVARS = %i[@headers].freeze
  199. 26 def ==(other)
  200. 1904 super || options_equals?(other)
  201. end
  202. 26 def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
  203. # headers and other request options do not play a role, as they are
  204. # relevant only for the request.
  205. 460 ivars = instance_variables - ignore_ivars
  206. 460 other_ivars = other.instance_variables - ignore_ivars
  207. 460 return false if ivars.size != other_ivars.size
  208. 444 return false if ivars.sort != other_ivars.sort
  209. 444 ivars.all? do |ivar|
  210. 10679 instance_variable_get(ivar) == other.instance_variable_get(ivar)
  211. end
  212. end
  213. 26 def merge(other)
  214. 34323 ivar_map = nil
  215. 34323 other_ivars = case other
  216. when Hash
  217. 41191 ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
  218. 23724 ivar_map.keys
  219. else
  220. 10599 other.instance_variables
  221. end
  222. 34323 return self if other_ivars.empty?
  223. 284665 return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == access_option(other, ivar, ivar_map) }
  224. 12682 opts = dup
  225. 12682 other_ivars.each do |ivar|
  226. 91221 v = access_option(other, ivar, ivar_map)
  227. 91221 unless v
  228. 3398 opts.instance_variable_set(ivar, v)
  229. 3398 next
  230. end
  231. 87823 v = opts.__send__(:"option_#{ivar[1..-1]}", v)
  232. 87807 orig_v = instance_variable_get(ivar)
  233. 87807 v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
  234. 87807 opts.instance_variable_set(ivar, v)
  235. end
  236. 12666 opts
  237. end
  238. 26 def to_hash
  239. 3382 instance_variables.each_with_object({}) do |ivar, hs|
  240. 79118 hs[ivar[1..-1].to_sym] = instance_variable_get(ivar)
  241. end
  242. end
  243. 26 def extend_with_plugin_classes(pl)
  244. 7094 if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
  245. 2016 @request_class = @request_class.dup
  246. 2016 @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
  247. 2016 @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
  248. end
  249. 7094 if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
  250. 2278 @response_class = @response_class.dup
  251. 2278 @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
  252. 2278 @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
  253. end
  254. 7094 if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
  255. 152 @headers_class = @headers_class.dup
  256. 152 @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
  257. 152 @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
  258. end
  259. 7094 if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
  260. 314 @request_body_class = @request_body_class.dup
  261. 314 @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
  262. 314 @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
  263. end
  264. 7094 if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
  265. 760 @response_body_class = @response_body_class.dup
  266. 760 @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
  267. 760 @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
  268. end
  269. 7094 if defined?(pl::PoolMethods)
  270. 555 @pool_class = @pool_class.dup
  271. 555 @pool_class.__send__(:include, pl::PoolMethods)
  272. end
  273. 7094 if defined?(pl::ConnectionMethods)
  274. 2899 @connection_class = @connection_class.dup
  275. 2899 @connection_class.__send__(:include, pl::ConnectionMethods)
  276. end
  277. 7094 return unless defined?(pl::OptionsMethods)
  278. 2916 @options_class = @options_class.dup
  279. 2916 @options_class.__send__(:include, pl::OptionsMethods)
  280. end
  281. 26 private
  282. 26 def do_initialize(options = {})
  283. 4531 defaults = DEFAULT_OPTIONS.merge(options)
  284. 4531 defaults.each do |k, v|
  285. 132297 next if v.nil?
  286. 118704 option_method_name = :"option_#{k}"
  287. 118704 raise Error, "unknown option: #{k}" unless respond_to?(option_method_name)
  288. 118696 value = __send__(option_method_name, v)
  289. 118688 instance_variable_set(:"@#{k}", value)
  290. end
  291. end
  292. 26 def access_option(obj, k, ivar_map)
  293. 318656 case obj
  294. when Hash
  295. 28134 obj[ivar_map[k]]
  296. else
  297. 326544 obj.instance_variable_get(k)
  298. end
  299. end
  300. end
  301. end

lib/httpx/parser/http1.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Parser
  4. 26 class Error < Error; end
  5. 26 class HTTP1
  6. 26 VERSIONS = %w[1.0 1.1].freeze
  7. 26 attr_reader :status_code, :http_version, :headers
  8. 26 def initialize(observer)
  9. 4479 @observer = observer
  10. 4479 @state = :idle
  11. 4479 @buffer = "".b
  12. 4479 @headers = {}
  13. end
  14. 26 def <<(chunk)
  15. 7194 @buffer << chunk
  16. 7194 parse
  17. end
  18. 26 def reset!
  19. 9145 @state = :idle
  20. 9145 @headers.clear
  21. 9145 @content_length = nil
  22. 9145 @_has_trailers = nil
  23. end
  24. 26 def upgrade?
  25. 4484 @upgrade
  26. end
  27. 26 def upgrade_data
  28. 32 @buffer
  29. end
  30. 26 private
  31. 26 def parse
  32. 7194 loop do
  33. 15208 state = @state
  34. 13619 case @state
  35. when :idle
  36. 4829 parse_headline
  37. when :headers, :trailers
  38. 4917 parse_headers
  39. when :data
  40. 5460 parse_data
  41. end
  42. 11147 return if @buffer.empty? || state == @state
  43. end
  44. end
  45. 26 def parse_headline
  46. 4829 idx = @buffer.index("\n")
  47. 4829 return unless idx
  48. 4829 (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
  49. raise(Error, "wrong head line format")
  50. 4821 version, code, _ = m.captures
  51. 4821 raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
  52. 4813 @http_version = version.split(".").map(&:to_i)
  53. 4813 @status_code = code.to_i
  54. 4813 raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
  55. 4805 @buffer = @buffer.byteslice((idx + 1)..-1)
  56. 4805 nextstate(:headers)
  57. end
  58. 26 def parse_headers
  59. 4919 headers = @headers
  60. 4919 buffer = @buffer
  61. 38107 while (idx = buffer.index("\n"))
  62. # @type var line: String
  63. 37499 line = buffer.byteslice(0..idx)
  64. 37499 raise Error, "wrong header format" if line.start_with?("\s", "\t")
  65. 37491 line.lstrip!
  66. 37491 buffer = @buffer = buffer.byteslice((idx + 1)..-1)
  67. 37491 if line.empty?
  68. 4311 case @state
  69. when :headers
  70. 4789 prepare_data(headers)
  71. 4789 @observer.on_headers(headers)
  72. 4178 return unless @state == :headers
  73. # state might have been reset
  74. # in the :headers callback
  75. 4106 nextstate(:data)
  76. 4106 headers.clear
  77. when :trailers
  78. 16 @observer.on_trailers(headers)
  79. 16 headers.clear
  80. 16 nextstate(:complete)
  81. end
  82. 4114 return
  83. end
  84. 32686 separator_index = line.index(":")
  85. 32686 raise Error, "wrong header format" unless separator_index
  86. # @type var key: String
  87. 32678 key = line.byteslice(0..(separator_index - 1))
  88. 32678 key.rstrip! # was lstripped previously!
  89. # @type var value: String
  90. 32678 value = line.byteslice((separator_index + 1)..-1)
  91. 32678 value.strip!
  92. 32678 raise Error, "wrong header format" if value.nil?
  93. 32678 (headers[key.downcase] ||= []) << value
  94. end
  95. end
  96. 26 def parse_data
  97. 5460 if @buffer.respond_to?(:each)
  98. 203 @buffer.each do |chunk|
  99. 234 @observer.on_data(chunk)
  100. end
  101. 5256 elsif @content_length
  102. # @type var data: String
  103. 5217 data = @buffer.byteslice(0, @content_length)
  104. 5217 @buffer = @buffer.byteslice(@content_length..-1) || "".b
  105. 4663 @content_length -= data.bytesize
  106. 5217 @observer.on_data(data)
  107. 5203 data.clear
  108. else
  109. 40 @observer.on_data(@buffer)
  110. 40 @buffer.clear
  111. end
  112. 5438 return unless no_more_data?
  113. 3964 @buffer = @buffer.to_s
  114. 3964 if @_has_trailers
  115. 16 nextstate(:trailers)
  116. else
  117. 3948 nextstate(:complete)
  118. end
  119. end
  120. 26 def prepare_data(headers)
  121. 4789 @upgrade = headers.key?("upgrade")
  122. 4789 @_has_trailers = headers.key?("trailer")
  123. 4789 if (tr_encodings = headers["transfer-encoding"])
  124. 114 tr_encodings.reverse_each do |tr_encoding|
  125. 114 tr_encoding.split(/ *, */).each do |encoding|
  126. 100 case encoding
  127. when "chunked"
  128. 114 @buffer = Transcoder::Chunker::Decoder.new(@buffer, @_has_trailers)
  129. end
  130. end
  131. end
  132. else
  133. 4675 @content_length = headers["content-length"][0].to_i if headers.key?("content-length")
  134. end
  135. end
  136. 26 def no_more_data?
  137. 5438 if @content_length
  138. 5203 @content_length <= 0
  139. 234 elsif @buffer.respond_to?(:finished?)
  140. 195 @buffer.finished?
  141. else
  142. 40 false
  143. end
  144. end
  145. 26 def nextstate(state)
  146. 12891 @state = state
  147. 11576 case state
  148. when :headers
  149. 4805 @observer.on_start
  150. when :complete
  151. 3964 @observer.on_complete
  152. 576 reset!
  153. 576 nextstate(:idle) unless @buffer.empty?
  154. end
  155. end
  156. end
  157. end
  158. end

lib/httpx/plugins/auth.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds a shim +authorization+ method to the session, which will fill
  6. # the HTTP Authorization header, and another, +bearer_auth+, which fill the "Bearer " prefix
  7. # in its value.
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/Auth#auth
  10. #
  11. 8 module Auth
  12. 8 module InstanceMethods
  13. 8 def authorization(token)
  14. 144 with(headers: { "authorization" => token })
  15. end
  16. 8 def bearer_auth(token)
  17. 16 authorization("Bearer #{token}")
  18. end
  19. end
  20. end
  21. 8 register_plugin :auth, Auth
  22. end
  23. end

lib/httpx/plugins/auth/basic.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 9 require "httpx/base64"
  3. 9 module HTTPX
  4. 9 module Plugins
  5. 9 module Authentication
  6. 9 class Basic
  7. 9 def initialize(user, password, **)
  8. 274 @user = user
  9. 274 @password = password
  10. end
  11. 9 def authenticate(*)
  12. 255 "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
  13. end
  14. end
  15. end
  16. end
  17. end

lib/httpx/plugins/auth/digest.rb

100.0% lines covered

61 relevant lines. 61 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "time"
  3. 8 require "securerandom"
  4. 8 require "digest"
  5. 8 module HTTPX
  6. 8 module Plugins
  7. 8 module Authentication
  8. 8 class Digest
  9. 8 def initialize(user, password, hashed: false, **)
  10. 176 @user = user
  11. 176 @password = password
  12. 176 @nonce = 0
  13. 176 @hashed = hashed
  14. end
  15. 8 def can_authenticate?(authenticate)
  16. 160 authenticate && /Digest .*/.match?(authenticate)
  17. end
  18. 8 def authenticate(request, authenticate)
  19. 160 "Digest #{generate_header(request.verb, request.path, authenticate)}"
  20. end
  21. 8 private
  22. 8 def generate_header(meth, uri, authenticate)
  23. # discard first token, it's Digest
  24. 160 auth_info = authenticate[/^(\w+) (.*)/, 2]
  25. 160 params = auth_info.split(/ *, */)
  26. 832 .to_h { |val| val.split("=", 2) }
  27. 832 .transform_values { |v| v.delete("\"") }
  28. 160 nonce = params["nonce"]
  29. 160 nc = next_nonce
  30. # verify qop
  31. 160 qop = params["qop"]
  32. 160 if params["algorithm"] =~ /(.*?)(-sess)?$/
  33. 144 alg = Regexp.last_match(1)
  34. 144 algorithm = ::Digest.const_get(alg)
  35. 144 raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
  36. 144 sess = Regexp.last_match(2)
  37. else
  38. 16 algorithm = ::Digest::MD5
  39. end
  40. 160 if qop || sess
  41. 160 cnonce = make_cnonce
  42. 160 nc = format("%<nonce>08x", nonce: nc)
  43. end
  44. 160 a1 = if sess
  45. 4 [
  46. 32 (@hashed ? @password : algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}")),
  47. nonce,
  48. cnonce,
  49. 3 ].join ":"
  50. else
  51. 128 @hashed ? @password : "#{@user}:#{params["realm"]}:#{@password}"
  52. end
  53. 160 ha1 = algorithm.hexdigest(a1)
  54. 160 ha2 = algorithm.hexdigest("#{meth}:#{uri}")
  55. 160 request_digest = [ha1, nonce]
  56. 160 request_digest.push(nc, cnonce, qop) if qop
  57. 160 request_digest << ha2
  58. 160 request_digest = request_digest.join(":")
  59. 40 header = [
  60. 140 %(username="#{@user}"),
  61. 20 %(nonce="#{nonce}"),
  62. 20 %(uri="#{uri}"),
  63. 20 %(response="#{algorithm.hexdigest(request_digest)}"),
  64. ]
  65. 160 header << %(realm="#{params["realm"]}") if params.key?("realm")
  66. 160 header << %(algorithm=#{params["algorithm"]}) if params.key?("algorithm")
  67. 160 header << %(cnonce="#{cnonce}") if cnonce
  68. 160 header << %(nc=#{nc})
  69. 160 header << %(qop=#{qop}) if qop
  70. 160 header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
  71. 160 header.join ", "
  72. end
  73. 8 def make_cnonce
  74. 180 ::Digest::MD5.hexdigest [
  75. Time.now.to_i,
  76. Process.pid,
  77. SecureRandom.random_number(2**32),
  78. ].join ":"
  79. end
  80. 8 def next_nonce
  81. 140 @nonce += 1
  82. end
  83. end
  84. end
  85. end
  86. end

lib/httpx/plugins/auth/ntlm.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 3 require "httpx/base64"
  3. 3 require "ntlm"
  4. 3 module HTTPX
  5. 3 module Plugins
  6. 3 module Authentication
  7. 3 class Ntlm
  8. 3 def initialize(user, password, domain: nil)
  9. 4 @user = user
  10. 4 @password = password
  11. 4 @domain = domain
  12. end
  13. 3 def can_authenticate?(authenticate)
  14. 2 authenticate && /NTLM .*/.match?(authenticate)
  15. end
  16. 3 def negotiate
  17. 4 "NTLM #{NTLM.negotiate(domain: @domain).to_base64}"
  18. end
  19. 3 def authenticate(_req, www)
  20. 2 challenge = www[/NTLM (.*)/, 1]
  21. 2 challenge = Base64.decode64(challenge)
  22. 2 ntlm_challenge = NTLM.authenticate(challenge, @user, @domain, @password).to_base64
  23. 2 "NTLM #{ntlm_challenge}"
  24. end
  25. end
  26. end
  27. end
  28. end

lib/httpx/plugins/auth/socks5.rb

100.0% lines covered

11 relevant lines. 11 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 module Plugins
  4. 10 module Authentication
  5. 10 class Socks5
  6. 10 def initialize(user, password, **)
  7. 48 @user = user
  8. 48 @password = password
  9. end
  10. 10 def can_authenticate?(*)
  11. 48 @user && @password
  12. end
  13. 10 def authenticate(*)
  14. 48 [0x01, @user.bytesize, @user, @password.bytesize, @password].pack("CCA*CA*")
  15. end
  16. end
  17. end
  18. end
  19. end

lib/httpx/plugins/aws_sdk_authentication.rb

100.0% lines covered

43 relevant lines. 43 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
  6. #
  7. # It requires the "aws-sdk-core" gem.
  8. #
  9. 8 module AwsSdkAuthentication
  10. # Mock configuration, to be used only when resolving credentials
  11. 8 class Configuration
  12. 8 attr_reader :profile
  13. 8 def initialize(profile)
  14. 32 @profile = profile
  15. end
  16. 8 def respond_to_missing?(*)
  17. 16 true
  18. end
  19. 8 def method_missing(*); end
  20. end
  21. #
  22. # encapsulates access to an AWS SDK credentials store.
  23. #
  24. 8 class Credentials
  25. 8 def initialize(aws_credentials)
  26. 16 @aws_credentials = aws_credentials
  27. end
  28. 8 def username
  29. 16 @aws_credentials.access_key_id
  30. end
  31. 8 def password
  32. 16 @aws_credentials.secret_access_key
  33. end
  34. 8 def security_token
  35. 16 @aws_credentials.session_token
  36. end
  37. end
  38. 8 class << self
  39. 8 def load_dependencies(_klass)
  40. 16 require "aws-sdk-core"
  41. end
  42. 8 def configure(klass)
  43. 16 klass.plugin(:aws_sigv4)
  44. end
  45. 8 def extra_options(options)
  46. 16 options.merge(max_concurrent_requests: 1)
  47. end
  48. 8 def credentials(profile)
  49. 16 mock_configuration = Configuration.new(profile)
  50. 16 Credentials.new(Aws::CredentialProviderChain.new(mock_configuration).resolve)
  51. end
  52. 8 def region(profile)
  53. # https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb#L62
  54. 16 keys = %w[AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION]
  55. 16 env_region = ENV.values_at(*keys).compact.first
  56. 16 env_region = nil if env_region == ""
  57. 16 cfg_region = Aws.shared_config.region(profile: profile)
  58. 16 env_region || cfg_region
  59. end
  60. end
  61. # adds support for the following options:
  62. #
  63. # :aws_profile :: AWS account profile to retrieve credentials from.
  64. 8 module OptionsMethods
  65. 8 def option_aws_profile(value)
  66. 80 String(value)
  67. end
  68. end
  69. 8 module InstanceMethods
  70. #
  71. # aws_authentication
  72. # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
  73. # aws_authentication()
  74. #
  75. 8 def aws_sdk_authentication(
  76. credentials: AwsSdkAuthentication.credentials(@options.aws_profile),
  77. region: AwsSdkAuthentication.region(@options.aws_profile),
  78. **options
  79. )
  80. 16 aws_sigv4_authentication(
  81. credentials: credentials,
  82. region: region,
  83. provider_prefix: "aws",
  84. header_provider_field: "amz",
  85. **options
  86. )
  87. end
  88. 8 alias_method :aws_auth, :aws_sdk_authentication
  89. end
  90. end
  91. 8 register_plugin :aws_sdk_authentication, AwsSdkAuthentication
  92. end
  93. end

lib/httpx/plugins/aws_sigv4.rb

100.0% lines covered

120 relevant lines. 120 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds AWS Sigv4 authentication.
  6. #
  7. # https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/AWS-SigV4
  10. #
  11. 8 module AWSSigV4
  12. 8 Credentials = Struct.new(:username, :password, :security_token)
  13. # Signs requests using the AWS sigv4 signing.
  14. 8 class Signer
  15. 8 def initialize(
  16. service:,
  17. region:,
  18. credentials: nil,
  19. username: nil,
  20. password: nil,
  21. security_token: nil,
  22. provider_prefix: "aws",
  23. header_provider_field: "amz",
  24. unsigned_headers: [],
  25. apply_checksum_header: true,
  26. algorithm: "SHA256"
  27. )
  28. 152 @credentials = credentials || Credentials.new(username, password, security_token)
  29. 152 @service = service
  30. 152 @region = region
  31. 152 @unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
  32. 152 @unsigned_headers << "authorization"
  33. 152 @unsigned_headers << "x-amzn-trace-id"
  34. 152 @unsigned_headers << "expect"
  35. 152 @apply_checksum_header = apply_checksum_header
  36. 152 @provider_prefix = provider_prefix
  37. 152 @header_provider_field = header_provider_field
  38. 152 @algorithm = algorithm
  39. end
  40. 8 def sign!(request)
  41. 152 lower_provider_prefix = "#{@provider_prefix}4"
  42. 152 upper_provider_prefix = lower_provider_prefix.upcase
  43. 152 downcased_algorithm = @algorithm.downcase
  44. 152 datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
  45. 152 date = datetime[0, 8]
  46. 152 content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
  47. 144 request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
  48. 144 request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
  49. 144 signature_headers = request.headers.each.reject do |k, _|
  50. 984 @unsigned_headers.include?(k)
  51. end
  52. # aws sigv4 needs to declare the host, regardless of protocol version
  53. 144 signature_headers << ["host", request.authority] unless request.headers.key?("host")
  54. 144 signature_headers.sort_by!(&:first)
  55. 144 signed_headers = signature_headers.map(&:first).join(";")
  56. 144 canonical_headers = signature_headers.map do |k, v|
  57. # eliminate whitespace between value fields, unless it's a quoted value
  58. 847 "#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
  59. end.join
  60. # canonical request
  61. 144 creq = "#{request.verb}" \
  62. 18 "\n#{request.canonical_path}" \
  63. 18 "\n#{request.canonical_query}" \
  64. 18 "\n#{canonical_headers}" \
  65. 18 "\n#{signed_headers}" \
  66. 18 "\n#{content_hashed}"
  67. 144 credential_scope = "#{date}" \
  68. 18 "/#{@region}" \
  69. 18 "/#{@service}" \
  70. 18 "/#{lower_provider_prefix}_request"
  71. 144 algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
  72. # string to sign
  73. 144 sts = "#{algo_line}" \
  74. 18 "\n#{datetime}" \
  75. 18 "\n#{credential_scope}" \
  76. 18 "\n#{OpenSSL::Digest.new(@algorithm).hexdigest(creq)}"
  77. # signature
  78. 144 k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
  79. 144 k_region = hmac(k_date, @region)
  80. 144 k_service = hmac(k_region, @service)
  81. 144 k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
  82. 144 sig = hexhmac(k_credentials, sts)
  83. 144 credential = "#{@credentials.username}/#{credential_scope}"
  84. # apply signature
  85. 126 request.headers["authorization"] =
  86. 18 "#{algo_line} " \
  87. 18 "Credential=#{credential}, " \
  88. 18 "SignedHeaders=#{signed_headers}, " \
  89. 18 "Signature=#{sig}"
  90. end
  91. 8 private
  92. 8 def hexdigest(value)
  93. 144 digest = OpenSSL::Digest.new(@algorithm)
  94. 144 if value.respond_to?(:read)
  95. 32 if value.respond_to?(:to_path)
  96. # files, pathnames
  97. 8 digest.file(value.to_path).hexdigest
  98. else
  99. # gzipped request bodies
  100. 24 raise Error, "request body must be rewindable" unless value.respond_to?(:rewind)
  101. 24 buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  102. 2 begin
  103. 24 IO.copy_stream(value, buffer)
  104. 24 buffer.flush
  105. 24 digest.file(buffer.to_path).hexdigest
  106. ensure
  107. 24 value.rewind
  108. 24 buffer.close
  109. 24 buffer.unlink
  110. end
  111. end
  112. else
  113. # error on endless generators
  114. 112 raise Error, "hexdigest for endless enumerators is not supported" if value.unbounded_body?
  115. 104 mb_buffer = value.each.with_object("".b) do |chunk, b|
  116. 56 b << chunk
  117. 56 break if b.bytesize >= 1024 * 1024
  118. end
  119. 104 digest.hexdigest(mb_buffer)
  120. end
  121. end
  122. 8 def hmac(key, value)
  123. 576 OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
  124. end
  125. 8 def hexhmac(key, value)
  126. 144 OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
  127. end
  128. end
  129. 8 class << self
  130. 8 def load_dependencies(*)
  131. 152 require "set"
  132. 152 require "digest/sha2"
  133. end
  134. 8 def configure(klass)
  135. 152 klass.plugin(:expect)
  136. end
  137. end
  138. # adds support for the following options:
  139. #
  140. # :sigv4_signer :: instance of HTTPX::Plugins::AWSSigV4 used to sign requests.
  141. 8 module OptionsMethods
  142. 8 def option_sigv4_signer(value)
  143. 320 value.is_a?(Signer) ? value : Signer.new(value)
  144. end
  145. end
  146. 8 module InstanceMethods
  147. 8 def aws_sigv4_authentication(**options)
  148. 152 with(sigv4_signer: Signer.new(**options))
  149. end
  150. 8 def build_request(*)
  151. 152 request = super
  152. 152 return request if request.headers.key?("authorization")
  153. 152 signer = request.options.sigv4_signer
  154. 152 return request unless signer
  155. 152 signer.sign!(request)
  156. 144 request
  157. end
  158. end
  159. 8 module RequestMethods
  160. 8 def canonical_path
  161. 144 path = uri.path.dup
  162. 144 path << "/" if path.empty?
  163. 176 path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
  164. end
  165. 8 def canonical_query
  166. 176 params = query.split("&")
  167. # params = params.map { |p| p.match(/=/) ? p : p + '=' }
  168. # From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
  169. # Sort the parameter names by character code point in ascending order.
  170. # Parameters with duplicate names should be sorted by value.
  171. #
  172. # Default sort <=> in JRuby will swap members
  173. # occasionally when <=> is 0 (considered still sorted), but this
  174. # causes our normalized query string to not match the sent querystring.
  175. # When names match, we then sort by their values. When values also
  176. # match then we sort by their original order
  177. 176 params.each.with_index.sort do |a, b|
  178. 64 a, a_offset = a
  179. 64 b, b_offset = b
  180. 64 a_name, a_value = a.split("=", 2)
  181. 64 b_name, b_value = b.split("=", 2)
  182. 64 if a_name == b_name
  183. 32 if a_value == b_value
  184. 16 a_offset <=> b_offset
  185. else
  186. 16 a_value <=> b_value
  187. end
  188. else
  189. 32 a_name <=> b_name
  190. end
  191. end.map(&:first).join("&")
  192. end
  193. end
  194. end
  195. 8 register_plugin :aws_sigv4, AWSSigV4
  196. end
  197. end

lib/httpx/plugins/basic_auth.rb

100.0% lines covered

12 relevant lines. 12 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
  8. #
  9. 8 module BasicAuth
  10. 8 class << self
  11. 8 def load_dependencies(_klass)
  12. 112 require_relative "auth/basic"
  13. end
  14. 8 def configure(klass)
  15. 112 klass.plugin(:auth)
  16. end
  17. end
  18. 8 module InstanceMethods
  19. 8 def basic_auth(user, password)
  20. 128 authorization(Authentication::Basic.new(user, password).authenticate)
  21. end
  22. end
  23. end
  24. 8 register_plugin :basic_auth, BasicAuth
  25. end
  26. end

lib/httpx/plugins/brotli.rb

100.0% lines covered

25 relevant lines. 25 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module Brotli
  5. 6 class Deflater < Transcoder::Deflater
  6. 6 def deflate(chunk)
  7. 24 return unless chunk
  8. 12 ::Brotli.deflate(chunk)
  9. end
  10. end
  11. 6 module RequestBodyClassMethods
  12. 6 def initialize_deflater_body(body, encoding)
  13. 24 return Brotli.encode(body) if encoding == "br"
  14. 12 super
  15. end
  16. end
  17. 6 module ResponseBodyClassMethods
  18. 6 def initialize_inflater_by_encoding(encoding, response, **kwargs)
  19. 24 return Brotli.decode(response, **kwargs) if encoding == "br"
  20. 12 super
  21. end
  22. end
  23. 6 module_function
  24. 6 def load_dependencies(*)
  25. 24 require "brotli"
  26. end
  27. 6 def self.extra_options(options)
  28. 24 options.merge(supported_compression_formats: %w[br] + options.supported_compression_formats)
  29. end
  30. 6 def encode(body)
  31. 12 Deflater.new(body)
  32. end
  33. 6 def decode(_response, **)
  34. 12 ::Brotli.method(:inflate)
  35. end
  36. end
  37. 6 register_plugin :brotli, Brotli
  38. end
  39. end

lib/httpx/plugins/callbacks.rb

100.0% lines covered

48 relevant lines. 48 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Plugins
  4. #
  5. # This plugin adds suppoort for callbacks around the request/response lifecycle.
  6. #
  7. # https://gitlab.com/os85/httpx/-/wikis/Events
  8. #
  9. 26 module Callbacks
  10. # connection closed user-space errors happen after errors can be surfaced to requests,
  11. # so they need to pierce through the scheduler, which is only possible by simulating an
  12. # interrupt.
  13. 26 class CallbackError < Exception; end # rubocop:disable Lint/InheritException
  14. 26 module InstanceMethods
  15. 26 include HTTPX::Callbacks
  16. 26 %i[
  17. connection_opened connection_closed
  18. request_error
  19. request_started request_body_chunk request_completed
  20. response_started response_body_chunk response_completed
  21. ].each do |meth|
  22. 234 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  23. 9 def on_#{meth}(&blk) # def on_connection_opened(&blk)
  24. 9 on(:#{meth}, &blk) # on(:connection_opened, &blk)
  25. self # self
  26. end # end
  27. MOD
  28. end
  29. 26 private
  30. 26 def do_init_connection(connection, selector)
  31. 209 super
  32. 209 connection.on(:open) do
  33. 195 next unless connection.current_session == self
  34. 195 emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
  35. end
  36. 209 connection.on(:close) do
  37. 211 next unless connection.current_session == self
  38. 211 emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
  39. end
  40. 209 connection
  41. end
  42. 26 def set_request_callbacks(request)
  43. 211 super
  44. 211 request.on(:headers) do
  45. 179 emit_or_callback_error(:request_started, request)
  46. end
  47. 211 request.on(:body_chunk) do |chunk|
  48. 16 emit_or_callback_error(:request_body_chunk, request, chunk)
  49. end
  50. 211 request.on(:done) do
  51. 163 emit_or_callback_error(:request_completed, request)
  52. end
  53. 211 request.on(:response_started) do |res|
  54. 163 if res.is_a?(Response)
  55. 147 emit_or_callback_error(:response_started, request, res)
  56. 131 res.on(:chunk_received) do |chunk|
  57. 154 emit_or_callback_error(:response_body_chunk, request, res, chunk)
  58. end
  59. else
  60. 16 emit_or_callback_error(:request_error, request, res.error)
  61. end
  62. end
  63. 211 request.on(:response) do |res|
  64. 131 emit_or_callback_error(:response_completed, request, res)
  65. end
  66. end
  67. 26 def emit_or_callback_error(*args)
  68. 1196 emit(*args)
  69. rescue StandardError => e
  70. 112 ex = CallbackError.new(e.message)
  71. 112 ex.set_backtrace(e.backtrace)
  72. 112 raise ex
  73. end
  74. 26 def receive_requests(*)
  75. 211 super
  76. rescue CallbackError => e
  77. 104 raise e.cause
  78. end
  79. 26 def close(*)
  80. 209 super
  81. rescue CallbackError => e
  82. 8 raise e.cause
  83. end
  84. end
  85. end
  86. 26 register_plugin :callbacks, Callbacks
  87. end
  88. end

lib/httpx/plugins/circuit_breaker.rb

100.0% lines covered

65 relevant lines. 65 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin implements a circuit breaker around connection errors.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Circuit-Breaker
  8. #
  9. 8 module CircuitBreaker
  10. 8 using URIExtensions
  11. 8 def self.load_dependencies(*)
  12. 56 require_relative "circuit_breaker/circuit"
  13. 56 require_relative "circuit_breaker/circuit_store"
  14. end
  15. 8 def self.extra_options(options)
  16. 56 options.merge(
  17. circuit_breaker_max_attempts: 3,
  18. circuit_breaker_reset_attempts_in: 60,
  19. circuit_breaker_break_in: 60,
  20. circuit_breaker_half_open_drip_rate: 1
  21. )
  22. end
  23. 8 module InstanceMethods
  24. 8 include HTTPX::Callbacks
  25. 8 def initialize(*)
  26. 56 super
  27. 56 @circuit_store = CircuitStore.new(@options)
  28. end
  29. 8 %i[circuit_open].each do |meth|
  30. 8 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  31. 1 def on_#{meth}(&blk) # def on_circuit_open(&blk)
  32. 1 on(:#{meth}, &blk) # on(:circuit_open, &blk)
  33. self # self
  34. end # end
  35. MOD
  36. end
  37. 8 private
  38. 8 def send_requests(*requests)
  39. # @type var short_circuit_responses: Array[response]
  40. 224 short_circuit_responses = []
  41. # run all requests through the circuit breaker, see if the circuit is
  42. # open for any of them.
  43. 224 real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
  44. 224 short_circuit_response = @circuit_store.try_respond(req)
  45. 224 if short_circuit_response.nil?
  46. 176 real_reqs << req
  47. 176 next
  48. end
  49. 42 short_circuit_responses[idx] = short_circuit_response
  50. end
  51. # run requests for the remainder
  52. 224 unless real_requests.empty?
  53. 176 responses = super(*real_requests)
  54. 176 real_requests.each_with_index do |request, idx|
  55. 154 short_circuit_responses[requests.index(request)] = responses[idx]
  56. end
  57. end
  58. 224 short_circuit_responses
  59. end
  60. 8 def on_response(request, response)
  61. 176 emit(:circuit_open, request) if try_circuit_open(request, response)
  62. 176 super
  63. end
  64. 8 def try_circuit_open(request, response)
  65. 176 if response.is_a?(ErrorResponse)
  66. 112 case response.error
  67. when RequestTimeoutError
  68. 80 @circuit_store.try_open(request.uri, response)
  69. else
  70. 48 @circuit_store.try_open(request.origin, response)
  71. end
  72. 48 elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
  73. 16 @circuit_store.try_open(request.uri, response)
  74. else
  75. 32 @circuit_store.try_close(request.uri)
  76. 12 nil
  77. end
  78. end
  79. end
  80. # adds support for the following options:
  81. #
  82. # :circuit_breaker_max_attempts :: the number of attempts the circuit allows, before it is opened (defaults to <tt>3</tt>).
  83. # :circuit_breaker_reset_attempts_in :: the time a circuit stays open at most, before it resets (defaults to <tt>60</tt>).
  84. # :circuit_breaker_break_on :: callable defining an alternative rule for a response to break
  85. # (i.e. <tt>->(res) { res.status == 429 } </tt>)
  86. # :circuit_breaker_break_in :: the time that must elapse before an open circuit can transit to the half-open state
  87. # (defaults to <tt><60</tt>).
  88. # :circuit_breaker_half_open_drip_rate :: the rate of requests a circuit allows to be performed when in an half-open state
  89. # (defaults to <tt>1</tt>).
  90. 8 module OptionsMethods
  91. 8 def option_circuit_breaker_max_attempts(value)
  92. 112 attempts = Integer(value)
  93. 112 raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
  94. 112 attempts
  95. end
  96. 8 def option_circuit_breaker_reset_attempts_in(value)
  97. 64 timeout = Float(value)
  98. 64 raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
  99. 64 timeout
  100. end
  101. 8 def option_circuit_breaker_break_in(value)
  102. 88 timeout = Float(value)
  103. 88 raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
  104. 88 timeout
  105. end
  106. 8 def option_circuit_breaker_half_open_drip_rate(value)
  107. 88 ratio = Float(value)
  108. 88 raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
  109. 88 ratio
  110. end
  111. 8 def option_circuit_breaker_break_on(value)
  112. 16 raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
  113. 16 value
  114. end
  115. end
  116. end
  117. 8 register_plugin :circuit_breaker, CircuitBreaker
  118. end
  119. end

lib/httpx/plugins/circuit_breaker/circuit.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::CircuitBreaker
  4. #
  5. # A circuit is assigned to a given absoolute url or origin.
  6. #
  7. # It sets +max_attempts+, the number of attempts the circuit allows, before it is opened.
  8. # It sets +reset_attempts_in+, the time a circuit stays open at most, before it resets.
  9. # It sets +break_in+, the time that must elapse before an open circuit can transit to the half-open state.
  10. # It sets +circuit_breaker_half_open_drip_rate+, the rate of requests a circuit allows to be performed when in an half-open state.
  11. #
  12. 8 class Circuit
  13. 8 def initialize(max_attempts, reset_attempts_in, break_in, circuit_breaker_half_open_drip_rate)
  14. 56 @max_attempts = max_attempts
  15. 56 @reset_attempts_in = reset_attempts_in
  16. 56 @break_in = break_in
  17. 56 @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
  18. 56 @attempts = 0
  19. 56 total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
  20. 56 @drip_factor = (@max_attempts / total_real_attempts).round
  21. 56 @state = :closed
  22. end
  23. 8 def respond
  24. 224 try_close
  25. 196 case @state
  26. when :closed
  27. 51 nil
  28. when :half_open
  29. 49 @attempts += 1
  30. # do real requests while drip rate valid
  31. 56 if (@real_attempts % @drip_factor).zero?
  32. 35 @real_attempts += 1
  33. 35 return
  34. end
  35. 16 @response
  36. when :open
  37. 32 @response
  38. end
  39. end
  40. 8 def try_open(response)
  41. 126 case @state
  42. when :closed
  43. 120 now = Utils.now
  44. 120 if @attempts.positive?
  45. # reset if error happened long ago
  46. 48 @attempts = 0 if now - @attempted_at > @reset_attempts_in
  47. else
  48. 72 @attempted_at = now
  49. end
  50. 105 @attempts += 1
  51. 120 return unless @attempts >= @max_attempts
  52. 64 @state = :open
  53. 64 @opened_at = now
  54. 64 @response = response
  55. when :half_open
  56. # open immediately
  57. 24 @state = :open
  58. 24 @attempted_at = @opened_at = Utils.now
  59. 24 @response = response
  60. end
  61. end
  62. 8 def try_close
  63. 224 case @state
  64. when :closed
  65. 51 nil
  66. when :half_open
  67. # do not close circuit unless attempts exhausted
  68. 48 return unless @attempts >= @max_attempts
  69. # reset!
  70. 16 @attempts = 0
  71. 16 @opened_at = @attempted_at = @response = nil
  72. 16 @state = :closed
  73. when :open
  74. 72 if Utils.elapsed_time(@opened_at) > @break_in
  75. 40 @state = :half_open
  76. 40 @attempts = 0
  77. 40 @real_attempts = 0
  78. end
  79. end
  80. end
  81. end
  82. end
  83. end

lib/httpx/plugins/circuit_breaker/circuit_store.rb

100.0% lines covered

23 relevant lines. 23 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX::Plugins::CircuitBreaker
  3. 8 using HTTPX::URIExtensions
  4. 8 class CircuitStore
  5. 8 def initialize(options)
  6. 56 @circuits = Hash.new do |h, k|
  7. 49 h[k] = Circuit.new(
  8. options.circuit_breaker_max_attempts,
  9. options.circuit_breaker_reset_attempts_in,
  10. options.circuit_breaker_break_in,
  11. options.circuit_breaker_half_open_drip_rate
  12. )
  13. end
  14. 56 @circuits_mutex = Thread::Mutex.new
  15. end
  16. 8 def try_open(uri, response)
  17. 288 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(uri) }
  18. 144 circuit.try_open(response)
  19. end
  20. 8 def try_close(uri)
  21. 32 circuit = @circuits_mutex.synchronize do
  22. 32 return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
  23. 32 get_circuit_for_uri(uri)
  24. end
  25. 32 circuit.try_close
  26. end
  27. # if circuit is open, it'll respond with the stored response.
  28. # if not, nil.
  29. 8 def try_respond(request)
  30. 448 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(request.uri) }
  31. 224 circuit.respond
  32. end
  33. 8 private
  34. 8 def get_circuit_for_uri(uri)
  35. 400 if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
  36. 288 @circuits[uri.origin]
  37. else
  38. 112 @circuits[uri.to_s]
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/content_digest.rb

100.0% lines covered

102 relevant lines. 102 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds `Content-Digest` headers to requests
  6. # and can validate these headers on responses
  7. #
  8. # https://datatracker.ietf.org/doc/html/rfc9530
  9. #
  10. 8 module ContentDigest
  11. 8 class Error < HTTPX::Error; end
  12. # Error raised on response "content-digest" header validation.
  13. 8 class ValidationError < Error
  14. 8 attr_reader :response
  15. 8 def initialize(message, response)
  16. 48 super(message)
  17. 48 @response = response
  18. end
  19. end
  20. 8 class MissingContentDigestError < ValidationError; end
  21. 8 class InvalidContentDigestError < ValidationError; end
  22. 2 SUPPORTED_ALGORITHMS = {
  23. 6 "sha-256" => OpenSSL::Digest::SHA256,
  24. "sha-512" => OpenSSL::Digest::SHA512,
  25. }.freeze
  26. 8 class << self
  27. 8 def extra_options(options)
  28. 208 options.merge(encode_content_digest: true, validate_content_digest: false, content_digest_algorithm: "sha-256")
  29. end
  30. end
  31. # add support for the following options:
  32. #
  33. # :content_digest_algorithm :: the digest algorithm to use. Currently supports `sha-256` and `sha-512`. (defaults to `sha-256`)
  34. # :encode_content_digest :: whether a <tt>Content-Digest</tt> header should be computed for the request;
  35. # can also be a callable object (i.e. <tt>->(req) { ... }</tt>, defaults to <tt>true</tt>)
  36. # :validate_content_digest :: whether a <tt>Content-Digest</tt> header in the response should be validated;
  37. # can also be a callable object (i.e. <tt>->(res) { ... }</tt>, defaults to <tt>false</tt>)
  38. 8 module OptionsMethods
  39. 8 def option_content_digest_algorithm(value)
  40. 216 raise TypeError, ":content_digest_algorithm must be one of 'sha-256', 'sha-512'" unless SUPPORTED_ALGORITHMS.key?(value)
  41. 216 value
  42. end
  43. 8 def option_encode_content_digest(value)
  44. 208 value
  45. end
  46. 8 def option_validate_content_digest(value)
  47. 144 value
  48. end
  49. end
  50. 8 module ResponseBodyMethods
  51. 8 attr_reader :content_digest_buffer
  52. 8 def initialize(response, options)
  53. 176 super
  54. 176 return unless response.headers.key?("content-digest")
  55. 128 should_validate = options.validate_content_digest
  56. 128 should_validate = should_validate.call(response) if should_validate.respond_to?(:call)
  57. 128 return unless should_validate
  58. 112 @content_digest_buffer = Response::Buffer.new(
  59. threshold_size: @options.body_threshold_size,
  60. bytesize: @length,
  61. encoding: @encoding
  62. )
  63. end
  64. 8 def write(chunk)
  65. 290 @content_digest_buffer.write(chunk) if @content_digest_buffer
  66. 290 super
  67. end
  68. 8 def close
  69. 112 if @content_digest_buffer
  70. 112 @content_digest_buffer.close
  71. 112 @content_digest_buffer = nil
  72. end
  73. 112 super
  74. end
  75. end
  76. 8 module InstanceMethods
  77. 8 def build_request(*)
  78. 224 request = super
  79. 224 return request if request.empty?
  80. 48 return request if request.headers.key?("content-digest")
  81. 48 perform_encoding = @options.encode_content_digest
  82. 48 perform_encoding = perform_encoding.call(request) if perform_encoding.respond_to?(:call)
  83. 48 return request unless perform_encoding
  84. 40 digest = base64digest(request.body)
  85. 40 request.headers.add("content-digest", "#{@options.content_digest_algorithm}=:#{digest}:")
  86. 40 request
  87. end
  88. 8 private
  89. 8 def fetch_response(request, _, _)
  90. 606 response = super
  91. 606 return response unless response.is_a?(Response)
  92. 176 perform_validation = @options.validate_content_digest
  93. 176 perform_validation = perform_validation.call(response) if perform_validation.respond_to?(:call)
  94. 176 validate_content_digest(response) if perform_validation
  95. 128 response
  96. rescue ValidationError => e
  97. 48 ErrorResponse.new(request, e)
  98. end
  99. 8 def validate_content_digest(response)
  100. 128 content_digest_header = response.headers["content-digest"]
  101. 128 raise MissingContentDigestError.new("response is missing a `content-digest` header", response) unless content_digest_header
  102. 112 digests = extract_content_digests(content_digest_header)
  103. 112 included_algorithms = SUPPORTED_ALGORITHMS.keys & digests.keys
  104. 112 raise MissingContentDigestError.new("unsupported algorithms: #{digests.keys.join(", ")}", response) if included_algorithms.empty?
  105. 112 content_buffer = response.body.content_digest_buffer
  106. 112 included_algorithms.each do |algorithm|
  107. 112 digest = SUPPORTED_ALGORITHMS.fetch(algorithm).new
  108. 112 digest_received = digests[algorithm]
  109. 14 digest_computed =
  110. 111 if content_buffer.respond_to?(:to_path)
  111. 16 content_buffer.flush
  112. 16 digest.file(content_buffer.to_path).base64digest
  113. else
  114. 96 digest.base64digest(content_buffer.to_s)
  115. end
  116. 4 raise InvalidContentDigestError.new("#{algorithm} digest does not match content",
  117. 111 response) unless digest_received == digest_computed
  118. end
  119. end
  120. 8 def extract_content_digests(header)
  121. 112 header.split(",").to_h do |entry|
  122. 128 algorithm, digest = entry.split("=", 2)
  123. 128 raise Error, "#{entry} is an invalid digest format" unless algorithm && digest
  124. 128 [algorithm, digest.byteslice(1..-2)]
  125. end
  126. end
  127. 8 def base64digest(body)
  128. 40 digest = SUPPORTED_ALGORITHMS.fetch(@options.content_digest_algorithm).new
  129. 40 if body.respond_to?(:read)
  130. 32 if body.respond_to?(:to_path)
  131. 8 digest.file(body.to_path).base64digest
  132. else
  133. 24 raise ContentDigestError, "request body must be rewindable" unless body.respond_to?(:rewind)
  134. 24 buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  135. 2 begin
  136. 24 IO.copy_stream(body, buffer)
  137. 24 buffer.flush
  138. 24 digest.file(buffer.to_path).base64digest
  139. ensure
  140. 24 body.rewind
  141. 24 buffer.close
  142. 24 buffer.unlink
  143. end
  144. end
  145. else
  146. 8 raise ContentDigestError, "base64digest for endless enumerators is not supported" if body.unbounded_body?
  147. 8 buffer = "".b
  148. 16 body.each { |chunk| buffer << chunk }
  149. 8 digest.base64digest(buffer)
  150. end
  151. end
  152. end
  153. end
  154. 8 register_plugin :content_digest, ContentDigest
  155. end
  156. end

lib/httpx/plugins/cookies.rb

100.0% lines covered

51 relevant lines. 51 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "forwardable"
  3. 8 module HTTPX
  4. 8 module Plugins
  5. #
  6. # This plugin implements a persistent cookie jar for the duration of a session.
  7. #
  8. # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Cookies
  11. #
  12. 8 module Cookies
  13. 8 def self.load_dependencies(*)
  14. 144 require "httpx/plugins/cookies/jar"
  15. 144 require "httpx/plugins/cookies/cookie"
  16. 144 require "httpx/plugins/cookies/set_cookie_parser"
  17. end
  18. 8 module InstanceMethods
  19. 8 extend Forwardable
  20. 8 def_delegator :@options, :cookies
  21. 8 def initialize(options = {}, &blk)
  22. 288 super({ cookies: Jar.new }.merge(options), &blk)
  23. end
  24. 8 def wrap
  25. 16 return super unless block_given?
  26. 16 super do |session|
  27. 16 old_cookies_jar = @options.cookies.dup
  28. 1 begin
  29. 16 yield session
  30. ensure
  31. 16 @options = @options.merge(cookies: old_cookies_jar)
  32. end
  33. end
  34. end
  35. 8 def build_request(*)
  36. 320 request = super
  37. 320 request.headers.set_cookie(request.options.cookies[request.uri])
  38. 320 request
  39. end
  40. 8 private
  41. 8 def on_response(_request, response)
  42. 320 if response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
  43. 64 log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
  44. 64 @options.cookies.parse(set_cookie)
  45. end
  46. 320 super
  47. end
  48. end
  49. 8 module HeadersMethods
  50. 8 def set_cookie(cookies)
  51. 320 return if cookies.empty?
  52. 272 header_value = cookies.sort.join("; ")
  53. 272 add("cookie", header_value)
  54. end
  55. end
  56. # adds support for the following options:
  57. #
  58. # :cookies :: cookie jar for the session (can be a Hash, an Array, an instance of HTTPX::Plugins::Cookies::CookieJar)
  59. 8 module OptionsMethods
  60. 8 def option_headers(*)
  61. 320 value = super
  62. 320 merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
  63. 320 value
  64. end
  65. 8 def option_cookies(value)
  66. 480 jar = value.is_a?(Jar) ? value : Jar.new(value)
  67. 480 merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
  68. 480 jar
  69. end
  70. 8 private
  71. 8 def merge_cookie_in_jar(cookies, jar)
  72. 16 cookies.each do |ck|
  73. 16 ck.split(/ *; */).each do |cookie|
  74. 32 name, value = cookie.split("=", 2)
  75. 32 jar.add(Cookie.new(name, value))
  76. end
  77. end
  78. end
  79. end
  80. end
  81. 8 register_plugin :cookies, Cookies
  82. end
  83. end

lib/httpx/plugins/cookies/cookie.rb

100.0% lines covered

77 relevant lines. 77 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::Cookies
  4. # The HTTP Cookie.
  5. #
  6. # Contains the single cookie info: name, value and attributes.
  7. 8 class Cookie
  8. 8 include Comparable
  9. # Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
  10. # least)
  11. 8 MAX_LENGTH = 4096
  12. 8 attr_reader :domain, :path, :name, :value, :created_at
  13. 8 def path=(path)
  14. 184 path = String(path)
  15. 184 @path = path.start_with?("/") ? path : "/"
  16. end
  17. # See #domain.
  18. 8 def domain=(domain)
  19. 40 domain = String(domain)
  20. 40 if domain.start_with?(".")
  21. 16 @for_domain = true
  22. 16 domain = domain[1..-1]
  23. end
  24. 40 return if domain.empty?
  25. 40 @domain_name = DomainName.new(domain)
  26. # RFC 6265 5.3 5.
  27. 40 @for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
  28. 40 @domain = @domain_name.hostname
  29. end
  30. # Compares the cookie with another. When there are many cookies with
  31. # the same name for a URL, the value of the smallest must be used.
  32. 8 def <=>(other)
  33. # RFC 6265 5.4
  34. # Precedence: 1. longer path 2. older creation
  35. 699 (@name <=> other.name).nonzero? ||
  36. 60 (other.path.length <=> @path.length).nonzero? ||
  37. 35 (@created_at <=> other.created_at).nonzero? || 0
  38. end
  39. 8 class << self
  40. 8 def new(cookie, *args)
  41. 504 return cookie if cookie.is_a?(self)
  42. 504 super
  43. end
  44. # Tests if +target_path+ is under +base_path+ as described in RFC
  45. # 6265 5.1.4. +base_path+ must be an absolute path.
  46. # +target_path+ may be empty, in which case it is treated as the
  47. # root path.
  48. #
  49. # e.g.
  50. #
  51. # path_match?('/admin/', '/admin/index') == true
  52. # path_match?('/admin/', '/Admin/index') == false
  53. # path_match?('/admin/', '/admin/') == true
  54. # path_match?('/admin/', '/admin') == false
  55. #
  56. # path_match?('/admin', '/admin') == true
  57. # path_match?('/admin', '/Admin') == false
  58. # path_match?('/admin', '/admins') == false
  59. # path_match?('/admin', '/admin/') == true
  60. # path_match?('/admin', '/admin/index') == true
  61. 8 def path_match?(base_path, target_path)
  62. 1352 base_path.start_with?("/") || (return false)
  63. # RFC 6265 5.1.4
  64. 1352 bsize = base_path.size
  65. 1352 tsize = target_path.size
  66. 1352 return bsize == 1 if tsize.zero? # treat empty target_path as "/"
  67. 1352 return false unless target_path.start_with?(base_path)
  68. 1344 return true if bsize == tsize || base_path.end_with?("/")
  69. 16 target_path[bsize] == "/"
  70. end
  71. end
  72. 8 def initialize(arg, *attrs)
  73. 504 @created_at = Time.now
  74. 504 if attrs.empty?
  75. 24 attr_hash = Hash.try_convert(arg)
  76. else
  77. 480 @name = arg
  78. 480 @value, attr_hash = attrs
  79. 480 attr_hash = Hash.try_convert(attr_hash)
  80. end
  81. 33 attr_hash.each do |key, val|
  82. 312 key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
  83. 273 case key
  84. when :domain, :path
  85. 201 __send__(:"#{key}=", val)
  86. else
  87. 88 instance_variable_set(:"@#{key}", val)
  88. end
  89. 503 end if attr_hash
  90. 504 @path ||= "/"
  91. 504 raise ArgumentError, "name must be specified" if @name.nil?
  92. 504 @name = @name.to_s
  93. end
  94. 8 def expires
  95. 760 @expires || (@created_at && @max_age ? @created_at + @max_age : nil)
  96. end
  97. 8 def expired?(time = Time.now)
  98. 728 return false unless expires
  99. 32 expires <= time
  100. end
  101. # Returns a string for use in the Cookie header, i.e. `name=value`
  102. # or `name="value"`.
  103. 8 def cookie_value
  104. 483 "#{@name}=#{Scanner.quote(@value.to_s)}"
  105. end
  106. 8 alias_method :to_s, :cookie_value
  107. # Tests if it is OK to send this cookie to a given `uri`. A
  108. # RuntimeError is raised if the cookie's domain is unknown.
  109. 8 def valid_for_uri?(uri)
  110. 712 uri = URI(uri)
  111. # RFC 6265 5.4
  112. 712 return false if @secure && uri.scheme != "https"
  113. 704 acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
  114. end
  115. 8 private
  116. # Tests if it is OK to accept this cookie if it is sent from a given
  117. # URI/URL, `uri`.
  118. 8 def acceptable_from_uri?(uri)
  119. 736 uri = URI(uri)
  120. 736 host = DomainName.new(uri.host)
  121. # RFC 6265 5.3
  122. 736 if host.hostname == @domain
  123. 16 true
  124. 719 elsif @for_domain # !host-only-flag
  125. 32 host.cookie_domain?(@domain_name)
  126. else
  127. 688 @domain.nil?
  128. end
  129. end
  130. 8 module Scanner
  131. 8 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  132. 8 module_function
  133. 8 def quote(s)
  134. 552 return s unless s.match(RE_BAD_CHAR)
  135. 8 "\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
  136. end
  137. end
  138. end
  139. end
  140. end

lib/httpx/plugins/cookies/jar.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::Cookies
  4. # The Cookie Jar
  5. #
  6. # It holds a bunch of cookies.
  7. 8 class Jar
  8. 8 using URIExtensions
  9. 8 include Enumerable
  10. 8 def initialize_dup(orig)
  11. 216 super
  12. 216 @cookies = orig.instance_variable_get(:@cookies).dup
  13. end
  14. 8 def initialize(cookies = nil)
  15. 536 @cookies = []
  16. 120 cookies.each do |elem|
  17. 176 cookie = case elem
  18. when Cookie
  19. 16 elem
  20. when Array
  21. 144 Cookie.new(*elem)
  22. else
  23. 16 Cookie.new(elem)
  24. end
  25. 176 @cookies << cookie
  26. 535 end if cookies
  27. end
  28. 8 def parse(set_cookie)
  29. 144 SetCookieParser.call(set_cookie) do |name, value, attrs|
  30. 208 add(Cookie.new(name, value, attrs))
  31. end
  32. end
  33. 8 def add(cookie, path = nil)
  34. 456 c = cookie.dup
  35. 456 c.path = path if path && c.path == "/"
  36. # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
  37. # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
  38. 864 @cookies.delete_if { |ck| ck.name == c.name && ck.domain == c.domain && ck.path == c.path }
  39. 456 @cookies << c
  40. end
  41. 8 def [](uri)
  42. 472 each(uri).sort
  43. end
  44. 8 def each(uri = nil, &blk)
  45. 1184 return enum_for(__method__, uri) unless blk
  46. 680 return @cookies.each(&blk) unless uri
  47. 472 uri = URI(uri)
  48. 472 now = Time.now
  49. 472 tpath = uri.path
  50. 472 @cookies.delete_if do |cookie|
  51. 728 if cookie.expired?(now)
  52. 16 true
  53. else
  54. 712 yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
  55. 712 false
  56. end
  57. end
  58. end
  59. 8 def merge(other)
  60. 200 cookies_dup = dup
  61. 200 other.each do |elem|
  62. 216 cookie = case elem
  63. when Cookie
  64. 200 elem
  65. when Array
  66. 8 Cookie.new(*elem)
  67. else
  68. 8 Cookie.new(elem)
  69. end
  70. 216 cookies_dup.add(cookie)
  71. end
  72. 200 cookies_dup
  73. end
  74. end
  75. end
  76. end

lib/httpx/plugins/cookies/set_cookie_parser.rb

100.0% lines covered

70 relevant lines. 70 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "strscan"
  3. 8 require "time"
  4. 8 module HTTPX
  5. 8 module Plugins::Cookies
  6. 8 module SetCookieParser
  7. # Whitespace.
  8. 8 RE_WSP = /[ \t]+/.freeze
  9. # A pattern that matches a cookie name or attribute name which may
  10. # be empty, capturing trailing whitespace.
  11. 8 RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
  12. 8 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  13. # A pattern that matches the comma in a (typically date) value.
  14. 8 RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
  15. 8 module_function
  16. 8 def scan_dquoted(scanner)
  17. 16 s = +""
  18. 22 until scanner.eos?
  19. 64 break if scanner.skip(/"/)
  20. 48 if scanner.skip(/\\/)
  21. 16 s << scanner.getch
  22. 31 elsif scanner.scan(/[^"\\]+/)
  23. 32 s << scanner.matched
  24. end
  25. end
  26. 16 s
  27. end
  28. 8 def scan_value(scanner, comma_as_separator = false)
  29. 440 value = +""
  30. 498 until scanner.eos?
  31. 760 if scanner.scan(/[^,;"]+/)
  32. 432 value << scanner.matched
  33. 327 elsif scanner.skip(/"/)
  34. # RFC 6265 2.2
  35. # A cookie-value may be DQUOTE'd.
  36. 16 value << scan_dquoted(scanner)
  37. 311 elsif scanner.check(/;/)
  38. 232 break
  39. 79 elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
  40. 64 break
  41. else
  42. 16 value << scanner.getch
  43. end
  44. end
  45. 440 value.rstrip!
  46. 440 value
  47. end
  48. 8 def scan_name_value(scanner, comma_as_separator = false)
  49. 440 name = scanner.scan(RE_NAME)
  50. 440 name.rstrip! if name
  51. 440 if scanner.skip(/=/)
  52. 432 value = scan_value(scanner, comma_as_separator)
  53. else
  54. 8 scan_value(scanner, comma_as_separator)
  55. 8 value = nil
  56. end
  57. 440 [name, value]
  58. end
  59. 8 def call(set_cookie)
  60. 144 scanner = StringScanner.new(set_cookie)
  61. # RFC 6265 4.1.1 & 5.2
  62. 170 until scanner.eos?
  63. 208 start = scanner.pos
  64. 208 len = nil
  65. 208 scanner.skip(RE_WSP)
  66. 208 name, value = scan_name_value(scanner, true)
  67. 208 value = nil if name.empty?
  68. 208 attrs = {}
  69. 237 until scanner.eos?
  70. 296 if scanner.skip(/,/)
  71. # The comma is used as separator for concatenating multiple
  72. # values of a header.
  73. 64 len = (scanner.pos - 1) - start
  74. 64 break
  75. 231 elsif scanner.skip(/;/)
  76. 232 scanner.skip(RE_WSP)
  77. 232 aname, avalue = scan_name_value(scanner, true)
  78. 232 next if aname.empty? || value.nil?
  79. 232 aname.downcase!
  80. 203 case aname
  81. when "expires"
  82. # RFC 6265 5.2.1
  83. 16 (avalue &&= Time.parse(avalue)) || next
  84. when "max-age"
  85. # RFC 6265 5.2.2
  86. 8 next unless /\A-?\d+\z/.match?(avalue)
  87. 8 avalue = Integer(avalue)
  88. when "domain"
  89. # RFC 6265 5.2.3
  90. # An empty value SHOULD be ignored.
  91. 24 next if avalue.nil? || avalue.empty?
  92. when "path"
  93. # RFC 6265 5.2.4
  94. # A relative path must be ignored rather than normalizing it
  95. # to "/".
  96. 176 next unless avalue.start_with?("/")
  97. when "secure", "httponly"
  98. # RFC 6265 5.2.5, 5.2.6
  99. 7 avalue = true
  100. end
  101. 203 attrs[aname] = avalue
  102. end
  103. end
  104. 208 len ||= scanner.pos - start
  105. 208 next if len > Cookie::MAX_LENGTH
  106. 208 yield(name, value, attrs) if name && !name.empty? && value
  107. end
  108. end
  109. end
  110. end
  111. end

lib/httpx/plugins/digest_auth.rb

100.0% lines covered

29 relevant lines. 29 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
  8. #
  9. 8 module DigestAuth
  10. 8 DigestError = Class.new(Error)
  11. 8 class << self
  12. 8 def extra_options(options)
  13. 160 options.merge(max_concurrent_requests: 1)
  14. end
  15. 8 def load_dependencies(*)
  16. 160 require_relative "auth/digest"
  17. end
  18. end
  19. # adds support for the following options:
  20. #
  21. # :digest :: instance of HTTPX::Plugins::Authentication::Digest, used to authenticate requests in the session.
  22. 8 module OptionsMethods
  23. 8 def option_digest(value)
  24. 320 raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)
  25. 320 value
  26. end
  27. end
  28. 8 module InstanceMethods
  29. 8 def digest_auth(user, password, hashed: false)
  30. 160 with(digest: Authentication::Digest.new(user, password, hashed: hashed))
  31. end
  32. 8 private
  33. 8 def send_requests(*requests)
  34. 192 requests.flat_map do |request|
  35. 192 digest = request.options.digest
  36. 192 next super(request) unless digest
  37. 320 probe_response = wrap { super(request).first }
  38. 160 return probe_response unless probe_response.is_a?(Response)
  39. 160 if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
  40. 144 request.transition(:idle)
  41. 126 request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
  42. 144 super(request)
  43. else
  44. 16 probe_response
  45. end
  46. end
  47. end
  48. end
  49. end
  50. 8 register_plugin :digest_auth, DigestAuth
  51. end
  52. end

lib/httpx/plugins/expect.rb

100.0% lines covered

56 relevant lines. 56 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Expect#expect
  8. #
  9. 8 module Expect
  10. 8 EXPECT_TIMEOUT = 2
  11. 8 class << self
  12. 8 def no_expect_store
  13. 180 @no_expect_store ||= []
  14. end
  15. 8 def extra_options(options)
  16. 216 options.merge(expect_timeout: EXPECT_TIMEOUT)
  17. end
  18. end
  19. # adds support for the following options:
  20. #
  21. # :expect_timeout :: time (in seconds) to wait for a 100-expect response,
  22. # before retrying without the Expect header (defaults to <tt>2</tt>).
  23. # :expect_threshold_size :: min threshold (in bytes) of the request payload to enable the 100-continue negotiation on.
  24. 8 module OptionsMethods
  25. 8 def option_expect_timeout(value)
  26. 384 seconds = Float(value)
  27. 384 raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
  28. 384 seconds
  29. end
  30. 8 def option_expect_threshold_size(value)
  31. 16 bytes = Integer(value)
  32. 16 raise TypeError, ":expect_threshold_size must be positive" unless bytes.positive?
  33. 16 bytes
  34. end
  35. end
  36. 8 module RequestMethods
  37. 8 def initialize(*)
  38. 248 super
  39. 248 return if @body.empty?
  40. 168 threshold = @options.expect_threshold_size
  41. 168 return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
  42. 152 return if Expect.no_expect_store.include?(origin)
  43. 126 @headers["expect"] = "100-continue"
  44. end
  45. 8 def response=(response)
  46. 184 if response.is_a?(Response) &&
  47. response.status == 100 &&
  48. !@headers.key?("expect") &&
  49. 4 (@state == :body || @state == :done)
  50. # if we're past this point, this means that we just received a 100-Continue response,
  51. # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
  52. #
  53. # this means that expect was deactivated for this request too soon, i.e. response took longer.
  54. #
  55. # so we have to reactivate it again.
  56. 9 @headers["expect"] = "100-continue"
  57. 10 @informational_status = 100
  58. 10 Expect.no_expect_store.delete(origin)
  59. end
  60. 184 super
  61. end
  62. end
  63. 8 module ConnectionMethods
  64. 8 def send_request_to_parser(request)
  65. 112 super
  66. 112 return unless request.headers["expect"] == "100-continue"
  67. 80 expect_timeout = request.options.expect_timeout
  68. 80 return if expect_timeout.nil? || expect_timeout.infinite?
  69. 80 set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
  70. # expect timeout expired
  71. 18 if request.state == :expect && !request.expects?
  72. 18 Expect.no_expect_store << request.origin
  73. 18 request.headers.delete("expect")
  74. 18 consume
  75. end
  76. end
  77. end
  78. end
  79. 8 module InstanceMethods
  80. 8 def fetch_response(request, selector, options)
  81. 426 response = super
  82. 426 return unless response
  83. 112 if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
  84. 16 response.close
  85. 16 request.headers.delete("expect")
  86. 16 request.transition(:idle)
  87. 16 send_request(request, selector, options)
  88. 14 return
  89. end
  90. 96 response
  91. end
  92. end
  93. end
  94. 8 register_plugin :expect, Expect
  95. end
  96. end

lib/httpx/plugins/follow_redirects.rb

100.0% lines covered

106 relevant lines. 106 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 15 module HTTPX
  3. 15 InsecureRedirectError = Class.new(Error)
  4. 15 module Plugins
  5. #
  6. # This plugin adds support for automatically following redirect (status 30X) responses.
  7. #
  8. # It has a default upper bound of followed redirects (see *MAX_REDIRECTS* and the *max_redirects* option),
  9. # after which it will return the last redirect response. It will **not** raise an exception.
  10. #
  11. # It doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
  12. #
  13. # It doesn't propagate authorization related headers to requests redirecting to different origins
  14. # (see *allow_auth_to_other_origins*) to override.
  15. #
  16. # It allows customization of when to redirect via the *redirect_on* callback option).
  17. #
  18. # https://gitlab.com/os85/httpx/wikis/Follow-Redirects
  19. #
  20. 15 module FollowRedirects
  21. 15 MAX_REDIRECTS = 3
  22. 15 REDIRECT_STATUS = (300..399).freeze
  23. 15 REQUEST_BODY_HEADERS = %w[transfer-encoding content-encoding content-type content-length content-language content-md5 trailer].freeze
  24. 15 using URIExtensions
  25. # adds support for the following options:
  26. #
  27. # :max_redirects :: max number of times a request will be redirected (defaults to <tt>3</tt>).
  28. # :follow_insecure_redirects :: whether redirects to an "http://" URI, when coming from an "https//", are allowed
  29. # (defaults to <tt>false</tt>).
  30. # :allow_auth_to_other_origins :: whether auth-related headers, such as "Authorization", are propagated on redirection
  31. # (defaults to <tt>false</tt>).
  32. # :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
  33. 15 module OptionsMethods
  34. 15 def option_max_redirects(value)
  35. 462 num = Integer(value)
  36. 462 raise TypeError, ":max_redirects must be positive" if num.negative?
  37. 462 num
  38. end
  39. 15 def option_follow_insecure_redirects(value)
  40. 24 value
  41. end
  42. 15 def option_allow_auth_to_other_origins(value)
  43. 24 value
  44. end
  45. 15 def option_redirect_on(value)
  46. 48 raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
  47. 48 value
  48. end
  49. end
  50. 15 module InstanceMethods
  51. # returns a session with the *max_redirects* option set to +n+
  52. 15 def max_redirects(n)
  53. 48 with(max_redirects: n.to_i)
  54. end
  55. 15 private
  56. 15 def fetch_response(request, selector, options)
  57. 4562854 redirect_request = request.redirect_request
  58. 4562854 response = super(redirect_request, selector, options)
  59. 4562854 return unless response
  60. 568 max_redirects = redirect_request.max_redirects
  61. 568 return response unless response.is_a?(Response)
  62. 552 return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
  63. 361 return response unless max_redirects.positive?
  64. 329 redirect_uri = __get_location_from_response(response)
  65. 329 if options.redirect_on
  66. 32 redirect_allowed = options.redirect_on.call(redirect_uri)
  67. 32 return response unless redirect_allowed
  68. end
  69. # build redirect request
  70. 313 request_body = redirect_request.body
  71. 313 redirect_method = "GET"
  72. 313 redirect_params = {}
  73. 313 if response.status == 305 && options.respond_to?(:proxy)
  74. 8 request_body.rewind
  75. # The requested resource MUST be accessed through the proxy given by
  76. # the Location field. The Location field gives the URI of the proxy.
  77. 8 redirect_options = options.merge(headers: redirect_request.headers,
  78. proxy: { uri: redirect_uri },
  79. max_redirects: max_redirects - 1)
  80. 7 redirect_params[:body] = request_body
  81. 8 redirect_uri = redirect_request.uri
  82. 8 options = redirect_options
  83. else
  84. 305 redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
  85. 305 redirect_opts = Hash[options]
  86. 269 redirect_params[:max_redirects] = max_redirects - 1
  87. 305 unless request_body.empty?
  88. 24 if response.status == 307
  89. # The method and the body of the original request are reused to perform the redirected request.
  90. 8 redirect_method = redirect_request.verb
  91. 8 request_body.rewind
  92. 7 redirect_params[:body] = request_body
  93. else
  94. # redirects are **ALWAYS** GET, so remove body-related headers
  95. 16 REQUEST_BODY_HEADERS.each do |h|
  96. 112 redirect_headers.delete(h)
  97. end
  98. 14 redirect_params[:body] = nil
  99. end
  100. end
  101. 305 options = options.class.new(redirect_opts.merge(headers: redirect_headers.to_h))
  102. end
  103. 313 redirect_uri = Utils.to_uri(redirect_uri)
  104. 313 if !options.follow_insecure_redirects &&
  105. response.uri.scheme == "https" &&
  106. redirect_uri.scheme == "http"
  107. 8 error = InsecureRedirectError.new(redirect_uri.to_s)
  108. 8 error.set_backtrace(caller)
  109. 7 return ErrorResponse.new(request, error)
  110. end
  111. 305 retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
  112. 305 request.redirect_request = retry_request
  113. 305 redirect_after = response.headers["retry-after"]
  114. 305 if redirect_after
  115. # Servers send the "Retry-After" header field to indicate how long the
  116. # user agent ought to wait before making a follow-up request.
  117. # When sent with any 3xx (Redirection) response, Retry-After indicates
  118. # the minimum time that the user agent is asked to wait before issuing
  119. # the redirected request.
  120. #
  121. 31 redirect_after = Utils.parse_retry_after(redirect_after)
  122. 31 retry_start = Utils.now
  123. 31 log { "redirecting after #{redirect_after} secs..." }
  124. 31 selector.after(redirect_after) do
  125. 31 if request.response
  126. # request has terminated abruptly meanwhile
  127. 15 retry_request.emit(:response, request.response)
  128. else
  129. 16 log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
  130. 16 send_request(retry_request, selector, options)
  131. end
  132. end
  133. else
  134. 274 send_request(retry_request, selector, options)
  135. end
  136. 115 nil
  137. end
  138. # :nodoc:
  139. 15 def redirect_request_headers(original_uri, redirect_uri, headers, options)
  140. 305 headers = headers.dup
  141. 305 return headers if options.allow_auth_to_other_origins
  142. 297 return headers unless headers.key?("authorization")
  143. 8 return headers if original_uri.origin == redirect_uri.origin
  144. 8 headers.delete("authorization")
  145. 8 headers
  146. end
  147. # :nodoc:
  148. 15 def __get_location_from_response(response)
  149. # @type var location_uri: http_uri
  150. 329 location_uri = URI(response.headers["location"])
  151. 329 location_uri = response.uri.merge(location_uri) if location_uri.relative?
  152. 329 location_uri
  153. end
  154. end
  155. 15 module RequestMethods
  156. # returns the top-most original HTTPX::Request from the redirect chain
  157. 15 attr_accessor :root_request
  158. # returns the follow-up redirect request, or itself
  159. 15 def redirect_request
  160. 4562854 @redirect_request || self
  161. end
  162. # sets the follow-up redirect request
  163. 15 def redirect_request=(req)
  164. 305 @redirect_request = req
  165. 305 req.root_request = @root_request || self
  166. 305 @response = nil
  167. end
  168. 15 def response
  169. 1843 return super unless @redirect_request && @response.nil?
  170. 73 @redirect_request.response
  171. end
  172. 15 def max_redirects
  173. 568 @options.max_redirects || MAX_REDIRECTS
  174. end
  175. end
  176. 15 module ConnectionMethods
  177. 15 private
  178. 15 def set_request_request_timeout(request)
  179. 541 return unless request.root_request.nil?
  180. 257 super
  181. end
  182. end
  183. end
  184. 15 register_plugin :follow_redirects, FollowRedirects
  185. end
  186. end

lib/httpx/plugins/grpc.rb

100.0% lines covered

133 relevant lines. 133 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 GRPCError = Class.new(Error) do
  4. 6 attr_reader :status, :details, :metadata
  5. 6 def initialize(status, details, metadata)
  6. 24 @status = status
  7. 24 @details = details
  8. 24 @metadata = metadata
  9. 24 super("GRPC error, code=#{status}, details=#{details}, metadata=#{metadata}")
  10. end
  11. end
  12. 6 module Plugins
  13. #
  14. # This plugin adds DSL to build GRPC interfaces.
  15. #
  16. # https://gitlab.com/os85/httpx/wikis/GRPC
  17. #
  18. 6 module GRPC
  19. 6 unless String.method_defined?(:underscore)
  20. 6 module StringExtensions
  21. 6 refine String do
  22. 6 def underscore
  23. 312 s = dup # Avoid mutating the argument, as it might be frozen.
  24. 312 s.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
  25. 312 s.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
  26. 312 s.tr!("-", "_")
  27. 312 s.downcase!
  28. 312 s
  29. end
  30. end
  31. end
  32. 6 using StringExtensions
  33. end
  34. 6 DEADLINE = 60
  35. 6 MARSHAL_METHOD = :encode
  36. 6 UNMARSHAL_METHOD = :decode
  37. 6 HEADERS = {
  38. "content-type" => "application/grpc",
  39. "te" => "trailers",
  40. "accept" => "application/grpc",
  41. # metadata fits here
  42. # ex "foo-bin" => base64("bar")
  43. }.freeze
  44. 6 class << self
  45. 6 def load_dependencies(*)
  46. 138 require "stringio"
  47. 138 require "httpx/plugins/grpc/grpc_encoding"
  48. 138 require "httpx/plugins/grpc/message"
  49. 138 require "httpx/plugins/grpc/call"
  50. end
  51. 6 def configure(klass)
  52. 138 klass.plugin(:persistent)
  53. 138 klass.plugin(:stream)
  54. end
  55. 6 def extra_options(options)
  56. 138 options.merge(
  57. fallback_protocol: "h2",
  58. grpc_rpcs: {}.freeze,
  59. grpc_compression: false,
  60. grpc_deadline: DEADLINE
  61. )
  62. end
  63. end
  64. 6 module OptionsMethods
  65. 6 def option_grpc_service(value)
  66. 120 String(value)
  67. end
  68. 6 def option_grpc_compression(value)
  69. 162 case value
  70. when true, false
  71. 138 value
  72. else
  73. 24 value.to_s
  74. end
  75. end
  76. 6 def option_grpc_rpcs(value)
  77. 1116 Hash[value]
  78. end
  79. 6 def option_grpc_deadline(value)
  80. 804 raise TypeError, ":grpc_deadline must be positive" unless value.positive?
  81. 804 value
  82. end
  83. 6 def option_call_credentials(value)
  84. 18 raise TypeError, ":call_credentials must respond to #call" unless value.respond_to?(:call)
  85. 18 value
  86. end
  87. end
  88. 6 module ResponseMethods
  89. 6 attr_reader :trailing_metadata
  90. 6 def merge_headers(trailers)
  91. 114 @trailing_metadata = Hash[trailers]
  92. 114 super
  93. end
  94. end
  95. 6 module RequestBodyMethods
  96. 6 def initialize(*, **)
  97. 126 super
  98. 126 if (compression = @headers["grpc-encoding"])
  99. 12 deflater_body = self.class.initialize_deflater_body(@body, compression)
  100. 12 @body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
  101. else
  102. 114 @body = Transcoder::GRPCEncoding.encode(@body, compressed: false)
  103. end
  104. end
  105. end
  106. 6 module InstanceMethods
  107. 6 def with_channel_credentials(ca_path, key = nil, cert = nil, **ssl_opts)
  108. # @type var ssl_params: ::Hash[::Symbol, untyped]
  109. 72 ssl_params = {
  110. **ssl_opts,
  111. ca_file: ca_path,
  112. }
  113. 72 if key
  114. 72 key = File.read(key) if File.file?(key)
  115. 72 ssl_params[:key] = OpenSSL::PKey.read(key)
  116. end
  117. 72 if cert
  118. 72 cert = File.read(cert) if File.file?(cert)
  119. 72 ssl_params[:cert] = OpenSSL::X509::Certificate.new(cert)
  120. end
  121. 72 with(ssl: ssl_params)
  122. end
  123. 6 def rpc(rpc_name, input, output, **opts)
  124. 312 rpc_name = rpc_name.to_s
  125. 312 raise Error, "rpc #{rpc_name} already defined" if @options.grpc_rpcs.key?(rpc_name)
  126. rpc_opts = {
  127. 312 deadline: @options.grpc_deadline,
  128. }.merge(opts)
  129. 312 local_rpc_name = rpc_name.underscore
  130. 312 session_class = Class.new(self.class) do
  131. # define rpc method with ruby style name
  132. 312 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  133. def #{local_rpc_name}(input, **opts) # def grpc_action(input, **opts)
  134. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  135. end # end
  136. OUT
  137. # define rpc method with original name
  138. 312 unless local_rpc_name == rpc_name
  139. 12 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  140. def #{rpc_name}(input, **opts) # def grpcAction(input, **opts)
  141. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  142. end # end
  143. OUT
  144. end
  145. end
  146. 312 session_class.new(@options.merge(
  147. grpc_rpcs: @options.grpc_rpcs.merge(
  148. local_rpc_name => [rpc_name, input, output, rpc_opts]
  149. ).freeze
  150. ))
  151. end
  152. 6 def build_stub(origin, service: nil, compression: false)
  153. 138 scheme = @options.ssl.empty? ? "http" : "https"
  154. 138 origin = URI.parse("#{scheme}://#{origin}")
  155. 138 session = self
  156. 138 if service && service.respond_to?(:rpc_descs)
  157. # it's a grpc generic service
  158. 60 service.rpc_descs.each do |rpc_name, rpc_desc|
  159. rpc_opts = {
  160. 300 marshal_method: rpc_desc.marshal_method,
  161. unmarshal_method: rpc_desc.unmarshal_method,
  162. }
  163. 300 input = rpc_desc.input
  164. 300 input = input.type if input.respond_to?(:type)
  165. 300 output = rpc_desc.output
  166. 300 if output.respond_to?(:type)
  167. 120 rpc_opts[:stream] = true
  168. 120 output = output.type
  169. end
  170. 300 session = session.rpc(rpc_name, input, output, **rpc_opts)
  171. end
  172. 60 service = service.service_name
  173. end
  174. 138 session.with(origin: origin, grpc_service: service, grpc_compression: compression)
  175. end
  176. 6 def execute(rpc_method, input,
  177. deadline: DEADLINE,
  178. metadata: nil,
  179. **opts)
  180. 126 grpc_request = build_grpc_request(rpc_method, input, deadline: deadline, metadata: metadata, **opts)
  181. 126 response = request(grpc_request, **opts)
  182. 126 response.raise_for_status unless opts[:stream]
  183. 114 GRPC::Call.new(response)
  184. end
  185. 6 private
  186. 6 def rpc_execute(rpc_name, input, **opts)
  187. 60 rpc_name, input_enc, output_enc, rpc_opts = @options.grpc_rpcs[rpc_name]
  188. 60 exec_opts = rpc_opts.merge(opts)
  189. 60 marshal_method ||= exec_opts.delete(:marshal_method) || MARSHAL_METHOD
  190. 60 unmarshal_method ||= exec_opts.delete(:unmarshal_method) || UNMARSHAL_METHOD
  191. 60 messages = if input.respond_to?(:each)
  192. 24 Enumerator.new do |y|
  193. 24 input.each do |message|
  194. 48 y << input_enc.__send__(marshal_method, message)
  195. end
  196. end
  197. else
  198. 36 input_enc.__send__(marshal_method, input)
  199. end
  200. 60 call = execute(rpc_name, messages, **exec_opts)
  201. 60 call.decoder = output_enc.method(unmarshal_method)
  202. 60 call
  203. end
  204. 6 def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **)
  205. 126 uri = @options.origin.dup
  206. 126 rpc_method = "/#{rpc_method}" unless rpc_method.start_with?("/")
  207. 126 rpc_method = "/#{@options.grpc_service}#{rpc_method}" if @options.grpc_service
  208. 126 uri.path = rpc_method
  209. 126 headers = HEADERS.merge(
  210. "grpc-accept-encoding" => ["identity", *@options.supported_compression_formats]
  211. )
  212. 126 unless deadline == Float::INFINITY
  213. # convert to milliseconds
  214. 126 deadline = (deadline * 1000.0).to_i
  215. 126 headers["grpc-timeout"] = "#{deadline}m"
  216. end
  217. 126 headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
  218. # prepare compressor
  219. 126 compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
  220. 126 headers["grpc-encoding"] = compression if compression
  221. 126 headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
  222. 126 build_request("POST", uri, headers: headers, body: input)
  223. end
  224. end
  225. end
  226. 6 register_plugin :grpc, GRPC
  227. end
  228. end

lib/httpx/plugins/grpc/call.rb

90.91% lines covered

33 relevant lines. 30 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module GRPC
  5. # Encapsulates call information
  6. 6 class Call
  7. 6 attr_writer :decoder
  8. 6 def initialize(response)
  9. 114 @response = response
  10. 156 @decoder = ->(z) { z }
  11. 114 @consumed = false
  12. 114 @grpc_response = nil
  13. end
  14. 6 def inspect
  15. "#GRPC::Call(#{grpc_response})"
  16. end
  17. 6 def to_s
  18. 66 grpc_response.to_s
  19. end
  20. 6 def metadata
  21. response.headers
  22. end
  23. 6 def trailing_metadata
  24. 72 return unless @consumed
  25. 48 @response.trailing_metadata
  26. end
  27. 6 private
  28. 6 def grpc_response
  29. 186 @grpc_response ||= if @response.respond_to?(:each)
  30. 24 Enumerator.new do |y|
  31. 24 Message.stream(@response).each do |message|
  32. 48 y << @decoder.call(message)
  33. end
  34. 24 @consumed = true
  35. end
  36. else
  37. 90 @consumed = true
  38. 90 @decoder.call(Message.unary(@response))
  39. end
  40. end
  41. 6 def respond_to_missing?(meth, *args, &blk)
  42. 24 grpc_response.respond_to?(meth, *args) || super
  43. end
  44. 6 def method_missing(meth, *args, &blk)
  45. 48 return grpc_response.__send__(meth, *args, &blk) if grpc_response.respond_to?(meth)
  46. super
  47. end
  48. end
  49. end
  50. end
  51. end

lib/httpx/plugins/grpc/grpc_encoding.rb

97.87% lines covered

47 relevant lines. 46 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Transcoder
  4. 6 module GRPCEncoding
  5. 6 class Deflater
  6. 6 extend Forwardable
  7. 6 attr_reader :content_type
  8. 6 def initialize(body, compressed:)
  9. 126 @content_type = body.content_type
  10. 126 @body = BodyReader.new(body)
  11. 126 @compressed = compressed
  12. end
  13. 6 def bytesize
  14. 402 return @body.bytesize if @body.respond_to?(:bytesize)
  15. Float::INFINITY
  16. end
  17. 6 def read(length = nil, outbuf = nil)
  18. 264 buf = @body.read(length, outbuf)
  19. 252 return unless buf
  20. 138 compressed_flag = @compressed ? 1 : 0
  21. 138 buf = outbuf if outbuf
  22. 138 buf = buf.b if buf.frozen?
  23. 138 buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
  24. 138 buf
  25. end
  26. end
  27. 6 class Inflater
  28. 6 def initialize(response)
  29. 90 @response = response
  30. 90 @grpc_encodings = nil
  31. end
  32. 6 def call(message, &blk)
  33. 114 data = "".b
  34. 114 until message.empty?
  35. 114 compressed, size = message.unpack("CL>")
  36. 114 encoded_data = message.byteslice(5..size + 5 - 1)
  37. 114 if compressed == 1
  38. 12 grpc_encodings.reverse_each do |encoding|
  39. 12 decoder = @response.body.class.initialize_inflater_by_encoding(encoding, @response, bytesize: encoded_data.bytesize)
  40. 12 encoded_data = decoder.call(encoded_data)
  41. 12 blk.call(encoded_data) if blk
  42. 12 data << encoded_data
  43. end
  44. else
  45. 102 blk.call(encoded_data) if blk
  46. 102 data << encoded_data
  47. end
  48. 114 message = message.byteslice((size + 5)..-1)
  49. end
  50. 114 data
  51. end
  52. 6 private
  53. 6 def grpc_encodings
  54. 12 @grpc_encodings ||= @response.headers.get("grpc-encoding")
  55. end
  56. end
  57. 6 def self.encode(*args, **kwargs)
  58. 126 Deflater.new(*args, **kwargs)
  59. end
  60. 6 def self.decode(response)
  61. 90 Inflater.new(response)
  62. end
  63. end
  64. end
  65. end

lib/httpx/plugins/grpc/message.rb

95.83% lines covered

24 relevant lines. 23 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module GRPC
  5. # Encoding module for GRPC responses
  6. #
  7. # Can encode and decode grpc messages.
  8. 6 module Message
  9. 6 module_function
  10. # decodes a unary grpc response
  11. 6 def unary(response)
  12. 90 verify_status(response)
  13. 66 decoder = Transcoder::GRPCEncoding.decode(response)
  14. 66 decoder.call(response.to_s)
  15. end
  16. # lazy decodes a grpc stream response
  17. 6 def stream(response, &block)
  18. 48 return enum_for(__method__, response) unless block
  19. 24 decoder = Transcoder::GRPCEncoding.decode(response)
  20. 24 response.each do |frame|
  21. 48 decoder.call(frame, &block)
  22. end
  23. 24 verify_status(response)
  24. end
  25. 6 def cancel(request)
  26. request.emit(:refuse, :client_cancellation)
  27. end
  28. # interprets the grpc call trailing metadata, and raises an
  29. # exception in case of error code
  30. 6 def verify_status(response)
  31. # return standard errors if need be
  32. 114 response.raise_for_status
  33. 114 status = Integer(response.headers["grpc-status"])
  34. 114 message = response.headers["grpc-message"]
  35. 114 return if status.zero?
  36. 24 response.close
  37. 24 raise GRPCError.new(status, message, response.trailing_metadata)
  38. end
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/h2c.rb

94.64% lines covered

56 relevant lines. 53 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
  6. # (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
  9. #
  10. 8 module H2C
  11. 8 VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
  12. 8 class << self
  13. 8 def load_dependencies(klass)
  14. 16 klass.plugin(:upgrade)
  15. end
  16. 8 def call(connection, request, response)
  17. 16 connection.upgrade_to_h2c(request, response)
  18. end
  19. 8 def extra_options(options)
  20. 16 options.merge(max_concurrent_requests: 1, upgrade_handlers: options.upgrade_handlers.merge("h2c" => self))
  21. end
  22. end
  23. 8 class H2CParser < Connection::HTTP2
  24. 8 def upgrade(request, response)
  25. # skip checks, it is assumed that this is the first
  26. # request in the connection
  27. 16 stream = @connection.upgrade
  28. # on_settings
  29. 16 handle_stream(stream, request)
  30. 14 @streams[request] = stream
  31. # clean up data left behind in the buffer, if the server started
  32. # sending frames
  33. 16 data = response.read
  34. 16 @connection << data
  35. end
  36. end
  37. 8 module ConnectionMethods
  38. 8 using URIExtensions
  39. 8 def initialize(*)
  40. 16 super
  41. 16 @h2c_handshake = false
  42. end
  43. 8 def send(request)
  44. 56 return super if @h2c_handshake
  45. 16 return super unless VALID_H2C_VERBS.include?(request.verb) && request.scheme == "http"
  46. 16 return super if @upgrade_protocol == "h2c"
  47. 16 @h2c_handshake = true
  48. # build upgrade request
  49. 16 request.headers.add("connection", "upgrade")
  50. 16 request.headers.add("connection", "http2-settings")
  51. 14 request.headers["upgrade"] = "h2c"
  52. 14 request.headers["http2-settings"] = ::HTTP2::Client.settings_header(request.options.http2_settings)
  53. 16 super
  54. end
  55. 8 def upgrade_to_h2c(request, response)
  56. 16 prev_parser = @parser
  57. 16 if prev_parser
  58. 16 prev_parser.reset
  59. 14 @inflight -= prev_parser.requests.size
  60. end
  61. 16 @parser = H2CParser.new(@write_buffer, @options)
  62. 16 set_parser_callbacks(@parser)
  63. 14 @inflight += 1
  64. 16 @parser.upgrade(request, response)
  65. 16 @upgrade_protocol = "h2c"
  66. 16 prev_parser.requests.each do |req|
  67. 16 req.transition(:idle)
  68. 16 send(req)
  69. end
  70. end
  71. 8 private
  72. 8 def send_request_to_parser(request)
  73. 56 super
  74. 56 return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
  75. 16 max_concurrent_requests = parser.max_concurrent_requests
  76. 16 return if max_concurrent_requests == 1
  77. parser.max_concurrent_requests = 1
  78. request.once(:response) do
  79. parser.max_concurrent_requests = max_concurrent_requests
  80. end
  81. end
  82. end
  83. end
  84. 8 register_plugin(:h2c, H2C)
  85. end
  86. end

lib/httpx/plugins/ntlm_auth.rb

100.0% lines covered

30 relevant lines. 30 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 3 module HTTPX
  3. 3 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/Auth#ntlm-auth
  6. #
  7. 3 module NTLMAuth
  8. 3 class << self
  9. 3 def load_dependencies(_klass)
  10. 2 require_relative "auth/ntlm"
  11. end
  12. 3 def extra_options(options)
  13. 2 options.merge(max_concurrent_requests: 1)
  14. end
  15. end
  16. 3 module OptionsMethods
  17. 3 def option_ntlm(value)
  18. 8 raise TypeError, ":ntlm must be a #{Authentication::Ntlm}" unless value.is_a?(Authentication::Ntlm)
  19. 8 value
  20. end
  21. end
  22. 3 module InstanceMethods
  23. 3 def ntlm_auth(user, password, domain = nil)
  24. 4 with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
  25. end
  26. 3 private
  27. 3 def send_requests(*requests)
  28. 8 requests.flat_map do |request|
  29. 8 ntlm = request.options.ntlm
  30. 8 if ntlm
  31. 4 request.headers["authorization"] = ntlm.negotiate
  32. 8 probe_response = wrap { super(request).first }
  33. 4 return probe_response unless probe_response.is_a?(Response)
  34. 4 if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
  35. 2 request.transition(:idle)
  36. 2 request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
  37. 2 super(request)
  38. else
  39. 2 probe_response
  40. end
  41. else
  42. 4 super(request)
  43. end
  44. end
  45. end
  46. end
  47. end
  48. 3 register_plugin :ntlm_auth, NTLMAuth
  49. end
  50. end

lib/httpx/plugins/oauth.rb

100.0% lines covered

87 relevant lines. 87 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/OAuth
  6. #
  7. 8 module OAuth
  8. 8 class << self
  9. 8 def load_dependencies(_klass)
  10. 144 require_relative "auth/basic"
  11. end
  12. end
  13. 8 SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
  14. 8 SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
  15. 8 class OAuthSession
  16. 8 attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope
  17. 8 def initialize(
  18. issuer:,
  19. client_id:,
  20. client_secret:,
  21. access_token: nil,
  22. refresh_token: nil,
  23. scope: nil,
  24. token_endpoint: nil,
  25. response_type: nil,
  26. grant_type: nil,
  27. token_endpoint_auth_method: nil
  28. )
  29. 128 @issuer = URI(issuer)
  30. 128 @client_id = client_id
  31. 128 @client_secret = client_secret
  32. 128 @token_endpoint = URI(token_endpoint) if token_endpoint
  33. 128 @response_type = response_type
  34. 128 @scope = case scope
  35. when String
  36. 48 scope.split
  37. when Array
  38. 32 scope
  39. end
  40. 128 @access_token = access_token
  41. 128 @refresh_token = refresh_token
  42. 128 @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
  43. 128 @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
  44. 128 unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
  45. 16 raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
  46. end
  47. 112 return if SUPPORTED_GRANT_TYPES.include?(@grant_type)
  48. 16 raise Error, "#{@grant_type} is not a supported grant type"
  49. end
  50. 8 def token_endpoint
  51. 112 @token_endpoint || "#{@issuer}/token"
  52. end
  53. 8 def token_endpoint_auth_method
  54. 160 @token_endpoint_auth_method || "client_secret_basic"
  55. end
  56. 8 def load(http)
  57. 48 return if @grant_type && @scope
  58. 16 metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
  59. 16 @token_endpoint = metadata["token_endpoint"]
  60. 16 @scope = metadata["scopes_supported"]
  61. 64 @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
  62. 16 @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
  63. 16 SUPPORTED_AUTH_METHODS.include?(am)
  64. end
  65. 6 nil
  66. end
  67. 8 def merge(other)
  68. 96 obj = dup
  69. 84 case other
  70. when OAuthSession
  71. 48 other.instance_variables.each do |ivar|
  72. 432 val = other.instance_variable_get(ivar)
  73. 432 next unless val
  74. 336 obj.instance_variable_set(ivar, val)
  75. end
  76. when Hash
  77. 48 other.each do |k, v|
  78. 96 obj.instance_variable_set(:"@#{k}", v) if obj.instance_variable_defined?(:"@#{k}")
  79. end
  80. end
  81. 96 obj
  82. end
  83. end
  84. 8 module OptionsMethods
  85. 8 def option_oauth_session(value)
  86. 266 case value
  87. when Hash
  88. 16 OAuthSession.new(**value)
  89. when OAuthSession
  90. 272 value
  91. else
  92. 16 raise TypeError, ":oauth_session must be a #{OAuthSession}"
  93. end
  94. end
  95. end
  96. 8 module InstanceMethods
  97. 8 def oauth_auth(**args)
  98. 112 with(oauth_session: OAuthSession.new(**args))
  99. end
  100. 8 def with_access_token
  101. 48 oauth_session = @options.oauth_session
  102. 48 oauth_session.load(self)
  103. 48 grant_type = oauth_session.grant_type
  104. 48 headers = {}
  105. 48 form_post = { "grant_type" => grant_type, "scope" => Array(oauth_session.scope).join(" ") }.compact
  106. # auth
  107. 42 case oauth_session.token_endpoint_auth_method
  108. when "client_secret_post"
  109. 14 form_post["client_id"] = oauth_session.client_id
  110. 14 form_post["client_secret"] = oauth_session.client_secret
  111. when "client_secret_basic"
  112. 28 headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
  113. end
  114. 42 case grant_type
  115. when "client_credentials"
  116. # do nothing
  117. when "refresh_token"
  118. 14 form_post["refresh_token"] = oauth_session.refresh_token
  119. end
  120. 48 token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
  121. 48 token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
  122. 48 token_response = request(token_request)
  123. 48 token_response.raise_for_status
  124. 48 payload = token_response.json
  125. 48 access_token = payload["access_token"]
  126. 48 refresh_token = payload["refresh_token"]
  127. 48 with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
  128. end
  129. 8 def build_request(*)
  130. 128 request = super
  131. 128 return request if request.headers.key?("authorization")
  132. 96 oauth_session = @options.oauth_session
  133. 96 return request unless oauth_session && oauth_session.access_token
  134. 56 request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
  135. 64 request
  136. end
  137. end
  138. end
  139. 8 register_plugin :oauth, OAuth
  140. end
  141. end

lib/httpx/plugins/persistent.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 17 module HTTPX
  3. 17 module Plugins
  4. # This plugin implements a session that persists connections over the duration of the process.
  5. #
  6. # This will improve connection reuse in a long-running process.
  7. #
  8. # One important caveat to note is, although this session might not close connections,
  9. # other sessions from the same process that don't have this plugin turned on might.
  10. #
  11. # This session will still be able to work with it, as if, when expecting a connection
  12. # terminated by a different session, it will just retry on a new one and keep it open.
  13. #
  14. # This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
  15. # So when you use this, make sure that you don't fall into this trap.
  16. #
  17. # https://gitlab.com/os85/httpx/wikis/Persistent
  18. #
  19. 17 module Persistent
  20. 17 def self.load_dependencies(klass)
  21. 474 max_retries = if klass.default_options.respond_to?(:max_retries)
  22. 8 [klass.default_options.max_retries, 1].max
  23. else
  24. 466 1
  25. end
  26. 474 klass.plugin(:retries, max_retries: max_retries, retry_change_requests: true)
  27. end
  28. 17 def self.extra_options(options)
  29. 474 options.merge(persistent: true)
  30. end
  31. 17 module InstanceMethods
  32. 17 private
  33. 17 def get_current_selector
  34. 456 super(&nil) || begin
  35. 448 return unless block_given?
  36. 448 default = yield
  37. 448 set_current_selector(default)
  38. 448 default
  39. end
  40. end
  41. end
  42. end
  43. 17 register_plugin :persistent, Persistent
  44. end
  45. end

lib/httpx/plugins/proxy.rb

98.01% lines covered

151 relevant lines. 148 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 class HTTPProxyError < ConnectionError; end
  4. 10 module Plugins
  5. #
  6. # This plugin adds support for proxies. It ships with support for:
  7. #
  8. # * HTTP proxies
  9. # * HTTPS proxies
  10. # * Socks4/4a proxies
  11. # * Socks5 proxies
  12. #
  13. # https://gitlab.com/os85/httpx/wikis/Proxy
  14. #
  15. 10 module Proxy
  16. 10 Error = HTTPProxyError
  17. 10 PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
  18. 10 class << self
  19. 10 def configure(klass)
  20. 337 klass.plugin(:"proxy/http")
  21. 337 klass.plugin(:"proxy/socks4")
  22. 337 klass.plugin(:"proxy/socks5")
  23. end
  24. 10 def extra_options(options)
  25. 337 options.merge(supported_proxy_protocols: [])
  26. end
  27. end
  28. 10 class Parameters
  29. 10 attr_reader :uri, :username, :password, :scheme, :no_proxy
  30. 10 def initialize(uri: nil, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
  31. 371 @no_proxy = Array(no_proxy) if no_proxy
  32. 371 @uris = Array(uri)
  33. 371 uri = @uris.first
  34. 371 @username = username
  35. 371 @password = password
  36. 371 @ns = 0
  37. 371 if uri
  38. 331 @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
  39. 331 @username ||= @uri.user
  40. 331 @password ||= @uri.password
  41. end
  42. 371 @scheme = scheme
  43. 371 return unless @uri && @username && @password
  44. 210 @authenticator = nil
  45. 210 @scheme ||= infer_default_auth_scheme(@uri)
  46. 210 return unless @scheme
  47. 162 @authenticator = load_authenticator(@scheme, @username, @password, **extra)
  48. end
  49. 10 def shift
  50. # TODO: this operation must be synchronized
  51. 105 @ns += 1
  52. 120 @uri = @uris[@ns]
  53. 120 return unless @uri
  54. 16 @uri = URI(@uri) unless @uri.is_a?(URI::Generic)
  55. 16 scheme = infer_default_auth_scheme(@uri)
  56. 16 return unless scheme != @scheme
  57. 16 @scheme = scheme
  58. 16 @username = username || @uri.user
  59. 16 @password = password || @uri.password
  60. 16 @authenticator = load_authenticator(scheme, @username, @password)
  61. end
  62. 10 def can_authenticate?(*args)
  63. 184 return false unless @authenticator
  64. 64 @authenticator.can_authenticate?(*args)
  65. end
  66. 10 def authenticate(*args)
  67. 159 return unless @authenticator
  68. 159 @authenticator.authenticate(*args)
  69. end
  70. 10 def ==(other)
  71. 384 case other
  72. when Parameters
  73. 396 @uri == other.uri &&
  74. @username == other.username &&
  75. @password == other.password &&
  76. @scheme == other.scheme
  77. when URI::Generic, String
  78. 24 proxy_uri = @uri.dup
  79. 24 proxy_uri.user = @username
  80. 24 proxy_uri.password = @password
  81. 24 other_uri = other.is_a?(URI::Generic) ? other : URI.parse(other)
  82. 24 proxy_uri == other_uri
  83. else
  84. 16 super
  85. end
  86. end
  87. 10 private
  88. 10 def infer_default_auth_scheme(uri)
  89. 185 case uri.scheme
  90. when "socks5"
  91. 48 uri.scheme
  92. when "http", "https"
  93. 101 "basic"
  94. end
  95. end
  96. 10 def load_authenticator(scheme, username, password, **extra)
  97. 178 auth_scheme = scheme.to_s.capitalize
  98. 178 require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
  99. 178 Authentication.const_get(auth_scheme).new(username, password, **extra)
  100. end
  101. end
  102. # adds support for the following options:
  103. #
  104. # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
  105. # *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
  106. 10 module OptionsMethods
  107. 10 def option_proxy(value)
  108. 672 value.is_a?(Parameters) ? value : Parameters.new(**Hash[value])
  109. end
  110. 10 def option_supported_proxy_protocols(value)
  111. 1699 raise TypeError, ":supported_proxy_protocols must be an Array" unless value.is_a?(Array)
  112. 1699 value.map(&:to_s)
  113. end
  114. end
  115. 10 module InstanceMethods
  116. 10 def find_connection(request_uri, selector, options)
  117. 417 return super unless options.respond_to?(:proxy)
  118. 417 if (next_proxy = request_uri.find_proxy)
  119. 4 return super(request_uri, selector, options.merge(proxy: Parameters.new(uri: next_proxy)))
  120. end
  121. 413 proxy = options.proxy
  122. 413 return super unless proxy
  123. 403 next_proxy = proxy.uri
  124. 403 raise Error, "Failed to connect to proxy" unless next_proxy
  125. 1 raise Error,
  126. 387 "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
  127. 379 if (no_proxy = proxy.no_proxy)
  128. 16 no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
  129. # TODO: setting proxy to nil leaks the connection object in the pool
  130. 16 return super(request_uri, selector, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
  131. next_proxy.port, no_proxy)
  132. end
  133. 371 super(request_uri, selector, options.merge(proxy: proxy))
  134. end
  135. 10 private
  136. 10 def fetch_response(request, selector, options)
  137. 1574 response = super
  138. 1574 if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
  139. 120 options.proxy.shift
  140. # return last error response if no more proxies to try
  141. 120 return response if options.proxy.uri.nil?
  142. 16 log { "failed connecting to proxy, trying next..." }
  143. 16 request.transition(:idle)
  144. 16 send_request(request, selector, options)
  145. 14 return
  146. end
  147. 1454 response
  148. end
  149. 10 def proxy_error?(_request, response, options)
  150. 169 return false unless options.proxy
  151. 168 error = response.error
  152. 147 case error
  153. when NativeResolveError
  154. 16 proxy_uri = URI(options.proxy.uri)
  155. 16 peer = error.connection.peer
  156. # failed resolving proxy domain
  157. 16 peer.host == proxy_uri.host && peer.port == proxy_uri.port
  158. when ResolveError
  159. proxy_uri = URI(options.proxy.uri)
  160. error.message.end_with?(proxy_uri.to_s)
  161. when *PROXY_ERRORS
  162. # timeout errors connecting to proxy
  163. 152 true
  164. else
  165. false
  166. end
  167. end
  168. end
  169. 10 module ConnectionMethods
  170. 10 using URIExtensions
  171. 10 def initialize(*)
  172. 384 super
  173. 384 return unless @options.proxy
  174. # redefining the connection origin as the proxy's URI,
  175. # as this will be used as the tcp peer ip.
  176. 366 @proxy_uri = URI(@options.proxy.uri)
  177. end
  178. 10 def peer
  179. 950 @proxy_uri || super
  180. end
  181. 10 def connecting?
  182. 4554 return super unless @options.proxy
  183. 4395 super || @state == :connecting || @state == :connected
  184. end
  185. 10 def call
  186. 1121 super
  187. 1121 return unless @options.proxy
  188. 988 case @state
  189. when :connecting
  190. 304 consume
  191. end
  192. end
  193. 10 def reset
  194. 384 return super unless @options.proxy
  195. 367 @state = :open
  196. 367 super
  197. # emit(:close)
  198. end
  199. 10 private
  200. 10 def initialize_type(uri, options)
  201. 384 return super unless options.proxy
  202. 366 "tcp"
  203. end
  204. 10 def connect
  205. 1043 return super unless @options.proxy
  206. 891 case @state
  207. when :idle
  208. 712 transition(:connecting)
  209. when :connected
  210. 297 transition(:open)
  211. end
  212. end
  213. 10 def handle_transition(nextstate)
  214. 2118 return super unless @options.proxy
  215. 1791 case nextstate
  216. when :closing
  217. # this is a hack so that we can use the super method
  218. # and it'll think that the current state is open
  219. 375 @state = :open if @state == :connecting
  220. end
  221. 2031 super
  222. end
  223. end
  224. end
  225. 10 register_plugin :proxy, Proxy
  226. end
  227. 10 class ProxySSL < SSL
  228. 10 def initialize(tcp, request_uri, options)
  229. 89 @io = tcp.to_io
  230. 89 super(request_uri, tcp.addresses, options)
  231. 89 @hostname = request_uri.host
  232. 89 @state = :connected
  233. end
  234. end
  235. end

lib/httpx/plugins/proxy/http.rb

100.0% lines covered

102 relevant lines. 102 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 module Plugins
  4. 10 module Proxy
  5. 10 module HTTP
  6. 10 class << self
  7. 10 def extra_options(options)
  8. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[http])
  9. end
  10. end
  11. 10 module InstanceMethods
  12. 10 def with_proxy_basic_auth(opts)
  13. 8 with(proxy: opts.merge(scheme: "basic"))
  14. end
  15. 10 def with_proxy_digest_auth(opts)
  16. 24 with(proxy: opts.merge(scheme: "digest"))
  17. end
  18. 10 def with_proxy_ntlm_auth(opts)
  19. 8 with(proxy: opts.merge(scheme: "ntlm"))
  20. end
  21. 10 def fetch_response(request, selector, options)
  22. 1574 response = super
  23. 1574 if response &&
  24. response.is_a?(Response) &&
  25. response.status == 407 &&
  26. !request.headers.key?("proxy-authorization") &&
  27. response.headers.key?("proxy-authenticate") && options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  28. 8 request.transition(:idle)
  29. 7 request.headers["proxy-authorization"] =
  30. options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  31. 8 send_request(request, selector, options)
  32. 7 return
  33. end
  34. 1566 response
  35. end
  36. end
  37. 10 module ConnectionMethods
  38. 10 def connecting?
  39. 4554 super || @state == :connecting || @state == :connected
  40. end
  41. 10 private
  42. 10 def handle_transition(nextstate)
  43. 2357 return super unless @options.proxy && @options.proxy.uri.scheme == "http"
  44. 1020 case nextstate
  45. when :connecting
  46. 304 return unless @state == :idle
  47. 304 @io.connect
  48. 304 return unless @io.connected?
  49. 152 @parser || begin
  50. 144 @parser = self.class.parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
  51. 144 parser = @parser
  52. 144 parser.extend(ProxyParser)
  53. 144 parser.on(:response, &method(:__http_on_connect))
  54. 144 parser.on(:close) do |force|
  55. 57 next unless @parser
  56. 8 if force
  57. 8 reset
  58. 8 emit(:terminate)
  59. end
  60. end
  61. 144 parser.on(:reset) do
  62. 16 if parser.empty?
  63. 8 reset
  64. else
  65. 8 transition(:closing)
  66. 8 transition(:closed)
  67. 8 parser.reset if @parser
  68. 8 transition(:idle)
  69. 8 transition(:connecting)
  70. end
  71. end
  72. 144 __http_proxy_connect(parser)
  73. end
  74. 152 return if @state == :connected
  75. when :connected
  76. 136 return unless @state == :idle || @state == :connecting
  77. 121 case @state
  78. when :connecting
  79. 49 parser = @parser
  80. 49 @parser = nil
  81. 49 parser.close
  82. when :idle
  83. 87 @parser.callbacks.clear
  84. 87 set_parser_callbacks(@parser)
  85. end
  86. end
  87. 908 super
  88. end
  89. 10 def __http_proxy_connect(parser)
  90. 144 req = @pending.first
  91. 144 if req && req.uri.scheme == "https"
  92. # if the first request after CONNECT is to an https address, it is assumed that
  93. # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
  94. # and therefore, will share the connection.
  95. #
  96. 57 connect_request = ConnectRequest.new(req.uri, @options)
  97. 51 @inflight += 1
  98. 57 parser.send(connect_request)
  99. else
  100. 87 handle_transition(:connected)
  101. end
  102. end
  103. 10 def __http_on_connect(request, response)
  104. 58 @inflight -= 1
  105. 65 if response.is_a?(Response) && response.status == 200
  106. 49 req = @pending.first
  107. 49 request_uri = req.uri
  108. 49 @io = ProxySSL.new(@io, request_uri, @options)
  109. 49 transition(:connected)
  110. 49 throw(:called)
  111. 15 elsif response.is_a?(Response) &&
  112. response.status == 407 &&
  113. !request.headers.key?("proxy-authorization") &&
  114. @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  115. 8 request.transition(:idle)
  116. 7 request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  117. 8 @parser.send(request)
  118. 7 @inflight += 1
  119. else
  120. 8 pending = @pending + @parser.pending
  121. 21 while (req = pending.shift)
  122. 8 req.emit(:response, response)
  123. end
  124. 8 reset
  125. end
  126. end
  127. end
  128. 10 module ProxyParser
  129. 10 def join_headline(request)
  130. 144 return super if request.verb == "CONNECT"
  131. 70 "#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
  132. end
  133. 10 def set_protocol_headers(request)
  134. 152 extra_headers = super
  135. 152 proxy_params = @options.proxy
  136. 152 if proxy_params.scheme == "basic"
  137. # opt for basic auth
  138. 85 extra_headers["proxy-authorization"] = proxy_params.authenticate(extra_headers)
  139. end
  140. 152 extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
  141. 152 extra_headers
  142. end
  143. end
  144. 10 class ConnectRequest < Request
  145. 10 def initialize(uri, options)
  146. 57 super("CONNECT", uri, options)
  147. 57 @headers.delete("accept")
  148. end
  149. 10 def path
  150. 65 "#{@uri.hostname}:#{@uri.port}"
  151. end
  152. end
  153. end
  154. end
  155. 10 register_plugin :"proxy/http", Proxy::HTTP
  156. end
  157. end

lib/httpx/plugins/proxy/socks4.rb

97.47% lines covered

79 relevant lines. 77 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 require "resolv"
  3. 10 require "ipaddr"
  4. 10 module HTTPX
  5. 10 class Socks4Error < HTTPProxyError; end
  6. 10 module Plugins
  7. 10 module Proxy
  8. 10 module Socks4
  9. 10 VERSION = 4
  10. 10 CONNECT = 1
  11. 10 GRANTED = 0x5A
  12. 10 PROTOCOLS = %w[socks4 socks4a].freeze
  13. 10 Error = Socks4Error
  14. 10 class << self
  15. 10 def extra_options(options)
  16. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + PROTOCOLS)
  17. end
  18. end
  19. 10 module ConnectionMethods
  20. 10 def interests
  21. 3864 if @state == :connecting
  22. return @write_buffer.empty? ? :r : :w
  23. end
  24. 3864 super
  25. end
  26. 10 private
  27. 10 def handle_transition(nextstate)
  28. 2421 return super unless @options.proxy && PROTOCOLS.include?(@options.proxy.uri.scheme)
  29. 383 case nextstate
  30. when :connecting
  31. 128 return unless @state == :idle
  32. 128 @io.connect
  33. 128 return unless @io.connected?
  34. 64 req = @pending.first
  35. 64 return unless req
  36. 64 request_uri = req.uri
  37. 64 @write_buffer << Packet.connect(@options.proxy, request_uri)
  38. 64 __socks4_proxy_connect
  39. when :connected
  40. 48 return unless @state == :connecting
  41. 48 @parser = nil
  42. end
  43. 374 log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  44. 374 super
  45. end
  46. 10 def __socks4_proxy_connect
  47. 64 @parser = SocksParser.new(@write_buffer, @options)
  48. 64 @parser.once(:packet, &method(:__socks4_on_packet))
  49. end
  50. 10 def __socks4_on_packet(packet)
  51. 64 _version, status, _port, _ip = packet.unpack("CCnN")
  52. 64 if status == GRANTED
  53. 48 req = @pending.first
  54. 48 request_uri = req.uri
  55. 48 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  56. 48 transition(:connected)
  57. 48 throw(:called)
  58. else
  59. 16 on_socks4_error("socks error: #{status}")
  60. end
  61. end
  62. 10 def on_socks4_error(message)
  63. 16 ex = Error.new(message)
  64. 16 ex.set_backtrace(caller)
  65. 16 on_error(ex)
  66. 16 throw(:called)
  67. end
  68. end
  69. 10 class SocksParser
  70. 10 include HTTPX::Callbacks
  71. 10 def initialize(buffer, options)
  72. 64 @buffer = buffer
  73. 64 @options = options
  74. end
  75. 10 def close; end
  76. 10 def consume(*); end
  77. 10 def empty?
  78. true
  79. end
  80. 10 def <<(packet)
  81. 64 emit(:packet, packet)
  82. end
  83. end
  84. 10 module Packet
  85. 10 module_function
  86. 10 def connect(parameters, uri)
  87. 64 packet = [VERSION, CONNECT, uri.port].pack("CCn")
  88. 56 case parameters.uri.scheme
  89. when "socks4"
  90. 48 socks_host = uri.host
  91. 5 begin
  92. 96 ip = IPAddr.new(socks_host)
  93. 48 packet << ip.hton
  94. rescue IPAddr::InvalidAddressError
  95. 48 socks_host = Resolv.getaddress(socks_host)
  96. 48 retry
  97. end
  98. 48 packet << [parameters.username].pack("Z*")
  99. when "socks4a"
  100. 16 packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
  101. end
  102. 64 packet
  103. end
  104. end
  105. end
  106. end
  107. 10 register_plugin :"proxy/socks4", Proxy::Socks4
  108. end
  109. end

lib/httpx/plugins/proxy/socks5.rb

99.12% lines covered

113 relevant lines. 112 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 class Socks5Error < HTTPProxyError; end
  4. 10 module Plugins
  5. 10 module Proxy
  6. 10 module Socks5
  7. 10 VERSION = 5
  8. 10 NOAUTH = 0
  9. 10 PASSWD = 2
  10. 10 NONE = 0xff
  11. 10 CONNECT = 1
  12. 10 IPV4 = 1
  13. 10 DOMAIN = 3
  14. 10 IPV6 = 4
  15. 10 SUCCESS = 0
  16. 10 Error = Socks5Error
  17. 10 class << self
  18. 10 def load_dependencies(*)
  19. 337 require_relative "../auth/socks5"
  20. end
  21. 10 def extra_options(options)
  22. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[socks5])
  23. end
  24. end
  25. 10 module ConnectionMethods
  26. 10 def call
  27. 1121 super
  28. 1121 return unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  29. 312 case @state
  30. when :connecting,
  31. :negotiating,
  32. :authenticating
  33. 153 consume
  34. end
  35. end
  36. 10 def connecting?
  37. 4554 super || @state == :authenticating || @state == :negotiating
  38. end
  39. 10 def interests
  40. 6430 if @state == :connecting || @state == :authenticating || @state == :negotiating
  41. 2348 return @write_buffer.empty? ? :r : :w
  42. end
  43. 3864 super
  44. end
  45. 10 private
  46. 10 def handle_transition(nextstate)
  47. 2709 return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  48. 908 case nextstate
  49. when :connecting
  50. 288 return unless @state == :idle
  51. 288 @io.connect
  52. 288 return unless @io.connected?
  53. 144 @write_buffer << Packet.negotiate(@options.proxy)
  54. 144 __socks5_proxy_connect
  55. when :authenticating
  56. 48 return unless @state == :connecting
  57. 48 @write_buffer << Packet.authenticate(@options.proxy)
  58. when :negotiating
  59. 192 return unless @state == :connecting || @state == :authenticating
  60. 48 req = @pending.first
  61. 48 request_uri = req.uri
  62. 48 @write_buffer << Packet.connect(request_uri)
  63. when :connected
  64. 32 return unless @state == :negotiating
  65. 32 @parser = nil
  66. end
  67. 749 log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  68. 749 super
  69. end
  70. 10 def __socks5_proxy_connect
  71. 144 @parser = SocksParser.new(@write_buffer, @options)
  72. 144 @parser.on(:packet, &method(:__socks5_on_packet))
  73. 144 transition(:negotiating)
  74. end
  75. 10 def __socks5_on_packet(packet)
  76. 210 case @state
  77. when :connecting
  78. 144 version, method = packet.unpack("CC")
  79. 144 __socks5_check_version(version)
  80. 126 case method
  81. when PASSWD
  82. 48 transition(:authenticating)
  83. 18 nil
  84. when NONE
  85. 80 __on_socks5_error("no supported authorization methods")
  86. else
  87. 16 transition(:negotiating)
  88. end
  89. when :authenticating
  90. 48 _, status = packet.unpack("CC")
  91. 48 return transition(:negotiating) if status == SUCCESS
  92. 16 __on_socks5_error("socks authentication error: #{status}")
  93. when :negotiating
  94. 48 version, reply, = packet.unpack("CC")
  95. 48 __socks5_check_version(version)
  96. 48 __on_socks5_error("socks5 negotiation error: #{reply}") unless reply == SUCCESS
  97. 32 req = @pending.first
  98. 32 request_uri = req.uri
  99. 32 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  100. 32 transition(:connected)
  101. 32 throw(:called)
  102. end
  103. end
  104. 10 def __socks5_check_version(version)
  105. 192 __on_socks5_error("invalid SOCKS version (#{version})") if version != 5
  106. end
  107. 10 def __on_socks5_error(message)
  108. 112 ex = Error.new(message)
  109. 112 ex.set_backtrace(caller)
  110. 112 on_error(ex)
  111. 112 throw(:called)
  112. end
  113. end
  114. 10 class SocksParser
  115. 10 include HTTPX::Callbacks
  116. 10 def initialize(buffer, options)
  117. 144 @buffer = buffer
  118. 144 @options = options
  119. end
  120. 10 def close; end
  121. 10 def consume(*); end
  122. 10 def empty?
  123. true
  124. end
  125. 10 def <<(packet)
  126. 240 emit(:packet, packet)
  127. end
  128. end
  129. 10 module Packet
  130. 10 module_function
  131. 10 def negotiate(parameters)
  132. 144 methods = [NOAUTH]
  133. 144 methods << PASSWD if parameters.can_authenticate?
  134. 144 methods.unshift(methods.size)
  135. 144 methods.unshift(VERSION)
  136. 144 methods.pack("C*")
  137. end
  138. 10 def authenticate(parameters)
  139. 48 parameters.authenticate
  140. end
  141. 10 def connect(uri)
  142. 48 packet = [VERSION, CONNECT, 0].pack("C*")
  143. 5 begin
  144. 48 ip = IPAddr.new(uri.host)
  145. 16 ipcode = ip.ipv6? ? IPV6 : IPV4
  146. 16 packet << [ipcode].pack("C") << ip.hton
  147. rescue IPAddr::InvalidAddressError
  148. 32 packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
  149. end
  150. 48 packet << [uri.port].pack("n")
  151. 48 packet
  152. end
  153. end
  154. end
  155. end
  156. 10 register_plugin :"proxy/socks5", Proxy::Socks5
  157. end
  158. end

lib/httpx/plugins/proxy/ssh.rb

92.31% lines covered

52 relevant lines. 48 lines covered and 4 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "httpx/plugins/proxy"
  3. 6 module HTTPX
  4. 6 module Plugins
  5. 6 module Proxy
  6. 6 module SSH
  7. 6 class << self
  8. 6 def load_dependencies(*)
  9. 12 require "net/ssh/gateway"
  10. end
  11. end
  12. 6 module OptionsMethods
  13. 6 def option_proxy(value)
  14. 24 Hash[value]
  15. end
  16. end
  17. 6 module InstanceMethods
  18. 6 def request(*args, **options)
  19. 12 raise ArgumentError, "must perform at least one request" if args.empty?
  20. 12 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  21. 12 request = requests.first or return super
  22. 12 request_options = request.options
  23. 12 return super unless request_options.proxy
  24. 12 ssh_options = request_options.proxy
  25. 12 ssh_uris = ssh_options.delete(:uri)
  26. 12 ssh_uri = URI.parse(ssh_uris.shift)
  27. 12 return super unless ssh_uri.scheme == "ssh"
  28. 12 ssh_username = ssh_options.delete(:username)
  29. 12 ssh_options[:port] ||= ssh_uri.port || 22
  30. 12 if request_options.debug
  31. ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
  32. end
  33. 12 request_uri = URI(requests.first.uri)
  34. 12 @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
  35. begin
  36. 12 @_gateway.open(request_uri.host, request_uri.port) do |local_port|
  37. 12 io = build_gateway_socket(local_port, request_uri, request_options)
  38. 12 super(*args, **options.merge(io: io))
  39. end
  40. ensure
  41. 12 @_gateway.shutdown!
  42. end
  43. end
  44. 6 private
  45. 6 def build_gateway_socket(port, request_uri, options)
  46. 12 case request_uri.scheme
  47. when "https"
  48. 6 ctx = OpenSSL::SSL::SSLContext.new
  49. 6 ctx_options = SSL::TLS_OPTIONS.merge(options.ssl)
  50. 6 ctx.set_params(ctx_options) unless ctx_options.empty?
  51. 6 sock = TCPSocket.open("localhost", port)
  52. 6 io = OpenSSL::SSL::SSLSocket.new(sock, ctx)
  53. 6 io.hostname = request_uri.host
  54. 6 io.sync_close = true
  55. 6 io.connect
  56. 6 io.post_connection_check(request_uri.host) if ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
  57. 6 io
  58. when "http"
  59. 6 TCPSocket.open("localhost", port)
  60. else
  61. raise TypeError, "unexpected scheme: #{request_uri.scheme}"
  62. end
  63. end
  64. end
  65. 6 module ConnectionMethods
  66. # should not coalesce connections here, as the IP is the IP of the proxy
  67. 6 def coalescable?(*)
  68. return super unless @options.proxy
  69. false
  70. end
  71. end
  72. end
  73. end
  74. 6 register_plugin :"proxy/ssh", Proxy::SSH
  75. end
  76. end

lib/httpx/plugins/push_promise.rb

100.0% lines covered

41 relevant lines. 41 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for HTTP/2 Push responses.
  6. #
  7. # In order to benefit from this, requests are sent one at a time, so that
  8. # no push responses are received after corresponding request has been sent.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Server-Push
  11. #
  12. 8 module PushPromise
  13. 8 def self.extra_options(options)
  14. 16 options.merge(http2_settings: { settings_enable_push: 1 },
  15. max_concurrent_requests: 1)
  16. end
  17. 8 module ResponseMethods
  18. 8 def pushed?
  19. 16 @__pushed
  20. end
  21. 8 def mark_as_pushed!
  22. 8 @__pushed = true
  23. end
  24. end
  25. 8 module InstanceMethods
  26. 8 private
  27. 8 def promise_headers
  28. 16 @promise_headers ||= {}
  29. end
  30. 8 def on_promise(parser, stream)
  31. 16 stream.on(:promise_headers) do |h|
  32. 16 __on_promise_request(parser, stream, h)
  33. end
  34. 16 stream.on(:headers) do |h|
  35. 8 __on_promise_response(parser, stream, h)
  36. end
  37. end
  38. 8 def __on_promise_request(parser, stream, h)
  39. 16 log(level: 1, color: :yellow) do
  40. skipped # :nocov:
  41. skipped h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
  42. skipped # :nocov:
  43. end
  44. 16 headers = @options.headers_class.new(h)
  45. 16 path = headers[":path"]
  46. 16 authority = headers[":authority"]
  47. 24 request = parser.pending.find { |r| r.authority == authority && r.path == path }
  48. 16 if request
  49. 8 request.merge_headers(headers)
  50. 7 promise_headers[stream] = request
  51. 8 parser.pending.delete(request)
  52. 7 parser.streams[request] = stream
  53. 8 request.transition(:done)
  54. else
  55. 8 stream.refuse
  56. end
  57. end
  58. 8 def __on_promise_response(parser, stream, h)
  59. 8 request = promise_headers.delete(stream)
  60. 8 return unless request
  61. 8 parser.__send__(:on_stream_headers, stream, request, h)
  62. 8 response = request.response
  63. 8 response.mark_as_pushed!
  64. 8 stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
  65. 8 stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
  66. end
  67. end
  68. end
  69. 8 register_plugin(:push_promise, PushPromise)
  70. end
  71. end

lib/httpx/plugins/rate_limiter.rb

100.0% lines covered

18 relevant lines. 18 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when the request:
  6. #
  7. # * is rate limited;
  8. # * when the server is unavailable (503);
  9. # * when a 3xx request comes with a "retry-after" value
  10. #
  11. # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
  12. #
  13. 8 module RateLimiter
  14. 8 class << self
  15. 8 RATE_LIMIT_CODES = [429, 503].freeze
  16. 8 def configure(klass)
  17. 64 klass.plugin(:retries,
  18. retry_change_requests: true,
  19. 7 retry_on: method(:retry_on_rate_limited_response),
  20. retry_after: method(:retry_after_rate_limit))
  21. end
  22. 8 def retry_on_rate_limited_response(response)
  23. 128 return false unless response.is_a?(Response)
  24. 128 status = response.status
  25. 128 RATE_LIMIT_CODES.include?(status)
  26. end
  27. # Servers send the "Retry-After" header field to indicate how long the
  28. # user agent ought to wait before making a follow-up request. When
  29. # sent with a 503 (Service Unavailable) response, Retry-After indicates
  30. # how long the service is expected to be unavailable to the client.
  31. # When sent with any 3xx (Redirection) response, Retry-After indicates
  32. # the minimum time that the user agent is asked to wait before issuing
  33. # the redirected request.
  34. #
  35. 8 def retry_after_rate_limit(_, response)
  36. 64 return unless response.is_a?(Response)
  37. 64 retry_after = response.headers["retry-after"]
  38. 64 return unless retry_after
  39. 32 Utils.parse_retry_after(retry_after)
  40. end
  41. end
  42. end
  43. 8 register_plugin :rate_limiter, RateLimiter
  44. end
  45. end

lib/httpx/plugins/response_cache.rb

100.0% lines covered

82 relevant lines. 82 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when certain errors happen.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Response-Cache
  8. #
  9. 8 module ResponseCache
  10. 8 CACHEABLE_VERBS = %w[GET HEAD].freeze
  11. 8 CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
  12. 8 private_constant :CACHEABLE_VERBS
  13. 8 private_constant :CACHEABLE_STATUS_CODES
  14. 8 class << self
  15. 8 def load_dependencies(*)
  16. 176 require_relative "response_cache/store"
  17. end
  18. 8 def cacheable_request?(request)
  19. 248 CACHEABLE_VERBS.include?(request.verb) &&
  20. (
  21. 248 !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
  22. )
  23. end
  24. 8 def cacheable_response?(response)
  25. 168 response.is_a?(Response) &&
  26. (
  27. 168 response.cache_control.nil? ||
  28. # TODO: !response.cache_control.include?("private") && is shared cache
  29. !response.cache_control.include?("no-store")
  30. ) &&
  31. CACHEABLE_STATUS_CODES.include?(response.status) &&
  32. # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
  33. # 410 MAY be stored by a cache and used in reply to a subsequent
  34. # request, subject to the expiration mechanism, unless a cache-control
  35. # directive prohibits caching. However, a cache that does not support
  36. # the Range and Content-Range headers MUST NOT cache 206 (Partial
  37. # Content) responses.
  38. response.status != 206 && (
  39. 133 response.headers.key?("etag") || response.headers.key?("last-modified") || response.fresh?
  40. )
  41. end
  42. 8 def cached_response?(response)
  43. 80 response.is_a?(Response) && response.status == 304
  44. end
  45. 8 def extra_options(options)
  46. 176 options.merge(response_cache_store: Store.new)
  47. end
  48. end
  49. 8 module OptionsMethods
  50. 8 def option_response_cache_store(value)
  51. 176 raise TypeError, "must be an instance of #{Store}" unless value.is_a?(Store)
  52. 176 value
  53. end
  54. end
  55. 8 module InstanceMethods
  56. 8 def clear_response_cache
  57. 16 @options.response_cache_store.clear
  58. end
  59. 8 def build_request(*)
  60. 80 request = super
  61. 80 return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request)
  62. 32 @options.response_cache_store.prepare(request)
  63. 32 request
  64. end
  65. 8 def fetch_response(request, *)
  66. 277 response = super
  67. 277 return unless response
  68. 80 if ResponseCache.cached_response?(response)
  69. 32 log { "returning cached response for #{request.uri}" }
  70. 32 cached_response = @options.response_cache_store.lookup(request)
  71. 32 response.copy_from_cached(cached_response)
  72. else
  73. 48 @options.response_cache_store.cache(request, response)
  74. end
  75. 80 response
  76. end
  77. end
  78. 8 module RequestMethods
  79. 8 def response_cache_key
  80. 512 @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}-#{@uri}")
  81. end
  82. end
  83. 8 module ResponseMethods
  84. 8 def copy_from_cached(other)
  85. # 304 responses do not have content-type, which are needed for decoding.
  86. 32 @headers = @headers.class.new(other.headers.merge(@headers))
  87. 32 @body = other.body.dup
  88. 32 @body.rewind
  89. end
  90. # A response is fresh if its age has not yet exceeded its freshness lifetime.
  91. 8 def fresh?
  92. 256 if cache_control
  93. 40 return false if cache_control.include?("no-cache")
  94. # check age: max-age
  95. 48 max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
  96. 48 max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
  97. 24 max_age = max_age[/age=(\d+)/, 1] if max_age
  98. 24 max_age = max_age.to_i if max_age
  99. 24 return max_age > age if max_age
  100. end
  101. # check age: expires
  102. 216 if @headers.key?("expires")
  103. 2 begin
  104. 24 expires = Time.httpdate(@headers["expires"])
  105. rescue ArgumentError
  106. 8 return true
  107. end
  108. 14 return (expires - Time.now).to_i.positive?
  109. end
  110. 192 true
  111. end
  112. 8 def cache_control
  113. 552 return @cache_control if defined?(@cache_control)
  114. 48 @cache_control = begin
  115. 384 return unless @headers.key?("cache-control")
  116. 40 @headers["cache-control"].split(/ *, */)
  117. end
  118. end
  119. 8 def vary
  120. 280 return @vary if defined?(@vary)
  121. 28 @vary = begin
  122. 224 return unless @headers.key?("vary")
  123. 16 @headers["vary"].split(/ *, */)
  124. end
  125. end
  126. 8 private
  127. 8 def age
  128. 24 return @headers["age"].to_i if @headers.key?("age")
  129. 24 (Time.now - date).to_i
  130. end
  131. 8 def date
  132. 24 @date ||= Time.httpdate(@headers["date"])
  133. rescue NoMethodError, ArgumentError
  134. 8 Time.now
  135. end
  136. end
  137. end
  138. 8 register_plugin :response_cache, ResponseCache
  139. end
  140. end

lib/httpx/plugins/response_cache/store.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX::Plugins
  3. 8 module ResponseCache
  4. 8 class Store
  5. 8 def initialize
  6. 248 @store = {}
  7. 248 @store_mutex = Thread::Mutex.new
  8. end
  9. 8 def clear
  10. 32 @store_mutex.synchronize { @store.clear }
  11. end
  12. 8 def lookup(request)
  13. 312 responses = _get(request)
  14. 312 return unless responses
  15. 240 responses.find(&method(:match_by_vary?).curry(2)[request])
  16. end
  17. 8 def cached?(request)
  18. 112 lookup(request)
  19. end
  20. 8 def cache(request, response)
  21. 168 return unless ResponseCache.cacheable_request?(request) && ResponseCache.cacheable_response?(response)
  22. 152 _set(request, response)
  23. end
  24. 8 def prepare(request)
  25. 80 cached_response = lookup(request)
  26. 80 return unless cached_response
  27. 56 return unless match_by_vary?(request, cached_response)
  28. 56 if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
  29. 32 request.headers.add("if-modified-since", last_modified)
  30. end
  31. 56 if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
  32. 56 request.headers.add("if-none-match", etag)
  33. end
  34. end
  35. 8 private
  36. 8 def match_by_vary?(request, response)
  37. 280 vary = response.vary
  38. 280 return true unless vary
  39. 72 original_request = response.instance_variable_get(:@request)
  40. 72 return request.headers.same_headers?(original_request.headers) if vary == %w[*]
  41. 40 vary.all? do |cache_field|
  42. 40 cache_field.downcase!
  43. 40 !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
  44. end
  45. end
  46. 8 def _get(request)
  47. 312 @store_mutex.synchronize do
  48. 312 responses = @store[request.response_cache_key]
  49. 312 return unless responses
  50. 240 responses.select! do |res|
  51. 240 !res.body.closed? && res.fresh?
  52. end
  53. 240 responses
  54. end
  55. end
  56. 8 def _set(request, response)
  57. 152 @store_mutex.synchronize do
  58. 152 responses = (@store[request.response_cache_key] ||= [])
  59. 152 responses.reject! do |res|
  60. 16 res.body.closed? || !res.fresh? || match_by_vary?(request, res)
  61. end
  62. 152 responses << response
  63. end
  64. end
  65. end
  66. end
  67. end

lib/httpx/plugins/retries.rb

97.89% lines covered

95 relevant lines. 93 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 17 module HTTPX
  3. 17 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when errors happen.
  6. #
  7. # It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
  8. # after which it will return the last response, error or not. It will **not** raise an exception.
  9. #
  10. # It does not retry which are not considered idempotent (see *retry_change_requests* to override).
  11. #
  12. # https://gitlab.com/os85/httpx/wikis/Retries
  13. #
  14. 17 module Retries
  15. 17 MAX_RETRIES = 3
  16. # TODO: pass max_retries in a configure/load block
  17. 17 IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
  18. 2 RETRYABLE_ERRORS = [
  19. 15 IOError,
  20. EOFError,
  21. Errno::ECONNRESET,
  22. Errno::ECONNABORTED,
  23. Errno::EPIPE,
  24. Errno::EINVAL,
  25. Errno::ETIMEDOUT,
  26. Parser::Error,
  27. TLSError,
  28. TimeoutError,
  29. ConnectionError,
  30. Connection::HTTP2::GoawayError,
  31. ].freeze
  32. 17 DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
  33. 17 if ENV.key?("HTTPX_NO_JITTER")
  34. 16 def self.extra_options(options)
  35. 741 options.merge(max_retries: MAX_RETRIES)
  36. end
  37. else
  38. 1 def self.extra_options(options)
  39. 1 options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
  40. end
  41. end
  42. # adds support for the following options:
  43. #
  44. # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
  45. # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
  46. # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
  47. # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
  48. # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
  49. # (i.e. <tt>->(res) { ... }</tt>).
  50. 17 module OptionsMethods
  51. 17 def option_retry_after(value)
  52. # return early if callable
  53. 208 unless value.respond_to?(:call)
  54. 96 value = Float(value)
  55. 96 raise TypeError, ":retry_after must be positive" unless value.positive?
  56. end
  57. 208 value
  58. end
  59. 17 def option_retry_jitter(value)
  60. # return early if callable
  61. 51 raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
  62. 51 value
  63. end
  64. 17 def option_max_retries(value)
  65. 2247 num = Integer(value)
  66. 2247 raise TypeError, ":max_retries must be positive" unless num >= 0
  67. 2247 num
  68. end
  69. 17 def option_retry_change_requests(v)
  70. 1203 v
  71. end
  72. 17 def option_retry_on(value)
  73. 246 raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
  74. 246 value
  75. end
  76. end
  77. 17 module InstanceMethods
  78. 17 def max_retries(n)
  79. 96 with(max_retries: n)
  80. end
  81. 17 private
  82. 17 def fetch_response(request, selector, options)
  83. 6808333 response = super
  84. 6808333 if response &&
  85. request.retries.positive? &&
  86. __repeatable_request?(request, options) &&
  87. (
  88. 97 (
  89. 367 response.is_a?(ErrorResponse) && __retryable_error?(response.error)
  90. ) ||
  91. (
  92. 272 options.retry_on && options.retry_on.call(response)
  93. )
  94. )
  95. 464 __try_partial_retry(request, response)
  96. 464 log { "failed to get response, #{request.retries} tries to go..." }
  97. 464 request.retries -= 1
  98. 464 request.transition(:idle)
  99. 464 retry_after = options.retry_after
  100. 464 retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
  101. 464 if retry_after
  102. # apply jitter
  103. 96 if (jitter = request.options.retry_jitter)
  104. 16 retry_after = jitter.call(retry_after)
  105. end
  106. 96 retry_start = Utils.now
  107. 96 log { "retrying after #{retry_after} secs..." }
  108. 96 selector.after(retry_after) do
  109. 96 if request.response
  110. # request has terminated abruptly meanwhile
  111. request.emit(:response, request.response)
  112. else
  113. 96 log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
  114. 96 send_request(request, selector, options)
  115. end
  116. end
  117. else
  118. 368 send_request(request, selector, options)
  119. end
  120. 409 return
  121. end
  122. 6807869 response
  123. end
  124. 17 def __repeatable_request?(request, options)
  125. 1006 IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
  126. end
  127. 17 def __retryable_error?(ex)
  128. 3382 RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
  129. end
  130. 17 def proxy_error?(request, response, _)
  131. 64 super && !request.retries.positive?
  132. end
  133. #
  134. # Atttempt to set the request to perform a partial range request.
  135. # This happens if the peer server accepts byte-range requests, and
  136. # the last response contains some body payload.
  137. #
  138. 17 def __try_partial_retry(request, response)
  139. 464 response = response.response if response.is_a?(ErrorResponse)
  140. 464 return unless response
  141. 212 unless response.headers.key?("accept-ranges") &&
  142. response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
  143. 16 (original_body = response.body)
  144. 196 response.body.close
  145. 173 return
  146. end
  147. 16 request.partial_response = response
  148. 16 size = original_body.bytesize
  149. 14 request.headers["range"] = "bytes=#{size}-"
  150. end
  151. end
  152. 17 module RequestMethods
  153. 17 attr_accessor :retries
  154. 17 attr_writer :partial_response
  155. 17 def initialize(*args)
  156. 764 super
  157. 764 @retries = @options.max_retries
  158. end
  159. 17 def response=(response)
  160. 1266 if @partial_response
  161. 16 if response.is_a?(Response) && response.status == 206
  162. 16 response.from_partial_response(@partial_response)
  163. else
  164. @partial_response.close
  165. end
  166. 16 @partial_response = nil
  167. end
  168. 1266 super
  169. end
  170. end
  171. 17 module ResponseMethods
  172. 17 def from_partial_response(response)
  173. 16 @status = response.status
  174. 16 @headers = response.headers
  175. 16 @body = response.body
  176. end
  177. end
  178. end
  179. 17 register_plugin :retries, Retries
  180. end
  181. end

lib/httpx/plugins/ssrf_filter.rb

100.0% lines covered

61 relevant lines. 61 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 class ServerSideRequestForgeryError < Error; end
  4. 8 module Plugins
  5. #
  6. # This plugin adds support for preventing Server-Side Request Forgery attacks.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
  9. #
  10. 8 module SsrfFilter
  11. 8 module IPAddrExtensions
  12. 8 refine IPAddr do
  13. 8 def prefixlen
  14. 128 mask_addr = @mask_addr
  15. 128 raise "Invalid mask" if mask_addr.zero?
  16. 387 mask_addr >>= 1 while (mask_addr & 0x1).zero?
  17. 128 length = 0
  18. 381 while mask_addr & 0x1 == 0x1
  19. 1771 length += 1
  20. 1771 mask_addr >>= 1
  21. end
  22. 128 length
  23. end
  24. end
  25. end
  26. 8 using IPAddrExtensions
  27. # https://en.wikipedia.org/wiki/Reserved_IP_addresses
  28. 2 IPV4_BLACKLIST = [
  29. 6 IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
  30. IPAddr.new("10.0.0.0/8"), # Private network
  31. IPAddr.new("100.64.0.0/10"), # Shared Address Space
  32. IPAddr.new("127.0.0.0/8"), # Loopback
  33. IPAddr.new("169.254.0.0/16"), # Link-local
  34. IPAddr.new("172.16.0.0/12"), # Private network
  35. IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
  36. IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
  37. IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
  38. IPAddr.new("192.168.0.0/16"), # Private network
  39. IPAddr.new("198.18.0.0/15"), # Network benchmark tests
  40. IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
  41. IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
  42. IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
  43. IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
  44. IPAddr.new("255.255.255.255"), # Broadcast
  45. ].freeze
  46. 3 IPV6_BLACKLIST = ([
  47. 6 IPAddr.new("::1/128"), # Loopback
  48. IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
  49. IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
  50. IPAddr.new("2001::/32"), # Teredo tunneling
  51. IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
  52. IPAddr.new("2001:20::/28"), # ORCHIDv2
  53. IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
  54. IPAddr.new("2002::/16"), # 6to4
  55. IPAddr.new("fc00::/7"), # Unique local address
  56. IPAddr.new("fe80::/10"), # Link-local address
  57. IPAddr.new("ff00::/8"), # Multicast
  58. ] + IPV4_BLACKLIST.flat_map do |ipaddr|
  59. 128 prefixlen = ipaddr.prefixlen
  60. 128 ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
  61. 128 ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
  62. 128 [ipv4_compatible, ipv4_mapped]
  63. end).freeze
  64. 8 class << self
  65. 8 def extra_options(options)
  66. 70 options.merge(allowed_schemes: %w[https http])
  67. end
  68. 8 def unsafe_ip_address?(ipaddr)
  69. 94 range = ipaddr.to_range
  70. 94 return true if range.first != range.last
  71. 110 return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
  72. 944 IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
  73. end
  74. end
  75. # adds support for the following options:
  76. #
  77. # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
  78. 8 module OptionsMethods
  79. 8 def option_allowed_schemes(value)
  80. 78 Array(value)
  81. end
  82. end
  83. 8 module InstanceMethods
  84. 8 def send_requests(*requests)
  85. 86 responses = requests.map do |request|
  86. 86 next if @options.allowed_schemes.include?(request.uri.scheme)
  87. 8 error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
  88. 8 error.set_backtrace(caller)
  89. 8 response = ErrorResponse.new(request, error)
  90. 8 request.emit(:response, response)
  91. 8 response
  92. end
  93. 172 allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
  94. 86 allowed_responses = super(*allowed_requests)
  95. 86 allowed_responses.each_with_index do |res, idx|
  96. 78 req = allowed_requests[idx]
  97. 68 responses[requests.index(req)] = res
  98. end
  99. 86 responses
  100. end
  101. end
  102. 8 module ConnectionMethods
  103. 8 def initialize(*)
  104. begin
  105. 78 super
  106. 8 rescue ServerSideRequestForgeryError => e
  107. # may raise when IPs are passed as options via :addresses
  108. 16 throw(:resolve_error, e)
  109. end
  110. end
  111. 8 def addresses=(addrs)
  112. 172 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  113. 78 addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
  114. 78 raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
  115. 16 super
  116. end
  117. end
  118. end
  119. 8 register_plugin :ssrf_filter, SsrfFilter
  120. end
  121. end

lib/httpx/plugins/stream.rb

100.0% lines covered

74 relevant lines. 74 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 14 module HTTPX
  3. 14 class StreamResponse
  4. 14 def initialize(request, session)
  5. 170 @request = request
  6. 170 @session = session
  7. 170 @response = nil
  8. end
  9. 14 def each(&block)
  10. 218 return enum_for(__method__) unless block
  11. 154 @request.stream = self
  12. 13 begin
  13. 154 @on_chunk = block
  14. 154 if @request.response
  15. # if we've already started collecting the payload, yield it first
  16. # before proceeding.
  17. 16 body = @request.response.body
  18. 16 body.each do |chunk|
  19. 16 on_chunk(chunk)
  20. end
  21. end
  22. 154 response.raise_for_status
  23. ensure
  24. 154 @on_chunk = nil
  25. end
  26. end
  27. 14 def each_line
  28. 108 return enum_for(__method__) unless block_given?
  29. 54 line = "".b
  30. 54 each do |chunk|
  31. 45 line << chunk
  32. 129 while (idx = line.index("\n"))
  33. 54 yield line.byteslice(0..idx - 1)
  34. 54 line = line.byteslice(idx + 1..-1)
  35. end
  36. end
  37. 22 yield line unless line.empty?
  38. end
  39. # This is a ghost method. It's to be used ONLY internally, when processing streams
  40. 14 def on_chunk(chunk)
  41. 228 raise NoMethodError unless @on_chunk
  42. 228 @on_chunk.call(chunk)
  43. end
  44. skipped # :nocov:
  45. skipped def inspect
  46. skipped "#<StreamResponse:#{object_id}>"
  47. skipped end
  48. skipped # :nocov:
  49. 14 def to_s
  50. 16 response.to_s
  51. end
  52. 14 private
  53. 14 def response
  54. 542 return @response if @response
  55. 206 @request.response || begin
  56. 170 @response = @session.request(@request)
  57. end
  58. end
  59. 14 def respond_to_missing?(meth, *args)
  60. 16 response.respond_to?(meth, *args) || super
  61. end
  62. 14 def method_missing(meth, *args, &block)
  63. 178 return super unless response.respond_to?(meth)
  64. 178 response.__send__(meth, *args, &block)
  65. end
  66. end
  67. 14 module Plugins
  68. #
  69. # This plugin adds support for stream response (text/event-stream).
  70. #
  71. # https://gitlab.com/os85/httpx/wikis/Stream
  72. #
  73. 14 module Stream
  74. 14 def self.extra_options(options)
  75. 300 options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
  76. end
  77. 14 module InstanceMethods
  78. 14 def request(*args, stream: false, **options)
  79. 474 return super(*args, **options) unless stream
  80. 186 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  81. 186 raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
  82. 170 request = requests.first
  83. 170 StreamResponse.new(request, self)
  84. end
  85. end
  86. 14 module RequestMethods
  87. 14 attr_accessor :stream
  88. end
  89. 14 module ResponseMethods
  90. 14 def stream
  91. 282 request = @request.root_request if @request.respond_to?(:root_request)
  92. 282 request ||= @request
  93. 282 request.stream
  94. end
  95. end
  96. 14 module ResponseBodyMethods
  97. 14 def initialize(*)
  98. 282 super
  99. 282 @stream = @response.stream
  100. end
  101. 14 def write(chunk)
  102. 391 return super unless @stream
  103. 242 return 0 if chunk.empty?
  104. 212 chunk = decode_chunk(chunk)
  105. 212 @stream.on_chunk(chunk.dup)
  106. 212 chunk.size
  107. end
  108. 14 private
  109. 14 def transition(*)
  110. 185 return if @stream
  111. 185 super
  112. end
  113. end
  114. end
  115. 14 register_plugin :stream, Stream
  116. end
  117. end

lib/httpx/plugins/upgrade.rb

100.0% lines covered

34 relevant lines. 34 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
  6. # Upgrade header.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Upgrade
  9. #
  10. 8 module Upgrade
  11. 8 class << self
  12. 8 def configure(klass)
  13. 32 klass.plugin(:"upgrade/h2")
  14. end
  15. 8 def extra_options(options)
  16. 32 options.merge(upgrade_handlers: {})
  17. end
  18. end
  19. 8 module OptionsMethods
  20. 8 def option_upgrade_handlers(value)
  21. 88 raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
  22. 88 value
  23. end
  24. end
  25. 8 module InstanceMethods
  26. 8 def fetch_response(request, selector, options)
  27. 282 response = super
  28. 282 if response
  29. 88 return response unless response.is_a?(Response)
  30. 88 return response unless response.headers.key?("upgrade")
  31. 32 upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
  32. 32 return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
  33. 32 protocol_handler = options.upgrade_handlers[upgrade_protocol]
  34. 32 return response unless protocol_handler
  35. 32 log { "upgrading to #{upgrade_protocol}..." }
  36. 32 connection = find_connection(request.uri, selector, options)
  37. # do not upgrade already upgraded connections
  38. 32 return if connection.upgrade_protocol == upgrade_protocol
  39. 32 protocol_handler.call(connection, request, response)
  40. # keep in the loop if the server is switching, unless
  41. # the connection has been hijacked, in which case you want
  42. # to terminante immediately
  43. 32 return if response.status == 101 && !connection.hijacked
  44. end
  45. 210 response
  46. end
  47. end
  48. 8 module ConnectionMethods
  49. 8 attr_reader :upgrade_protocol, :hijacked
  50. 8 def hijack_io
  51. 8 @hijacked = true
  52. # connection is taken away from selector and not given back to the pool.
  53. 8 @current_session.deselect_connection(self, @current_selector, true)
  54. end
  55. end
  56. end
  57. 8 register_plugin(:upgrade, Upgrade)
  58. end
  59. end

lib/httpx/plugins/upgrade/h2.rb

91.67% lines covered

24 relevant lines. 22 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for upgrading an HTTP/1.1 connection to HTTP/2
  6. # via an Upgrade: h2 response declaration
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2
  9. #
  10. 8 module H2
  11. 8 class << self
  12. 8 def extra_options(options)
  13. 32 options.merge(upgrade_handlers: options.upgrade_handlers.merge("h2" => self))
  14. end
  15. 8 def call(connection, _request, _response)
  16. 8 connection.upgrade_to_h2
  17. end
  18. end
  19. 8 module ConnectionMethods
  20. 8 using URIExtensions
  21. 8 def upgrade_to_h2
  22. 8 prev_parser = @parser
  23. 8 if prev_parser
  24. 8 prev_parser.reset
  25. 7 @inflight -= prev_parser.requests.size
  26. end
  27. 8 @parser = Connection::HTTP2.new(@write_buffer, @options)
  28. 8 set_parser_callbacks(@parser)
  29. 8 @upgrade_protocol = "h2"
  30. # what's happening here:
  31. # a deviation from the state machine is done to perform the actions when a
  32. # connection is closed, without transitioning, so the connection is kept in the pool.
  33. # the state is reset to initial, so that the socket reconnect works out of the box,
  34. # while the parser is already here.
  35. 8 purge_after_closed
  36. 8 transition(:idle)
  37. 8 prev_parser.requests.each do |req|
  38. req.transition(:idle)
  39. send(req)
  40. end
  41. end
  42. end
  43. end
  44. 8 register_plugin(:"upgrade/h2", H2)
  45. end
  46. end

lib/httpx/plugins/webdav.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin implements convenience methods for performing WEBDAV requests.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/WebDav
  8. #
  9. 8 module WebDav
  10. 8 def self.configure(klass)
  11. 96 klass.plugin(:xml)
  12. end
  13. 8 module InstanceMethods
  14. 8 def copy(src, dest)
  15. 16 request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
  16. end
  17. 8 def move(src, dest)
  18. 16 request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
  19. end
  20. 8 def lock(path, timeout: nil, &blk)
  21. 48 headers = {}
  22. 42 headers["timeout"] = if timeout && timeout.positive?
  23. 16 "Second-#{timeout}"
  24. else
  25. 32 "Infinite, Second-4100000000"
  26. end
  27. 48 xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
  28. "<D:lockinfo xmlns:D=\"DAV:\">" \
  29. "<D:lockscope><D:exclusive/></D:lockscope>" \
  30. "<D:locktype><D:write/></D:locktype>" \
  31. "<D:owner>null</D:owner>" \
  32. "</D:lockinfo>"
  33. 48 response = request("LOCK", path, headers: headers, xml: xml)
  34. 48 return response unless response.is_a?(Response)
  35. 48 return response unless blk && response.status == 200
  36. 16 lock_token = response.headers["lock-token"]
  37. 1 begin
  38. 16 blk.call(response)
  39. ensure
  40. 16 unlock(path, lock_token)
  41. end
  42. 16 response
  43. end
  44. 8 def unlock(path, lock_token)
  45. 32 request("UNLOCK", path, headers: { "lock-token" => lock_token })
  46. end
  47. 8 def mkcol(dir)
  48. 16 request("MKCOL", dir)
  49. end
  50. 8 def propfind(path, xml = nil)
  51. 64 body = case xml
  52. when :acl
  53. 16 '<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
  54. "<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
  55. when nil
  56. 32 '<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
  57. else
  58. 16 xml
  59. end
  60. 64 request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
  61. end
  62. 8 def proppatch(path, xml)
  63. 6 body = "<?xml version=\"1.0\"?>" \
  64. 12 "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
  65. 16 request("PROPPATCH", path, xml: body)
  66. end
  67. # %i[ orderpatch acl report search]
  68. end
  69. end
  70. 8 register_plugin(:webdav, WebDav)
  71. end
  72. end

lib/httpx/plugins/xml.rb

100.0% lines covered

34 relevant lines. 34 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin supports request XML encoding/response decoding using the nokogiri gem.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/XML
  8. #
  9. 8 module XML
  10. 8 MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
  11. 8 module Transcoder
  12. 8 module_function
  13. 8 class Encoder
  14. 8 def initialize(xml)
  15. 160 @raw = xml
  16. end
  17. 8 def content_type
  18. 160 charset = @raw.respond_to?(:encoding) && @raw.encoding ? @raw.encoding.to_s.downcase : "utf-8"
  19. 160 "application/xml; charset=#{charset}"
  20. end
  21. 8 def bytesize
  22. 512 @raw.to_s.bytesize
  23. end
  24. 8 def to_s
  25. 160 @raw.to_s
  26. end
  27. end
  28. 8 def encode(xml)
  29. 160 Encoder.new(xml)
  30. end
  31. 8 def decode(response)
  32. 24 content_type = response.content_type.mime_type
  33. 24 raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
  34. 24 Nokogiri::XML.method(:parse)
  35. end
  36. end
  37. 8 class << self
  38. 8 def load_dependencies(*)
  39. 144 require "nokogiri"
  40. end
  41. end
  42. 8 module ResponseMethods
  43. # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
  44. # "application/xml" (requires the "nokogiri" gem).
  45. 8 def xml
  46. 16 decode(Transcoder)
  47. end
  48. end
  49. 8 module RequestBodyClassMethods
  50. # ..., xml: Nokogiri::XML::Node #=> xml encoder
  51. 8 def initialize_body(params)
  52. 592 if (xml = params.delete(:xml))
  53. # @type var xml: Nokogiri::XML::Node | String
  54. 140 return Transcoder.encode(xml)
  55. end
  56. 432 super
  57. end
  58. end
  59. end
  60. 8 register_plugin(:xml, XML)
  61. end
  62. end

lib/httpx/pmatch_extensions.rb

100.0% lines covered

17 relevant lines. 17 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module ResponsePatternMatchExtensions
  4. 26 def deconstruct
  5. 41 [@status, @headers, @body]
  6. end
  7. 26 def deconstruct_keys(_keys)
  8. 70 { status: @status, headers: @headers, body: @body }
  9. end
  10. end
  11. 26 module ErrorResponsePatternMatchExtensions
  12. 26 def deconstruct
  13. 11 [@error]
  14. end
  15. 26 def deconstruct_keys(_keys)
  16. 35 { error: @error }
  17. end
  18. end
  19. 26 module HeadersPatternMatchExtensions
  20. 26 def deconstruct
  21. 7 to_a
  22. end
  23. end
  24. 26 Headers.include HeadersPatternMatchExtensions
  25. 26 Response.include ResponsePatternMatchExtensions
  26. 26 ErrorResponse.include ErrorResponsePatternMatchExtensions
  27. end

lib/httpx/pool.rb

100.0% lines covered

68 relevant lines. 68 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "httpx/selector"
  3. 26 require "httpx/connection"
  4. 26 require "httpx/resolver"
  5. 26 module HTTPX
  6. 26 class Pool
  7. 26 using ArrayExtensions::FilterMap
  8. 26 using URIExtensions
  9. 26 POOL_TIMEOUT = 5
  10. # Sets up the connection pool with the given +options+, which can be the following:
  11. #
  12. # :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
  13. # :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
  14. #
  15. 26 def initialize(options)
  16. 10759 @max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
  17. 10759 @pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
  18. 16679 @resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
  19. 10759 @resolver_mtx = Thread::Mutex.new
  20. 10759 @connections = []
  21. 10759 @connection_mtx = Thread::Mutex.new
  22. 10759 @origin_counters = Hash.new(0)
  23. 15976 @origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
  24. end
  25. 26 def pop_connection
  26. 11250 @connection_mtx.synchronize do
  27. 11250 conn = @connections.shift
  28. 11250 @origin_conds.delete(conn.origin) if conn && (@origin_counters[conn.origin.to_s] -= 1).zero?
  29. 11250 conn
  30. end
  31. end
  32. # opens a connection to the IP reachable through +uri+.
  33. # Many hostnames are reachable through the same IP, so we try to
  34. # maximize pipelining by opening as few connections as possible.
  35. #
  36. 26 def checkout_connection(uri, options)
  37. 7785 return checkout_new_connection(uri, options) if options.io
  38. 7719 @connection_mtx.synchronize do
  39. 7719 acquire_connection(uri, options) || begin
  40. 7114 if @origin_counters[uri.origin] == @max_connections_per_origin
  41. 16 @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
  42. 16 return acquire_connection(uri, options) || raise(PoolTimeoutError.new(uri.origin, @pool_timeout))
  43. end
  44. 7098 @origin_counters[uri.origin] += 1
  45. 7098 checkout_new_connection(uri, options)
  46. end
  47. end
  48. end
  49. 26 def checkin_connection(connection)
  50. 7673 return if connection.options.io
  51. 7607 @connection_mtx.synchronize do
  52. 7607 @connections << connection
  53. 7607 @origin_conds[connection.origin.to_s].signal
  54. end
  55. end
  56. 26 def checkout_mergeable_connection(connection)
  57. 7075 return if connection.options.io
  58. 7075 @connection_mtx.synchronize do
  59. 7075 idx = @connections.find_index do |ch|
  60. 235 ch != connection && ch.mergeable?(connection)
  61. end
  62. 7075 @connections.delete_at(idx) if idx
  63. end
  64. end
  65. 26 def reset_resolvers
  66. 13924 @resolver_mtx.synchronize { @resolvers.clear }
  67. end
  68. 26 def checkout_resolver(options)
  69. 6914 resolver_type = options.resolver_class
  70. 6914 resolver_type = Resolver.resolver_for(resolver_type)
  71. 6914 @resolver_mtx.synchronize do
  72. 6914 resolvers = @resolvers[resolver_type]
  73. 6914 idx = resolvers.find_index do |res|
  74. 38 res.options == options
  75. end
  76. 6914 resolvers.delete_at(idx) if idx
  77. end || checkout_new_resolver(resolver_type, options)
  78. end
  79. 26 def checkin_resolver(resolver)
  80. 347 @resolver_mtx.synchronize do
  81. 347 resolvers = @resolvers[resolver.class]
  82. 347 resolver = resolver.multi
  83. 347 resolvers << resolver unless resolvers.include?(resolver)
  84. end
  85. end
  86. 26 private
  87. 26 def acquire_connection(uri, options)
  88. 7735 idx = @connections.find_index do |connection|
  89. 825 connection.match?(uri, options)
  90. end
  91. 7735 @connections.delete_at(idx) if idx
  92. end
  93. 26 def checkout_new_connection(uri, options)
  94. 7164 options.connection_class.new(uri, options)
  95. end
  96. 26 def checkout_new_resolver(resolver_type, options)
  97. 6884 if resolver_type.multi?
  98. 6859 Resolver::Multi.new(resolver_type, options)
  99. else
  100. 25 resolver_type.new(options)
  101. end
  102. end
  103. end
  104. end

lib/httpx/punycode.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Punycode
  4. 26 module_function
  5. begin
  6. 26 require "idnx"
  7. 25 def encode_hostname(hostname)
  8. 32 Idnx.to_punycode(hostname)
  9. end
  10. rescue LoadError
  11. 1 def encode_hostname(hostname)
  12. 1 warn "#{hostname} cannot be converted to punycode. Install the " \
  13. "\"idnx\" gem: https://github.com/HoneyryderChuck/idnx"
  14. 1 hostname
  15. end
  16. end
  17. end
  18. end

lib/httpx/request.rb

100.0% lines covered

124 relevant lines. 124 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "delegate"
  3. 26 require "forwardable"
  4. 26 module HTTPX
  5. # Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
  6. # as well as maintaining the state machine which manages streaming the request onto the wire.
  7. 26 class Request
  8. 26 extend Forwardable
  9. 26 include Callbacks
  10. 26 using URIExtensions
  11. 26 ALLOWED_URI_SCHEMES = %w[https http].freeze
  12. # default value used for "user-agent" header, when not overridden.
  13. 26 USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
  14. # the upcased string HTTP verb for this request.
  15. 26 attr_reader :verb
  16. # the absolute URI object for this request.
  17. 26 attr_reader :uri
  18. # an HTTPX::Headers object containing the request HTTP headers.
  19. 26 attr_reader :headers
  20. # an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
  21. 26 attr_reader :body
  22. # a symbol describing which frame is currently being flushed.
  23. 26 attr_reader :state
  24. # an HTTPX::Options object containing request options.
  25. 26 attr_reader :options
  26. # the corresponding HTTPX::Response object, when there is one.
  27. 26 attr_reader :response
  28. # Exception raised during enumerable body writes.
  29. 26 attr_reader :drain_error
  30. # The IP address from the peer server.
  31. 26 attr_accessor :peer_address
  32. 26 attr_writer :persistent
  33. 26 attr_reader :active_timeouts
  34. # will be +true+ when request body has been completely flushed.
  35. 26 def_delegator :@body, :empty?
  36. # closes the body
  37. 26 def_delegator :@body, :close
  38. # initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
  39. # an absolute or relative +uri+ (either as String or URI::HTTP object), the
  40. # request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
  41. #
  42. # Besides any of the options documented in HTTPX::Options (which would override or merge with what
  43. # +options+ sets), it accepts also the following:
  44. #
  45. # :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
  46. # :body :: to be encoded in the request body payload. can be a String, an IO object (i.e. a File), or an Enumerable.
  47. # :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
  48. # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
  49. # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
  50. #
  51. # :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
  52. 26 def initialize(verb, uri, options, params = EMPTY_HASH)
  53. 9659 @verb = verb.to_s.upcase
  54. 9659 @uri = Utils.to_uri(uri)
  55. 9658 @headers = options.headers.dup
  56. 9658 merge_headers(params.delete(:headers)) if params.key?(:headers)
  57. 9658 @headers["user-agent"] ||= USER_AGENT
  58. 9658 @headers["accept"] ||= "*/*"
  59. # forego compression in the Range request case
  60. 9658 if @headers.key?("range")
  61. 8 @headers.delete("accept-encoding")
  62. else
  63. 9650 @headers["accept-encoding"] ||= options.supported_compression_formats
  64. end
  65. 9658 @query_params = params.delete(:params) if params.key?(:params)
  66. 9658 @body = options.request_body_class.new(@headers, options, **params)
  67. 9650 @options = @body.options
  68. 9650 if @uri.relative? || @uri.host.nil?
  69. 600 origin = @options.origin
  70. 600 raise(Error, "invalid URI: #{@uri}") unless origin
  71. 576 base_path = @options.base_path
  72. 576 @uri = origin.merge("#{base_path}#{@uri}")
  73. end
  74. 9626 raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
  75. 9614 @state = :idle
  76. 9614 @response = nil
  77. 9614 @peer_address = nil
  78. 9614 @persistent = @options.persistent
  79. 9614 @active_timeouts = []
  80. end
  81. # the read timeout defined for this requet.
  82. 26 def read_timeout
  83. 18834 @options.timeout[:read_timeout]
  84. end
  85. # the write timeout defined for this requet.
  86. 26 def write_timeout
  87. 18834 @options.timeout[:write_timeout]
  88. end
  89. # the request timeout defined for this requet.
  90. 26 def request_timeout
  91. 18550 @options.timeout[:request_timeout]
  92. end
  93. 26 def persistent?
  94. 4756 @persistent
  95. end
  96. # if the request contains trailer headers
  97. 26 def trailers?
  98. 3070 defined?(@trailers)
  99. end
  100. # returns an instance of HTTPX::Headers containing the trailer headers
  101. 26 def trailers
  102. 88 @trailers ||= @options.headers_class.new
  103. end
  104. # returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
  105. 26 def interests
  106. 24610 return :r if @state == :done || @state == :expect
  107. 2926 :w
  108. end
  109. # merges +h+ into the instance of HTTPX::Headers of the request.
  110. 26 def merge_headers(h)
  111. 875 @headers = @headers.merge(h)
  112. end
  113. # the URI scheme of the request +uri+.
  114. 26 def scheme
  115. 3531 @uri.scheme
  116. end
  117. # sets the +response+ on this request.
  118. 26 def response=(response)
  119. 9077 return unless response
  120. 9077 if response.is_a?(Response) && response.status < 200
  121. # deal with informational responses
  122. 160 if response.status == 100 && @headers.key?("expect")
  123. 136 @informational_status = response.status
  124. 136 return
  125. end
  126. # 103 Early Hints advertises resources in document to browsers.
  127. # not very relevant for an HTTP client, discard.
  128. 24 return if response.status >= 103
  129. end
  130. 8941 @response = response
  131. 8941 emit(:response_started, response)
  132. end
  133. # returnns the URI path of the request +uri+.
  134. 26 def path
  135. 8437 path = uri.path.dup
  136. 8437 path = +"" if path.nil?
  137. 8437 path << "/" if path.empty?
  138. 8437 path << "?#{query}" unless query.empty?
  139. 8437 path
  140. end
  141. # returs the URI authority of the request.
  142. #
  143. # session.build_request("GET", "https://google.com/query").authority #=> "google.com"
  144. # session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
  145. 26 def authority
  146. 8527 @uri.authority
  147. end
  148. # returs the URI origin of the request.
  149. #
  150. # session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
  151. # session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
  152. 26 def origin
  153. 3815 @uri.origin
  154. end
  155. # returs the URI query string of the request (when available).
  156. #
  157. # session.build_request("GET", "https://search.com").query #=> ""
  158. # session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
  159. # session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
  160. # session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
  161. 26 def query
  162. 9336 return @query if defined?(@query)
  163. 7882 query = []
  164. 7882 if (q = @query_params)
  165. 152 query << Transcoder::Form.encode(q)
  166. end
  167. 7882 query << @uri.query if @uri.query
  168. 7882 @query = query.join("&")
  169. end
  170. # consumes and returns the next available chunk of request body that can be sent
  171. 26 def drain_body
  172. 9052 return nil if @body.nil?
  173. 9052 @drainer ||= @body.each
  174. 9052 chunk = @drainer.next.dup
  175. 6003 emit(:body_chunk, chunk)
  176. 6003 chunk
  177. rescue StopIteration
  178. 3037 nil
  179. rescue StandardError => e
  180. 12 @drain_error = e
  181. 12 nil
  182. end
  183. skipped # :nocov:
  184. skipped def inspect
  185. skipped "#<HTTPX::Request:#{object_id} " \
  186. skipped "#{@verb} " \
  187. skipped "#{uri} " \
  188. skipped "@headers=#{@headers} " \
  189. skipped "@body=#{@body}>"
  190. skipped end
  191. skipped # :nocov:
  192. # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
  193. 26 def transition(nextstate)
  194. 35803 case nextstate
  195. when :idle
  196. 711 @body.rewind
  197. 711 @response = nil
  198. 711 @drainer = nil
  199. 711 @active_timeouts.clear
  200. when :headers
  201. 10906 return unless @state == :idle
  202. when :body
  203. 10890 return unless @state == :headers ||
  204. @state == :expect
  205. 8810 if @headers.key?("expect")
  206. 515 if @informational_status && @informational_status == 100
  207. # check for 100 Continue response, and deallocate the var
  208. # if @informational_status == 100
  209. # @response = nil
  210. # end
  211. else
  212. 389 return if @state == :expect # do not re-set it
  213. 144 nextstate = :expect
  214. end
  215. end
  216. when :trailers
  217. 8796 return unless @state == :body
  218. when :done
  219. 8804 return if @state == :expect
  220. end
  221. 34525 @state = nextstate
  222. 34525 emit(@state, self)
  223. 13879 nil
  224. end
  225. # whether the request supports the 100-continue handshake and already processed the 100 response.
  226. 26 def expects?
  227. 7973 @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
  228. end
  229. 26 def set_timeout_callback(event, &callback)
  230. 93602 clb = once(event, &callback)
  231. # reset timeout callbacks when requests get rerouted to a different connection
  232. 93602 once(:idle) do
  233. 3320 callbacks(event).delete(clb)
  234. end
  235. end
  236. end
  237. end
  238. 26 require_relative "request/body"

lib/httpx/request/body.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
  4. 26 class Request::Body < SimpleDelegator
  5. 26 class << self
  6. 26 def new(_, options, body: nil, **params)
  7. 9666 if body.is_a?(self)
  8. # request derives its options from body
  9. 16 body.options = options.merge(params)
  10. 14 return body
  11. end
  12. 9650 super
  13. end
  14. end
  15. 26 attr_accessor :options
  16. # inits the instance with the request +headers+, +options+ and +params+, which contain the payload definition.
  17. # it wraps the given body with the appropriate encoder on initialization.
  18. #
  19. # ..., json: { foo: "bar" }) #=> json encoder
  20. # ..., form: { foo: "bar" }) #=> form urlencoded encoder
  21. # ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
  22. # ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
  23. # ..., form: { body: "bla") }) #=> raw data encoder
  24. 26 def initialize(h, options, **params)
  25. 9650 @headers = h
  26. 9650 @body = self.class.initialize_body(params)
  27. 9650 @options = options.merge(params)
  28. 9650 if @body
  29. 3156 if @options.compress_request_body && @headers.key?("content-encoding")
  30. 96 @headers.get("content-encoding").each do |encoding|
  31. 96 @body = self.class.initialize_deflater_body(@body, encoding)
  32. end
  33. end
  34. 3156 @headers["content-type"] ||= @body.content_type
  35. 3156 @headers["content-length"] = @body.bytesize unless unbounded_body?
  36. end
  37. 9642 super(@body)
  38. end
  39. # consumes and yields the request payload in chunks.
  40. 26 def each(&block)
  41. 6362 return enum_for(__method__) unless block
  42. 3185 return if @body.nil?
  43. 3113 body = stream(@body)
  44. 3113 if body.respond_to?(:read)
  45. 5410 while (chunk = body.read(16_384))
  46. 3363 block.call(chunk)
  47. end
  48. # TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
  49. # ::IO.copy_stream(body, ProcIO.new(block))
  50. 1735 elsif body.respond_to?(:each)
  51. 406 body.each(&block)
  52. else
  53. 1330 block[body.to_s]
  54. end
  55. end
  56. 26 def close
  57. 419 @body.close if @body.respond_to?(:close)
  58. end
  59. # if the +@body+ is rewindable, it rewinnds it.
  60. 26 def rewind
  61. 775 return if empty?
  62. 168 @body.rewind if @body.respond_to?(:rewind)
  63. end
  64. # return +true+ if the +body+ has been fully drained (or does nnot exist).
  65. 26 def empty?
  66. 18899 return true if @body.nil?
  67. 8621 return false if chunked?
  68. 8525 @body.bytesize.zero?
  69. end
  70. # returns the +@body+ payload size in bytes.
  71. 26 def bytesize
  72. 3328 return 0 if @body.nil?
  73. 128 @body.bytesize
  74. end
  75. # sets the body to yield using chunked trannsfer encoding format.
  76. 26 def stream(body)
  77. 3113 return body unless chunked?
  78. 96 Transcoder::Chunker.encode(body.enum_for(:each))
  79. end
  80. # returns whether the body yields infinitely.
  81. 26 def unbounded_body?
  82. 3680 return @unbounded_body if defined?(@unbounded_body)
  83. 3228 @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
  84. end
  85. # returns whether the chunked transfer encoding header is set.
  86. 26 def chunked?
  87. 19861 @headers["transfer-encoding"] == "chunked"
  88. end
  89. # sets the chunked transfer encoding header.
  90. 26 def chunk!
  91. 32 @headers.add("transfer-encoding", "chunked")
  92. end
  93. skipped # :nocov:
  94. skipped def inspect
  95. skipped "#<HTTPX::Request::Body:#{object_id} " \
  96. skipped "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
  97. skipped end
  98. skipped # :nocov:
  99. 26 class << self
  100. 26 def initialize_body(params)
  101. 9490 if (body = params.delete(:body))
  102. # @type var body: bodyIO
  103. 1408 Transcoder::Body.encode(body)
  104. 8082 elsif (form = params.delete(:form))
  105. # @type var form: Transcoder::urlencoded_input
  106. 1505 Transcoder::Form.encode(form)
  107. 6577 elsif (json = params.delete(:json))
  108. # @type var body: _ToJson
  109. 83 Transcoder::JSON.encode(json)
  110. end
  111. end
  112. # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
  113. 26 def initialize_deflater_body(body, encoding)
  114. 87 case encoding
  115. when "gzip"
  116. 48 Transcoder::GZIP.encode(body)
  117. when "deflate"
  118. 24 Transcoder::Deflate.encode(body)
  119. when "identity"
  120. 16 body
  121. else
  122. 8 body
  123. end
  124. end
  125. end
  126. end
  127. end

lib/httpx/resolver.rb

100.0% lines covered

82 relevant lines. 82 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "resolv"
  3. 26 require "ipaddr"
  4. 26 module HTTPX
  5. 26 module Resolver
  6. 26 RESOLVE_TIMEOUT = [2, 3].freeze
  7. 26 require "httpx/resolver/resolver"
  8. 26 require "httpx/resolver/system"
  9. 26 require "httpx/resolver/native"
  10. 26 require "httpx/resolver/https"
  11. 26 require "httpx/resolver/multi"
  12. 26 @lookup_mutex = Thread::Mutex.new
  13. 199 @lookups = Hash.new { |h, k| h[k] = [] }
  14. 26 @identifier_mutex = Thread::Mutex.new
  15. 26 @identifier = 1
  16. 26 @system_resolver = Resolv::Hosts.new
  17. 26 module_function
  18. 26 def resolver_for(resolver_type)
  19. 6175 case resolver_type
  20. 6777 when :native then Native
  21. 33 when :system then System
  22. 74 when :https then HTTPS
  23. else
  24. 76 return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
  25. 8 raise Error, "unsupported resolver type (#{resolver_type})"
  26. end
  27. end
  28. 26 def nolookup_resolve(hostname)
  29. 6759 ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
  30. end
  31. 26 def ip_resolve(hostname)
  32. 6759 [IPAddr.new(hostname)]
  33. rescue ArgumentError
  34. end
  35. 26 def system_resolve(hostname)
  36. 552 ips = @system_resolver.getaddresses(hostname)
  37. 552 return if ips.empty?
  38. 717 ips.map { |ip| IPAddr.new(ip) }
  39. rescue IOError
  40. end
  41. 26 def cached_lookup(hostname)
  42. 6230 now = Utils.now
  43. 6230 lookup_synchronize do |lookups|
  44. 6230 lookup(hostname, lookups, now)
  45. end
  46. end
  47. 26 def cached_lookup_set(hostname, family, entries)
  48. 231 now = Utils.now
  49. 231 entries.each do |entry|
  50. 294 entry["TTL"] += now
  51. end
  52. 231 lookup_synchronize do |lookups|
  53. 203 case family
  54. when Socket::AF_INET6
  55. 40 lookups[hostname].concat(entries)
  56. when Socket::AF_INET
  57. 191 lookups[hostname].unshift(*entries)
  58. end
  59. 231 entries.each do |entry|
  60. 294 next unless entry["name"] != hostname
  61. 30 case family
  62. when Socket::AF_INET6
  63. 8 lookups[entry["name"]] << entry
  64. when Socket::AF_INET
  65. 24 lookups[entry["name"]].unshift(entry)
  66. end
  67. end
  68. end
  69. end
  70. # do not use directly!
  71. 26 def lookup(hostname, lookups, ttl)
  72. 6238 return unless lookups.key?(hostname)
  73. 5681 entries = lookups[hostname] = lookups[hostname].select do |address|
  74. 12565 address["TTL"] > ttl
  75. end
  76. 5681 ips = entries.flat_map do |address|
  77. 12551 if address.key?("alias")
  78. 8 lookup(address["alias"], lookups, ttl)
  79. else
  80. 12543 IPAddr.new(address["data"])
  81. end
  82. end.compact
  83. 5681 ips unless ips.empty?
  84. end
  85. 26 def generate_id
  86. 1586 id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
  87. end
  88. 26 def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
  89. 741 Resolv::DNS::Message.new(message_id).tap do |query|
  90. 793 query.rd = 1
  91. 793 query.add_question(hostname, type)
  92. 103 end.encode
  93. end
  94. 26 def decode_dns_answer(payload)
  95. 51 begin
  96. 709 message = Resolv::DNS::Message.decode(payload)
  97. rescue Resolv::DNS::DecodeError => e
  98. 6 return :decode_error, e
  99. end
  100. # no domain was found
  101. 703 return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
  102. 259 return :message_truncated if message.tc == 1
  103. 247 return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
  104. 241 addresses = []
  105. 241 message.each_answer do |question, _, value|
  106. 997 case value
  107. when Resolv::DNS::Resource::IN::CNAME
  108. 18 addresses << {
  109. "name" => question.to_s,
  110. "TTL" => value.ttl,
  111. "alias" => value.name.to_s,
  112. }
  113. when Resolv::DNS::Resource::IN::A,
  114. Resolv::DNS::Resource::IN::AAAA
  115. 1006 addresses << {
  116. 21 "name" => question.to_s,
  117. "TTL" => value.ttl,
  118. "data" => value.address.to_s,
  119. }
  120. end
  121. end
  122. 241 [:ok, addresses]
  123. end
  124. 26 def lookup_synchronize
  125. 12922 @lookup_mutex.synchronize { yield(@lookups) }
  126. end
  127. 26 def id_synchronize(&block)
  128. 793 @identifier_mutex.synchronize(&block)
  129. end
  130. end
  131. end

lib/httpx/resolver/https.rb

86.11% lines covered

144 relevant lines. 124 lines covered and 20 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "resolv"
  3. 26 require "uri"
  4. 26 require "cgi"
  5. 26 require "forwardable"
  6. 26 require "httpx/base64"
  7. 26 module HTTPX
  8. 26 class Resolver::HTTPS < Resolver::Resolver
  9. 26 extend Forwardable
  10. 26 using URIExtensions
  11. 26 module DNSExtensions
  12. 26 refine Resolv::DNS do
  13. 26 def generate_candidates(name)
  14. 42 @config.generate_candidates(name)
  15. end
  16. end
  17. end
  18. 26 using DNSExtensions
  19. 26 NAMESERVER = "https://1.1.1.1/dns-query"
  20. 2 DEFAULTS = {
  21. 24 uri: NAMESERVER,
  22. use_get: false,
  23. }.freeze
  24. 26 def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?
  25. 26 def initialize(_, options)
  26. 90 super
  27. 90 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  28. 90 @queries = {}
  29. 90 @requests = {}
  30. 90 @connections = []
  31. 90 @uri = URI(@resolver_options[:uri])
  32. 90 @uri_addresses = nil
  33. 90 @resolver = Resolv::DNS.new
  34. 90 @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
  35. 90 @resolver.lazy_initialize
  36. end
  37. 26 def <<(connection)
  38. 90 return if @uri.origin == connection.peer.to_s
  39. 48 @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
  40. 48 if @uri_addresses.empty?
  41. 6 ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
  42. 6 ex.set_backtrace(caller)
  43. 6 connection.force_reset
  44. 6 throw(:resolve_error, ex)
  45. end
  46. 42 resolve(connection)
  47. end
  48. 26 def closed?
  49. true
  50. end
  51. 26 def empty?
  52. 84 true
  53. end
  54. 26 def resolver_connection
  55. # TODO: leaks connection object into the pool
  56. 66 @resolver_connection ||= @current_session.find_connection(@uri, @current_selector,
  57. @options.merge(ssl: { alpn_protocols: %w[h2] })).tap do |conn|
  58. 42 emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
  59. end
  60. end
  61. 26 private
  62. 26 def resolve(connection = @connections.first, hostname = nil)
  63. 66 return unless connection
  64. 66 hostname ||= @queries.key(connection)
  65. 66 if hostname.nil?
  66. 42 hostname = connection.peer.host
  67. log do
  68. "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
  69. 42 end if connection.peer.non_ascii_hostname
  70. 42 hostname = @resolver.generate_candidates(hostname).each do |name|
  71. 126 @queries[name.to_s] = connection
  72. end.first.to_s
  73. else
  74. 24 @queries[hostname] = connection
  75. end
  76. 66 log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
  77. begin
  78. 66 request = build_request(hostname)
  79. 66 request.on(:response, &method(:on_response).curry(2)[request])
  80. 66 request.on(:promise, &method(:on_promise))
  81. 66 @requests[request] = hostname
  82. 66 resolver_connection.send(request)
  83. 66 @connections << connection
  84. rescue ResolveError, Resolv::DNS::EncodeError => e
  85. reset_hostname(hostname)
  86. emit_resolve_error(connection, connection.peer.host, e)
  87. end
  88. end
  89. 26 def on_response(request, response)
  90. 66 response.raise_for_status
  91. rescue StandardError => e
  92. 6 hostname = @requests.delete(request)
  93. 6 connection = reset_hostname(hostname)
  94. 6 emit_resolve_error(connection, connection.peer.host, e)
  95. else
  96. # @type var response: HTTPX::Response
  97. 60 parse(request, response)
  98. ensure
  99. 66 @requests.delete(request)
  100. end
  101. 26 def on_promise(_, stream)
  102. log(level: 2) { "#{stream.id}: refusing stream!" }
  103. stream.refuse
  104. end
  105. 26 def parse(request, response)
  106. 60 code, result = decode_response_body(response)
  107. 60 case code
  108. when :ok
  109. 18 parse_addresses(result, request)
  110. when :no_domain_found
  111. # Indicates no such domain was found.
  112. 36 host = @requests.delete(request)
  113. 36 connection = reset_hostname(host, reset_candidates: false)
  114. 36 unless @queries.value?(connection)
  115. 12 emit_resolve_error(connection)
  116. 12 return
  117. end
  118. 24 resolve
  119. when :dns_error
  120. host = @requests.delete(request)
  121. connection = reset_hostname(host)
  122. emit_resolve_error(connection)
  123. when :decode_error
  124. 6 host = @requests.delete(request)
  125. 6 connection = reset_hostname(host)
  126. 6 emit_resolve_error(connection, connection.peer.host, result)
  127. end
  128. end
  129. 26 def parse_addresses(answers, request)
  130. 18 if answers.empty?
  131. # no address found, eliminate candidates
  132. host = @requests.delete(request)
  133. connection = reset_hostname(host)
  134. emit_resolve_error(connection)
  135. return
  136. else
  137. 42 answers = answers.group_by { |answer| answer["name"] }
  138. 18 answers.each do |hostname, addresses|
  139. 24 addresses = addresses.flat_map do |address|
  140. 24 if address.key?("alias")
  141. 6 alias_address = answers[address["alias"]]
  142. 6 if alias_address.nil?
  143. reset_hostname(address["name"])
  144. if early_resolve(connection, hostname: address["alias"])
  145. @connections.delete(connection)
  146. else
  147. resolve(connection, address["alias"])
  148. return # rubocop:disable Lint/NonLocalExitFromIterator
  149. end
  150. else
  151. 6 alias_address
  152. end
  153. else
  154. 18 address
  155. end
  156. end.compact
  157. 24 next if addresses.empty?
  158. 24 hostname.delete_suffix!(".") if hostname.end_with?(".")
  159. 24 connection = reset_hostname(hostname, reset_candidates: false)
  160. 24 next unless connection # probably a retried query for which there's an answer
  161. 18 @connections.delete(connection)
  162. # eliminate other candidates
  163. 54 @queries.delete_if { |_, conn| connection == conn }
  164. 18 Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
  165. 54 catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
  166. end
  167. end
  168. 18 return if @connections.empty?
  169. resolve
  170. end
  171. 26 def build_request(hostname)
  172. 60 uri = @uri.dup
  173. 60 rklass = @options.request_class
  174. 60 payload = Resolver.encode_dns_query(hostname, type: @record_type)
  175. 60 if @resolver_options[:use_get]
  176. 6 params = URI.decode_www_form(uri.query.to_s)
  177. 6 params << ["type", FAMILY_TYPES[@record_type]]
  178. 6 params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
  179. 6 uri.query = URI.encode_www_form(params)
  180. 6 request = rklass.new("GET", uri, @options)
  181. else
  182. 54 request = rklass.new("POST", uri, @options, body: [payload])
  183. 54 request.headers["content-type"] = "application/dns-message"
  184. end
  185. 60 request.headers["accept"] = "application/dns-message"
  186. 60 request
  187. end
  188. 26 def decode_response_body(response)
  189. 54 case response.headers["content-type"]
  190. when "application/dns-udpwireformat",
  191. "application/dns-message"
  192. 54 Resolver.decode_dns_answer(response.to_s)
  193. else
  194. raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
  195. end
  196. end
  197. 26 def reset_hostname(hostname, reset_candidates: true)
  198. 72 connection = @queries.delete(hostname)
  199. 72 return connection unless connection && reset_candidates
  200. # eliminate other candidates
  201. 36 candidates = @queries.select { |_, conn| connection == conn }.keys
  202. 36 @queries.delete_if { |h, _| candidates.include?(h) }
  203. 12 connection
  204. end
  205. end
  206. end

lib/httpx/resolver/multi.rb

88.24% lines covered

51 relevant lines. 45 lines covered and 6 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "forwardable"
  3. 26 require "resolv"
  4. 26 module HTTPX
  5. 26 class Resolver::Multi
  6. 26 include Callbacks
  7. 26 using ArrayExtensions::FilterMap
  8. 26 attr_reader :resolvers, :options
  9. 26 def initialize(resolver_type, options)
  10. 6859 @current_selector = nil
  11. 6859 @current_session = nil
  12. 6859 @options = options
  13. 6859 @resolver_options = @options.resolver_options
  14. 6859 @resolvers = options.ip_families.map do |ip_family|
  15. 6859 resolver = resolver_type.new(ip_family, options)
  16. 6859 resolver.multi = self
  17. 6859 resolver
  18. end
  19. 6859 @errors = Hash.new { |hs, k| hs[k] = [] }
  20. end
  21. 26 def current_selector=(s)
  22. 6889 @current_selector = s
  23. 13778 @resolvers.each { |r| r.__send__(__method__, s) }
  24. end
  25. 26 def current_session=(s)
  26. 6889 @current_session = s
  27. 13778 @resolvers.each { |r| r.__send__(__method__, s) }
  28. end
  29. 26 def closed?
  30. @resolvers.all?(&:closed?)
  31. end
  32. 26 def empty?
  33. @resolvers.all?(&:empty?)
  34. end
  35. 26 def inflight?
  36. @resolvers.any(&:inflight?)
  37. end
  38. 26 def timeout
  39. @resolvers.filter_map(&:timeout).min
  40. end
  41. 26 def close
  42. @resolvers.each(&:close)
  43. end
  44. 26 def connections
  45. @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
  46. end
  47. 26 def early_resolve(connection)
  48. 6891 hostname = connection.peer.host
  49. 6891 addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  50. 6891 return false unless addresses
  51. 6446 resolved = false
  52. 6685 addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
  53. # try to match the resolver by family. However, there are cases where that's not possible, as when
  54. # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
  55. 13338 resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
  56. 6669 next unless resolver # this should ever happen
  57. # it does not matter which resolver it is, as early-resolve code is shared.
  58. 6669 resolver.emit_addresses(connection, family, addrs, true)
  59. 6630 resolved = true
  60. end
  61. 6407 resolved
  62. end
  63. 26 def lazy_resolve(connection)
  64. 445 @resolvers.each do |resolver|
  65. 445 resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
  66. 433 next if resolver.empty?
  67. 349 @current_session.select_resolver(resolver, @current_selector)
  68. end
  69. end
  70. end
  71. end

lib/httpx/resolver/native.rb

88.39% lines covered

310 relevant lines. 274 lines covered and 36 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "forwardable"
  3. 26 require "resolv"
  4. 26 module HTTPX
  5. 26 class Resolver::Native < Resolver::Resolver
  6. 26 extend Forwardable
  7. 26 using URIExtensions
  8. 16 DEFAULTS = {
  9. 10 nameserver: nil,
  10. **Resolv::DNS::Config.default_config_hash,
  11. packet_size: 512,
  12. timeouts: Resolver::RESOLVE_TIMEOUT,
  13. }.freeze
  14. 26 DNS_PORT = 53
  15. 26 def_delegator :@connections, :empty?
  16. 26 attr_reader :state
  17. 26 def initialize(family, options)
  18. 6769 super
  19. 6769 @ns_index = 0
  20. 6769 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  21. 6769 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  22. 6769 @nameserver = if (nameserver = @resolver_options[:nameserver])
  23. 6763 nameserver = nameserver[family] if nameserver.is_a?(Hash)
  24. 6763 Array(nameserver)
  25. end
  26. 6769 @ndots = @resolver_options.fetch(:ndots, 1)
  27. 20307 @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
  28. 6769 @_timeouts = Array(@resolver_options[:timeouts])
  29. 8117 @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
  30. 6769 @connections = []
  31. 6769 @name = nil
  32. 6769 @queries = {}
  33. 6769 @read_buffer = "".b
  34. 6769 @write_buffer = Buffer.new(@resolver_options[:packet_size])
  35. 6769 @state = :idle
  36. end
  37. 26 def close
  38. 347 transition(:closed)
  39. end
  40. 26 def closed?
  41. 712 @state == :closed
  42. end
  43. 26 def to_io
  44. 1064 @io.to_io
  45. end
  46. 26 def call
  47. 882 case @state
  48. when :open
  49. 934 consume
  50. end
  51. end
  52. 26 def interests
  53. 17851 case @state
  54. when :idle
  55. 17233 transition(:open)
  56. when :closed
  57. 14 transition(:idle)
  58. 14 transition(:open)
  59. end
  60. 17910 calculate_interests
  61. end
  62. 26 def <<(connection)
  63. 355 if @nameserver.nil?
  64. 6 ex = ResolveError.new("No available nameserver")
  65. 6 ex.set_backtrace(caller)
  66. 6 connection.force_reset
  67. 6 throw(:resolve_error, ex)
  68. else
  69. 349 @connections << connection
  70. 349 resolve
  71. end
  72. end
  73. 26 def timeout
  74. 17910 return if @connections.empty?
  75. 17910 @start_timeout = Utils.now
  76. 17910 hosts = @queries.keys
  77. 17910 @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
  78. end
  79. 26 def handle_socket_timeout(interval); end
  80. 26 private
  81. 26 def calculate_interests
  82. 21965 return :w unless @write_buffer.empty?
  83. 20352 return :r unless @queries.empty?
  84. 371 nil
  85. end
  86. 26 def consume
  87. 946 loop do
  88. 1673 dread if calculate_interests == :r
  89. 1655 break unless calculate_interests == :w
  90. # do_retry
  91. 745 dwrite
  92. 727 break unless calculate_interests == :r
  93. end
  94. rescue Errno::EHOSTUNREACH => e
  95. 18 @ns_index += 1
  96. 18 nameserver = @nameserver
  97. 18 if nameserver && @ns_index < nameserver.size
  98. 12 log do
  99. "resolver #{FAMILY_TYPES[@record_type]}: " \
  100. "failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
  101. end
  102. 12 transition(:idle)
  103. 12 @timeouts.clear
  104. 12 retry
  105. else
  106. 6 handle_error(e)
  107. 6 emit(:close, self)
  108. end
  109. rescue NativeResolveError => e
  110. 18 handle_error(e)
  111. 18 close_or_resolve
  112. 18 retry unless closed?
  113. end
  114. 26 def schedule_retry
  115. 727 h = @name
  116. 727 return unless h
  117. 727 connection = @queries[h]
  118. 727 timeouts = @timeouts[h]
  119. 727 timeout = timeouts.shift
  120. 727 @timer = @current_selector.after(timeout) do
  121. 72 next unless @connections.include?(connection)
  122. 72 do_retry(h, connection, timeout)
  123. end
  124. end
  125. 26 def do_retry(h, connection, interval)
  126. 72 timeouts = @timeouts[h]
  127. 72 if !timeouts.empty?
  128. 42 log do
  129. "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..."
  130. end
  131. # must downgrade to tcp AND retry on same host as last
  132. 42 downgrade_socket
  133. 42 resolve(connection, h)
  134. 30 elsif @ns_index + 1 < @nameserver.size
  135. # try on the next nameserver
  136. 6 @ns_index += 1
  137. 6 log do
  138. "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
  139. end
  140. 6 transition(:idle)
  141. 6 @timeouts.clear
  142. 6 resolve(connection, h)
  143. else
  144. 24 @timeouts.delete(h)
  145. 24 reset_hostname(h, reset_candidates: false)
  146. 24 unless @queries.empty?
  147. 18 resolve(connection)
  148. 18 return
  149. end
  150. 6 @connections.delete(connection)
  151. 6 host = connection.peer.host
  152. # This loop_time passed to the exception is bogus. Ideally we would pass the total
  153. # resolve timeout, including from the previous retries.
  154. 6 ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
  155. 6 ex.set_backtrace(ex ? ex.backtrace : caller)
  156. 6 emit_resolve_error(connection, host, ex)
  157. 6 close_or_resolve
  158. end
  159. end
  160. 26 def dread(wsize = @resolver_options[:packet_size])
  161. 1236 loop do
  162. 1248 wsize = @large_packet.capacity if @large_packet
  163. 1248 siz = @io.read(wsize, @read_buffer)
  164. 1248 unless siz
  165. ex = EOFError.new("descriptor closed")
  166. ex.set_backtrace(caller)
  167. raise ex
  168. end
  169. 1248 return unless siz.positive?
  170. 667 if @socket_type == :tcp
  171. # packet may be incomplete, need to keep draining from the socket
  172. 30 if @large_packet
  173. # large packet buffer already exists, continue pumping
  174. 12 @large_packet << @read_buffer
  175. 12 next unless @large_packet.full?
  176. 12 parse(@large_packet.to_s)
  177. 12 @large_packet = nil
  178. # downgrade to udp again
  179. 12 downgrade_socket
  180. 12 return
  181. else
  182. 18 size = @read_buffer[0, 2].unpack1("n")
  183. 18 buffer = @read_buffer.byteslice(2..-1)
  184. 18 if size > @read_buffer.bytesize
  185. # only do buffer logic if it's worth it, and the whole packet isn't here already
  186. 12 @large_packet = Buffer.new(size)
  187. 12 @large_packet << buffer
  188. 12 next
  189. else
  190. 6 parse(buffer)
  191. end
  192. end
  193. else # udp
  194. 637 parse(@read_buffer)
  195. end
  196. 625 return if @state == :closed || !@write_buffer.empty?
  197. end
  198. end
  199. 26 def dwrite
  200. 727 loop do
  201. 1454 return if @write_buffer.empty?
  202. 727 siz = @io.write(@write_buffer)
  203. 727 unless siz
  204. ex = EOFError.new("descriptor closed")
  205. ex.set_backtrace(caller)
  206. raise ex
  207. end
  208. 727 return unless siz.positive?
  209. 727 schedule_retry if @write_buffer.empty?
  210. 727 return if @state == :closed
  211. end
  212. end
  213. 26 def parse(buffer)
  214. 655 @timer.cancel
  215. 655 code, result = Resolver.decode_dns_answer(buffer)
  216. 598 case code
  217. when :ok
  218. 223 parse_addresses(result)
  219. when :no_domain_found
  220. # Indicates no such domain was found.
  221. 408 hostname, connection = @queries.first
  222. 408 reset_hostname(hostname, reset_candidates: false)
  223. 714 other_candidate, _ = @queries.find { |_, conn| conn == connection }
  224. 408 if other_candidate
  225. 306 resolve(connection, other_candidate)
  226. else
  227. 102 @connections.delete(connection)
  228. 102 ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
  229. 102 ex.set_backtrace(ex ? ex.backtrace : caller)
  230. 102 emit_resolve_error(connection, connection.peer.host, ex)
  231. 102 close_or_resolve
  232. end
  233. when :message_truncated
  234. # TODO: what to do if it's already tcp??
  235. 12 return if @socket_type == :tcp
  236. 12 @socket_type = :tcp
  237. 12 hostname, _ = @queries.first
  238. 12 reset_hostname(hostname)
  239. 12 transition(:closed)
  240. when :dns_error
  241. 6 hostname, connection = @queries.first
  242. 6 reset_hostname(hostname)
  243. 6 @connections.delete(connection)
  244. 6 ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
  245. 6 raise ex
  246. when :decode_error
  247. 6 hostname, connection = @queries.first
  248. 6 reset_hostname(hostname)
  249. 6 @connections.delete(connection)
  250. 6 ex = NativeResolveError.new(connection, connection.peer.host, result.message)
  251. 6 ex.set_backtrace(result.backtrace)
  252. 6 raise ex
  253. end
  254. end
  255. 26 def parse_addresses(addresses)
  256. 223 if addresses.empty?
  257. # no address found, eliminate candidates
  258. 6 hostname, connection = @queries.first
  259. 6 reset_hostname(hostname)
  260. 6 @connections.delete(connection)
  261. 6 raise NativeResolveError.new(connection, connection.peer.host)
  262. else
  263. 217 address = addresses.first
  264. 217 name = address["name"]
  265. 217 connection = @queries.delete(name)
  266. 217 unless connection
  267. orig_name = name
  268. # absolute name
  269. name_labels = Resolv::DNS::Name.create(name).to_a
  270. name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
  271. # probably a retried query for which there's an answer
  272. unless name
  273. @timeouts.delete(orig_name)
  274. return
  275. end
  276. address["name"] = name
  277. connection = @queries.delete(name)
  278. end
  279. 1223 alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
  280. 217 if addresses.empty? && !alias_addresses.empty? # CNAME
  281. hostname_alias = alias_addresses.first["alias"]
  282. # clean up intermediate queries
  283. @timeouts.delete(name) unless connection.peer.host == name
  284. if early_resolve(connection, hostname: hostname_alias)
  285. @connections.delete(connection)
  286. else
  287. if @socket_type == :tcp
  288. # must downgrade to udp if tcp
  289. @socket_type = @resolver_options.fetch(:socket_type, :udp)
  290. transition(:idle)
  291. transition(:open)
  292. end
  293. log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
  294. resolve(connection, hostname_alias)
  295. return
  296. end
  297. else
  298. 217 reset_hostname(name, connection: connection)
  299. 217 @timeouts.delete(connection.peer.host)
  300. 217 @connections.delete(connection)
  301. 217 Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
  302. 1422 catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
  303. end
  304. end
  305. 217 close_or_resolve
  306. end
  307. 26 def resolve(connection = nil, hostname = nil)
  308. 735 @connections.shift until @connections.empty? || @connections.first.state != :closed
  309. 1100 connection ||= @connections.find { |c| !@queries.value?(c) }
  310. 735 raise Error, "no URI to resolve" unless connection
  311. 735 return unless @write_buffer.empty?
  312. 733 hostname ||= @queries.key(connection)
  313. 733 if hostname.nil?
  314. 361 hostname = connection.peer.host
  315. log do
  316. "resolver #{FAMILY_TYPES[@record_type]}: " \
  317. "resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
  318. 360 end if connection.peer.non_ascii_hostname
  319. 361 hostname = generate_candidates(hostname).each do |name|
  320. 1324 @queries[name] = connection
  321. end.first
  322. else
  323. 344 @queries[hostname] = connection
  324. end
  325. 733 @name = hostname
  326. 733 log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
  327. 51 begin
  328. 733 @write_buffer << encode_dns_query(hostname)
  329. rescue Resolv::DNS::EncodeError => e
  330. reset_hostname(hostname, connection: connection)
  331. @connections.delete(connection)
  332. emit_resolve_error(connection, hostname, e)
  333. close_or_resolve
  334. end
  335. end
  336. 26 def encode_dns_query(hostname)
  337. 733 message_id = Resolver.generate_id
  338. 733 msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
  339. 733 msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
  340. 733 msg
  341. end
  342. 26 def generate_candidates(name)
  343. 361 return [name] if name.end_with?(".")
  344. 361 candidates = []
  345. 361 name_parts = name.scan(/[^.]+/)
  346. 361 candidates = [name] if @ndots <= name_parts.size - 1
  347. 1083 candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
  348. 361 fname = "#{name}."
  349. 361 candidates << fname unless candidates.include?(fname)
  350. 361 candidates
  351. end
  352. 26 def build_socket
  353. 377 ip, port = @nameserver[@ns_index]
  354. 377 port ||= DNS_PORT
  355. 347 case @socket_type
  356. when :udp
  357. 359 log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
  358. 359 UDP.new(ip, port, @options)
  359. when :tcp
  360. 18 log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
  361. 18 origin = URI("tcp://#{ip}:#{port}")
  362. 18 TCP.new(origin, [ip], @options)
  363. end
  364. end
  365. 26 def downgrade_socket
  366. 54 return unless @socket_type == :tcp
  367. 12 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  368. 12 transition(:idle)
  369. 12 transition(:open)
  370. end
  371. 26 def transition(nextstate)
  372. 17602 case nextstate
  373. when :idle
  374. 44 if @io
  375. 38 @io.close
  376. 38 @io = nil
  377. end
  378. when :open
  379. 17259 return unless @state == :idle
  380. 17259 @io ||= build_socket
  381. 17259 @io.connect
  382. 17259 return unless @io.connected?
  383. 377 resolve if @queries.empty? && !@connections.empty?
  384. when :closed
  385. 359 return unless @state == :open
  386. 353 @io.close if @io
  387. 353 @start_timeout = nil
  388. 353 @write_buffer.clear
  389. 353 @read_buffer.clear
  390. end
  391. 774 @state = nextstate
  392. rescue Errno::ECONNREFUSED,
  393. Errno::EADDRNOTAVAIL,
  394. Errno::EHOSTUNREACH,
  395. SocketError,
  396. IOError,
  397. ConnectTimeoutError => e
  398. # these errors may happen during TCP handshake
  399. # treat them as resolve errors.
  400. handle_error(e)
  401. emit(:close, self)
  402. end
  403. 26 def handle_error(error)
  404. 24 if error.respond_to?(:connection) &&
  405. error.respond_to?(:host)
  406. 18 reset_hostname(error.host, connection: error.connection)
  407. 18 @connections.delete(error.connection)
  408. 18 emit_resolve_error(error.connection, error.host, error)
  409. else
  410. 6 @queries.each do |host, connection|
  411. 6 reset_hostname(host, connection: connection)
  412. 6 @connections.delete(connection)
  413. 6 emit_resolve_error(connection, host, error)
  414. end
  415. 12 while (connection = @connections.shift)
  416. emit_resolve_error(connection, host, error)
  417. end
  418. end
  419. end
  420. 26 def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
  421. 703 @timeouts.delete(hostname)
  422. 703 return unless connection && reset_candidates
  423. # eliminate other candidates
  424. 1036 candidates = @queries.select { |_, conn| connection == conn }.keys
  425. 1036 @queries.delete_if { |h, _| candidates.include?(h) }
  426. # reset timeouts
  427. 1012 @timeouts.delete_if { |h, _| candidates.include?(h) }
  428. end
  429. 26 def close_or_resolve
  430. # drop already closed connections
  431. 343 @connections.shift until @connections.empty? || @connections.first.state != :closed
  432. 343 if (@connections - @queries.values).empty?
  433. 341 emit(:close, self)
  434. else
  435. 2 resolve
  436. end
  437. end
  438. end
  439. end

lib/httpx/resolver/resolver.rb

83.13% lines covered

83 relevant lines. 69 lines covered and 14 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "resolv"
  3. 26 require "ipaddr"
  4. 26 module HTTPX
  5. 26 class Resolver::Resolver
  6. 26 include Callbacks
  7. 26 include Loggable
  8. 26 using ArrayExtensions::Intersect
  9. 2 RECORD_TYPES = {
  10. 24 Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
  11. Socket::AF_INET => Resolv::DNS::Resource::IN::A,
  12. }.freeze
  13. 2 FAMILY_TYPES = {
  14. 24 Resolv::DNS::Resource::IN::AAAA => "AAAA",
  15. Resolv::DNS::Resource::IN::A => "A",
  16. }.freeze
  17. 26 class << self
  18. 26 def multi?
  19. 6859 true
  20. end
  21. end
  22. 26 attr_reader :family, :options
  23. 26 attr_writer :current_selector, :current_session
  24. 26 attr_accessor :multi
  25. 26 def initialize(family, options)
  26. 6884 @family = family
  27. 6884 @record_type = RECORD_TYPES[family]
  28. 6884 @options = options
  29. 6884 set_resolver_callbacks
  30. end
  31. 26 def each_connection(&block)
  32. 215 enum_for(__method__) unless block
  33. 215 return unless @connections
  34. 215 @connections.each(&block)
  35. end
  36. 26 def close; end
  37. 26 alias_method :terminate, :close
  38. 26 def closed?
  39. true
  40. end
  41. 26 def empty?
  42. true
  43. end
  44. 26 def inflight?
  45. false
  46. end
  47. 26 def emit_addresses(connection, family, addresses, early_resolve = false)
  48. 6958 addresses.map! do |address|
  49. 14570 address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
  50. end
  51. # double emission check, but allow early resolution to work
  52. 6958 return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
  53. 6958 log do
  54. 64 "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
  55. 5 "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
  56. end
  57. 6958 if !early_resolve && # do not apply resolution delay for non-dns name resolution
  58. @current_selector && # just in case...
  59. family == Socket::AF_INET && # resolution delay only applies to IPv4
  60. !connection.io && # connection already has addresses and initiated/ended handshake
  61. connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
  62. addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
  63. log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
  64. @current_selector.after(0.05) do
  65. # double emission check
  66. unless connection.addresses && addresses.intersect?(connection.addresses)
  67. emit_resolved_connection(connection, addresses, early_resolve)
  68. end
  69. end
  70. else
  71. 6958 emit_resolved_connection(connection, addresses, early_resolve)
  72. end
  73. end
  74. 26 private
  75. 26 def emit_resolved_connection(connection, addresses, early_resolve)
  76. begin
  77. 6958 connection.addresses = addresses
  78. 6912 return if connection.state == :closed
  79. 6912 emit(:resolve, connection)
  80. 24 rescue StandardError => e
  81. 46 if early_resolve
  82. 39 connection.force_reset
  83. 39 throw(:resolve_error, e)
  84. else
  85. 7 emit(:error, connection, e)
  86. end
  87. end
  88. end
  89. 26 def early_resolve(connection, hostname: connection.peer.host)
  90. addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  91. return false unless addresses
  92. addresses = addresses.select { |addr| addr.family == @family }
  93. return false if addresses.empty?
  94. emit_addresses(connection, @family, addresses, true)
  95. true
  96. end
  97. 26 def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
  98. 169 emit_connection_error(connection, resolve_error(hostname, ex))
  99. end
  100. 26 def resolve_error(hostname, ex = nil)
  101. 169 return ex if ex.is_a?(ResolveError) || ex.is_a?(ResolveTimeoutError)
  102. 42 message = ex ? ex.message : "Can't resolve #{hostname}"
  103. 42 error = ResolveError.new(message)
  104. 42 error.set_backtrace(ex ? ex.backtrace : caller)
  105. 42 error
  106. end
  107. 26 def set_resolver_callbacks
  108. 6884 on(:resolve, &method(:resolve_connection))
  109. 6884 on(:error, &method(:emit_connection_error))
  110. 6884 on(:close, &method(:close_resolver))
  111. end
  112. 26 def resolve_connection(connection)
  113. 6912 @current_session.__send__(:on_resolver_connection, connection, @current_selector)
  114. end
  115. 26 def emit_connection_error(connection, error)
  116. 163 return connection.handle_connect_error(error) if connection.connecting?
  117. connection.emit(:error, error)
  118. end
  119. 26 def close_resolver(resolver)
  120. 347 @current_session.__send__(:on_resolver_close, resolver, @current_selector)
  121. end
  122. end
  123. end

lib/httpx/resolver/system.rb

81.06% lines covered

132 relevant lines. 107 lines covered and 25 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "resolv"
  3. 26 module HTTPX
  4. 26 class Resolver::System < Resolver::Resolver
  5. 26 using URIExtensions
  6. 26 RESOLV_ERRORS = [Resolv::ResolvError,
  7. Resolv::DNS::Requester::RequestError,
  8. Resolv::DNS::EncodeError,
  9. Resolv::DNS::DecodeError].freeze
  10. 26 DONE = 1
  11. 26 ERROR = 2
  12. 26 class << self
  13. 26 def multi?
  14. 25 false
  15. end
  16. end
  17. 26 attr_reader :state
  18. 26 def initialize(options)
  19. 25 super(nil, options)
  20. 25 @resolver_options = @options.resolver_options
  21. 25 resolv_options = @resolver_options.dup
  22. 25 timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
  23. 25 @_timeouts = Array(timeouts)
  24. 50 @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
  25. 25 resolv_options.delete(:cache)
  26. 25 @connections = []
  27. 25 @queries = []
  28. 25 @ips = []
  29. 25 @pipe_mutex = Thread::Mutex.new
  30. 25 @state = :idle
  31. end
  32. 26 def resolvers
  33. return enum_for(__method__) unless block_given?
  34. yield self
  35. end
  36. 26 def multi
  37. self
  38. end
  39. 26 def empty?
  40. true
  41. end
  42. 26 def close
  43. transition(:closed)
  44. end
  45. 26 def closed?
  46. @state == :closed
  47. end
  48. 26 def to_io
  49. @pipe_read.to_io
  50. end
  51. 26 def call
  52. case @state
  53. when :open
  54. consume
  55. end
  56. nil
  57. end
  58. 26 def interests
  59. return if @queries.empty?
  60. :r
  61. end
  62. 26 def timeout
  63. return unless @queries.empty?
  64. _, connection = @queries.first
  65. return unless connection
  66. @timeouts[connection.peer.host].first
  67. end
  68. 26 def <<(connection)
  69. 25 @connections << connection
  70. 25 resolve
  71. end
  72. 26 def early_resolve(connection, **)
  73. 25 self << connection
  74. 12 true
  75. end
  76. 26 def handle_socket_timeout(interval)
  77. error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
  78. error.set_backtrace(caller)
  79. on_error(error)
  80. end
  81. 26 private
  82. 26 def transition(nextstate)
  83. 25 case nextstate
  84. when :idle
  85. @timeouts.clear
  86. when :open
  87. 25 return unless @state == :idle
  88. 25 @pipe_read, @pipe_write = ::IO.pipe
  89. when :closed
  90. return unless @state == :open
  91. @pipe_write.close
  92. @pipe_read.close
  93. end
  94. 25 @state = nextstate
  95. end
  96. 26 def consume
  97. 25 return if @connections.empty?
  98. 25 if @pipe_read.wait_readable
  99. 25 event = @pipe_read.getbyte
  100. 25 case event
  101. when DONE
  102. 24 *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
  103. 12 @queries.delete(pair)
  104. 12 _, connection = pair
  105. 12 @connections.delete(connection)
  106. 12 family, connection = pair
  107. 24 catch(:coalesced) { emit_addresses(connection, family, addrs) }
  108. when ERROR
  109. 26 *pair, error = @pipe_mutex.synchronize { @ips.pop }
  110. 13 @queries.delete(pair)
  111. 13 @connections.delete(connection)
  112. 13 _, connection = pair
  113. 13 emit_resolve_error(connection, connection.peer.host, error)
  114. end
  115. end
  116. 12 return emit(:close, self) if @connections.empty?
  117. resolve
  118. end
  119. 26 def resolve(connection = @connections.first)
  120. 25 raise Error, "no URI to resolve" unless connection
  121. 25 return unless @queries.empty?
  122. 25 hostname = connection.peer.host
  123. 25 scheme = connection.origin.scheme
  124. log do
  125. "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
  126. 25 end if connection.peer.non_ascii_hostname
  127. 25 transition(:open)
  128. 25 connection.options.ip_families.each do |family|
  129. 25 @queries << [family, connection]
  130. end
  131. 25 async_resolve(connection, hostname, scheme)
  132. 25 consume
  133. end
  134. 26 def async_resolve(connection, hostname, scheme)
  135. 25 families = connection.options.ip_families
  136. 25 log { "resolver: query for #{hostname}" }
  137. 25 timeouts = @timeouts[connection.peer.host]
  138. 25 resolve_timeout = timeouts.first
  139. 25 Thread.start do
  140. 25 Thread.current.report_on_exception = false
  141. begin
  142. 25 addrs = if resolve_timeout
  143. 25 Timeout.timeout(resolve_timeout) do
  144. 25 __addrinfo_resolve(hostname, scheme)
  145. end
  146. else
  147. __addrinfo_resolve(hostname, scheme)
  148. end
  149. 12 addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
  150. 12 families.each do |family|
  151. 12 addresses = addrs[family]
  152. 12 next unless addresses
  153. 12 addresses.map!(&:ip_address)
  154. 12 addresses.uniq!
  155. 12 @pipe_mutex.synchronize do
  156. 12 @ips.unshift([family, connection, addresses])
  157. 12 @pipe_write.putc(DONE) unless @pipe_write.closed?
  158. end
  159. end
  160. rescue StandardError => e
  161. 13 if e.is_a?(Timeout::Error)
  162. 1 timeouts.shift
  163. 1 retry unless timeouts.empty?
  164. 1 e = ResolveTimeoutError.new(resolve_timeout, e.message)
  165. 1 e.set_backtrace(e.backtrace)
  166. end
  167. 13 @pipe_mutex.synchronize do
  168. 13 families.each do |family|
  169. 13 @ips.unshift([family, connection, e])
  170. 13 @pipe_write.putc(ERROR) unless @pipe_write.closed?
  171. end
  172. end
  173. end
  174. end
  175. end
  176. 26 def __addrinfo_resolve(host, scheme)
  177. 25 Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
  178. end
  179. 26 def emit_connection_error(_, error)
  180. 13 throw(:resolve_error, error)
  181. end
  182. 26 def close_resolver(resolver); end
  183. end
  184. end

lib/httpx/response.rb

100.0% lines covered

111 relevant lines. 111 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "objspace"
  3. 26 require "stringio"
  4. 26 require "tempfile"
  5. 26 require "fileutils"
  6. 26 require "forwardable"
  7. 26 module HTTPX
  8. # Defines a HTTP response is handled internally, with a few properties exposed as attributes.
  9. #
  10. # It delegates the following methods to the corresponding HTTPX::Request:
  11. #
  12. # * HTTPX::Request#uri
  13. # * HTTPX::Request#peer_address
  14. #
  15. # It implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads.
  16. #
  17. # It implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
  18. # (of payload chunks).
  19. #
  20. 26 class Response
  21. 26 extend Forwardable
  22. 26 include Callbacks
  23. # the HTTP response status code
  24. 26 attr_reader :status
  25. # an HTTPX::Headers object containing the response HTTP headers.
  26. 26 attr_reader :headers
  27. # a HTTPX::Response::Body object wrapping the response body. The following methods are delegated to it:
  28. #
  29. # * HTTPX::Response::Body#to_s
  30. # * HTTPX::Response::Body#to_str
  31. # * HTTPX::Response::Body#read
  32. # * HTTPX::Response::Body#copy_to
  33. # * HTTPX::Response::Body#close
  34. 26 attr_reader :body
  35. # The HTTP protocol version used to fetch the response.
  36. 26 attr_reader :version
  37. # returns the response body buffered in a string.
  38. 26 def_delegator :@body, :to_s
  39. 26 def_delegator :@body, :to_str
  40. # implements the IO reader +#read+ interface.
  41. 26 def_delegator :@body, :read
  42. # copies the response body to a different location.
  43. 26 def_delegator :@body, :copy_to
  44. # the corresponding request uri.
  45. 26 def_delegator :@request, :uri
  46. # the IP address of the peer server.
  47. 26 def_delegator :@request, :peer_address
  48. # inits the instance with the corresponding +request+ to this response, an the
  49. # response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
  50. 26 def initialize(request, status, version, headers)
  51. 8818 @request = request
  52. 8818 @options = request.options
  53. 8818 @version = version
  54. 8818 @status = Integer(status)
  55. 8818 @headers = @options.headers_class.new(headers)
  56. 8818 @body = @options.response_body_class.new(self, @options)
  57. 8818 @finished = complete?
  58. 8818 @content_type = nil
  59. end
  60. # closes the respective +@request+ and +@body+.
  61. 26 def close
  62. 419 @request.close
  63. 419 @body.close
  64. end
  65. # merges headers defined in +h+ into the response headers.
  66. 26 def merge_headers(h)
  67. 193 @headers = @headers.merge(h)
  68. end
  69. # writes +data+ chunk into the response body.
  70. 26 def <<(data)
  71. 11895 @body.write(data)
  72. end
  73. # returns the HTTPX::ContentType for the response, as per what's declared in the content-type header.
  74. #
  75. # response.content_type #=> #<HTTPX::ContentType:xxx @header_value="text/plain">
  76. # response.content_type.mime_type #=> "text/plain"
  77. 26 def content_type
  78. 9187 @content_type ||= ContentType.new(@headers["content-type"])
  79. end
  80. # returns whether the response has been fully fetched.
  81. 26 def finished?
  82. 4659 @finished
  83. end
  84. # marks the response as finished, freezes the headers.
  85. 26 def finish!
  86. 4527 @finished = true
  87. 4527 @headers.freeze
  88. end
  89. # returns whether the response contains body payload.
  90. 26 def bodyless?
  91. 8818 @request.verb == "HEAD" ||
  92. @status < 200 || # informational response
  93. @status == 204 ||
  94. @status == 205 ||
  95. @status == 304 || begin
  96. 8372 content_length = @headers["content-length"]
  97. 8372 return false if content_length.nil?
  98. 7255 content_length == "0"
  99. end
  100. end
  101. 26 def complete?
  102. 8818 bodyless? || (@request.verb == "CONNECT" && @status == 200)
  103. end
  104. skipped # :nocov:
  105. skipped def inspect
  106. skipped "#<Response:#{object_id} " \
  107. skipped "HTTP/#{version} " \
  108. skipped "@status=#{@status} " \
  109. skipped "@headers=#{@headers} " \
  110. skipped "@body=#{@body.bytesize}>"
  111. skipped end
  112. skipped # :nocov:
  113. # returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
  114. # status code, or nothing.
  115. #
  116. # ok_response.error #=> nil
  117. # not_found_response.error #=> HTTPX::HTTPError instance, status 404
  118. 26 def error
  119. 562 return if @status < 400
  120. 54 HTTPError.new(self)
  121. end
  122. # it raises the exception returned by +error+, or itself otherwise.
  123. #
  124. # ok_response.raise_for_status #=> ok_response
  125. # not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
  126. 26 def raise_for_status
  127. 522 return self unless (err = error)
  128. 38 raise err
  129. end
  130. # decodes the response payload into a ruby object **if** the payload is valid json.
  131. #
  132. # response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
  133. # response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
  134. 26 def json(*args)
  135. 129 decode(Transcoder::JSON, *args)
  136. end
  137. # decodes the response payload into a ruby object **if** the payload is valid
  138. # "application/x-www-urlencoded" or "multipart/form-data".
  139. 26 def form
  140. 64 decode(Transcoder::Form)
  141. end
  142. 26 def xml
  143. # TODO: remove at next major version.
  144. 8 warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
  145. 1 "Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
  146. 8 require "httpx/plugins/xml"
  147. 8 decode(Plugins::XML::Transcoder)
  148. end
  149. 26 private
  150. # decodes the response payload using the given +transcoder+, which implements the decoding logic.
  151. #
  152. # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
  153. # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
  154. 26 def decode(transcoder, *args)
  155. # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
  156. 217 decoder = transcoder.decode(self)
  157. 193 raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
  158. 193 @body.rewind
  159. 193 decoder.call(self, *args)
  160. end
  161. end
  162. # Helper class which decodes the HTTP "content-type" header.
  163. 26 class ContentType
  164. 26 MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
  165. 26 CHARSET_RE = /;\s*charset=([^;]+)/i.freeze
  166. 26 def initialize(header_value)
  167. 9149 @header_value = header_value
  168. end
  169. # returns the mime type declared in the header.
  170. #
  171. # ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
  172. 26 def mime_type
  173. 217 return @mime_type if defined?(@mime_type)
  174. 179 m = @header_value.to_s[MIME_TYPE_RE, 1]
  175. 179 m && @mime_type = m.strip.downcase
  176. end
  177. # returns the charset declared in the header.
  178. #
  179. # ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
  180. # ContentType.new("text/plain").charset #=> nil
  181. 26 def charset
  182. 8970 return @charset if defined?(@charset)
  183. 8970 m = @header_value.to_s[CHARSET_RE, 1]
  184. 8970 m && @charset = m.strip.delete('"')
  185. end
  186. end
  187. # Wraps an error which has happened while processing an HTTP Request. It has partial
  188. # public API parity with HTTPX::Response, so users should rely on it to infer whether
  189. # the returned response is one or the other.
  190. #
  191. # response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
  192. # response.raise_for_status #=> raises if it wraps an error
  193. 26 class ErrorResponse
  194. 26 include Loggable
  195. 26 extend Forwardable
  196. # the corresponding HTTPX::Request instance.
  197. 26 attr_reader :request
  198. # the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
  199. 26 attr_reader :response
  200. # the wrapped exception.
  201. 26 attr_reader :error
  202. # the request uri
  203. 26 def_delegator :@request, :uri
  204. # the IP address of the peer server.
  205. 26 def_delegator :@request, :peer_address
  206. 26 def initialize(request, error)
  207. 1187 @request = request
  208. 1187 @response = request.response if request.response.is_a?(Response)
  209. 1187 @error = error
  210. 1187 @options = request.options
  211. 1187 log_exception(@error)
  212. end
  213. # returns the exception full message.
  214. 26 def to_s
  215. 9 @error.full_message(highlight: false)
  216. end
  217. # closes the error resources.
  218. 26 def close
  219. 40 @response.close if @response
  220. end
  221. # always true for error responses.
  222. 26 def finished?
  223. 8 true
  224. end
  225. # raises the wrapped exception.
  226. 26 def raise_for_status
  227. 82 raise @error
  228. end
  229. # buffers lost chunks to error response
  230. 26 def <<(data)
  231. 8 return unless @response
  232. 8 @response << data
  233. end
  234. end
  235. end
  236. 26 require_relative "response/body"
  237. 26 require_relative "response/buffer"
  238. 26 require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"

lib/httpx/response/body.rb

100.0% lines covered

115 relevant lines. 115 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. # Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
  4. # (for buffering the response payload), the IO reader protocol (for consuming the response payload),
  5. # and can be iterated over (via #each, which yields the payload in chunks).
  6. 26 class Response::Body
  7. # the payload encoding (i.e. "utf-8", "ASCII-8BIT")
  8. 26 attr_reader :encoding
  9. # Array of encodings contained in the response "content-encoding" header.
  10. 26 attr_reader :encodings
  11. # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
  12. 26 def initialize(response, options)
  13. 8970 @response = response
  14. 8970 @headers = response.headers
  15. 8970 @options = options
  16. 8970 @window_size = options.window_size
  17. 8970 @encodings = []
  18. 8970 @length = 0
  19. 8970 @buffer = nil
  20. 8970 @reader = nil
  21. 8970 @state = :idle
  22. # initialize response encoding
  23. 8970 @encoding = if (enc = response.content_type.charset)
  24. 168 begin
  25. 1502 Encoding.find(enc)
  26. rescue ArgumentError
  27. 32 Encoding::BINARY
  28. end
  29. else
  30. 7468 Encoding::BINARY
  31. end
  32. 8970 initialize_inflaters
  33. end
  34. 26 def initialize_dup(other)
  35. 32 super
  36. 32 @buffer = other.instance_variable_get(:@buffer).dup
  37. end
  38. 26 def closed?
  39. 296 @state == :closed
  40. end
  41. # write the response payload +chunk+ into the buffer. Inflates the chunk when required
  42. # and supported.
  43. 26 def write(chunk)
  44. 11706 return if @state == :closed
  45. 11706 return 0 if chunk.empty?
  46. 11301 chunk = decode_chunk(chunk)
  47. 11301 size = chunk.bytesize
  48. 10051 @length += size
  49. 11301 transition(:open)
  50. 11301 @buffer.write(chunk)
  51. 11301 @response.emit(:chunk_received, chunk)
  52. 11285 size
  53. end
  54. # reads a chunk from the payload (implementation of the IO reader protocol).
  55. 26 def read(*args)
  56. 281 return unless @buffer
  57. 281 unless @reader
  58. 123 rewind
  59. 123 @reader = @buffer
  60. end
  61. 281 @reader.read(*args)
  62. end
  63. # size of the decoded response payload. May differ from "content-length" header if
  64. # response was encoded over-the-wire.
  65. 26 def bytesize
  66. 224 @length
  67. end
  68. # yields the payload in chunks.
  69. 26 def each
  70. 64 return enum_for(__method__) unless block_given?
  71. 5 begin
  72. 48 if @buffer
  73. 48 rewind
  74. 126 while (chunk = @buffer.read(@window_size))
  75. 48 yield(chunk.force_encoding(@encoding))
  76. end
  77. end
  78. ensure
  79. 48 close
  80. end
  81. end
  82. # returns the declared filename in the "contennt-disposition" header, when present.
  83. 26 def filename
  84. 48 return unless @headers.key?("content-disposition")
  85. 40 Utils.get_filename(@headers["content-disposition"])
  86. end
  87. # returns the full response payload as a string.
  88. 26 def to_s
  89. 4640 return "".b unless @buffer
  90. 4285 @buffer.to_s
  91. end
  92. 26 alias_method :to_str, :to_s
  93. # whether the payload is empty.
  94. 26 def empty?
  95. 32 @length.zero?
  96. end
  97. # copies the payload to +dest+.
  98. #
  99. # body.copy_to("path/to/file")
  100. # body.copy_to(Pathname.new("path/to/file"))
  101. # body.copy_to(File.new("path/to/file"))
  102. 26 def copy_to(dest)
  103. 48 return unless @buffer
  104. 48 rewind
  105. 48 if dest.respond_to?(:path) && @buffer.respond_to?(:path)
  106. 8 FileUtils.mv(@buffer.path, dest.path)
  107. else
  108. 40 ::IO.copy_stream(@buffer, dest)
  109. end
  110. end
  111. # closes/cleans the buffer, resets everything
  112. 26 def close
  113. 671 if @buffer
  114. 507 @buffer.close
  115. 507 @buffer = nil
  116. end
  117. 671 @length = 0
  118. 671 transition(:closed)
  119. end
  120. 26 def ==(other)
  121. 118 object_id == other.object_id || begin
  122. 118 if other.respond_to?(:read)
  123. 80 _with_same_buffer_pos { FileUtils.compare_stream(@buffer, other) }
  124. else
  125. 78 to_s == other.to_s
  126. end
  127. end
  128. end
  129. skipped # :nocov:
  130. skipped def inspect
  131. skipped "#<HTTPX::Response::Body:#{object_id} " \
  132. skipped "@state=#{@state} " \
  133. skipped "@length=#{@length}>"
  134. skipped end
  135. skipped # :nocov:
  136. # rewinds the response payload buffer.
  137. 26 def rewind
  138. 452 return unless @buffer
  139. # in case there's some reading going on
  140. 452 @reader = nil
  141. 452 @buffer.rewind
  142. end
  143. 26 private
  144. # prepares inflaters for the advertised encodings in "content-encoding" header.
  145. 26 def initialize_inflaters
  146. 8970 @inflaters = nil
  147. 8970 return unless @headers.key?("content-encoding")
  148. 187 return unless @options.decompress_response_body
  149. 171 @inflaters = @headers.get("content-encoding").filter_map do |encoding|
  150. 171 next if encoding == "identity"
  151. 171 inflater = self.class.initialize_inflater_by_encoding(encoding, @response)
  152. # do not uncompress if there is no decoder available. In fact, we can't reliably
  153. # continue decompressing beyond that, so ignore.
  154. 171 break unless inflater
  155. 171 @encodings << encoding
  156. 171 inflater
  157. end
  158. end
  159. # passes the +chunk+ through all inflaters to decode it.
  160. 26 def decode_chunk(chunk)
  161. 46 @inflaters.reverse_each do |inflater|
  162. 411 chunk = inflater.call(chunk)
  163. 11458 end if @inflaters
  164. 11459 chunk
  165. end
  166. # tries transitioning the body STM to the +nextstate+.
  167. 26 def transition(nextstate)
  168. 10650 case nextstate
  169. when :open
  170. 11301 return unless @state == :idle
  171. 6959 @buffer = Response::Buffer.new(
  172. threshold_size: @options.body_threshold_size,
  173. bytesize: @length,
  174. encoding: @encoding
  175. )
  176. when :closed
  177. 671 return if @state == :closed
  178. end
  179. 7630 @state = nextstate
  180. end
  181. 26 def _with_same_buffer_pos # :nodoc:
  182. 40 return yield unless @buffer && @buffer.respond_to?(:pos)
  183. # @type ivar @buffer: StringIO | Tempfile
  184. 40 current_pos = @buffer.pos
  185. 40 @buffer.rewind
  186. 4 begin
  187. 40 yield
  188. ensure
  189. 40 @buffer.pos = current_pos
  190. end
  191. end
  192. 26 class << self
  193. 26 def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
  194. 154 case encoding
  195. when "gzip"
  196. 155 Transcoder::GZIP.decode(response, **kwargs)
  197. when "deflate"
  198. 16 Transcoder::Deflate.decode(response, **kwargs)
  199. end
  200. end
  201. end
  202. end
  203. end

lib/httpx/response/buffer.rb

96.0% lines covered

50 relevant lines. 48 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "delegate"
  3. 26 require "stringio"
  4. 26 require "tempfile"
  5. 26 module HTTPX
  6. # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
  7. 26 class Response::Buffer < SimpleDelegator
  8. # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
  9. # the initial +bytesize+, and the +encoding+.
  10. 26 def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
  11. 7155 @threshold_size = threshold_size
  12. 7155 @bytesize = bytesize
  13. 7155 @encoding = encoding
  14. 7155 @buffer = StringIO.new("".b)
  15. 7155 super(@buffer)
  16. end
  17. 26 def initialize_dup(other)
  18. 32 super
  19. 32 @buffer = other.instance_variable_get(:@buffer).dup
  20. end
  21. # size in bytes of the buffered content.
  22. 26 def size
  23. 324 @bytesize
  24. end
  25. # writes the +chunk+ into the buffer.
  26. 26 def write(chunk)
  27. 10389 @bytesize += chunk.bytesize
  28. 11680 try_upgrade_buffer
  29. 11680 @buffer.write(chunk)
  30. end
  31. # returns the buffered content as a string.
  32. 26 def to_s
  33. 3870 case @buffer
  34. when StringIO
  35. 491 begin
  36. 4301 @buffer.string.force_encoding(@encoding)
  37. rescue ArgumentError
  38. @buffer.string
  39. end
  40. when Tempfile
  41. 80 rewind
  42. 160 content = _with_same_buffer_pos { @buffer.read }
  43. 9 begin
  44. 80 content.force_encoding(@encoding)
  45. rescue ArgumentError # ex: unknown encoding name - utf
  46. content
  47. end
  48. end
  49. end
  50. # closes the buffer.
  51. 26 def close
  52. 619 @buffer.close
  53. 619 @buffer.unlink if @buffer.respond_to?(:unlink)
  54. end
  55. 26 private
  56. # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
  57. # has been reached.
  58. 26 def try_upgrade_buffer
  59. 11680 return unless @bytesize > @threshold_size
  60. 440 return if @buffer.is_a?(Tempfile)
  61. 163 aux = @buffer
  62. 163 @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  63. 163 if aux
  64. 163 aux.rewind
  65. 163 ::IO.copy_stream(aux, @buffer)
  66. 163 aux.close
  67. end
  68. 163 __setobj__(@buffer)
  69. end
  70. 26 def _with_same_buffer_pos # :nodoc:
  71. 80 current_pos = @buffer.pos
  72. 80 @buffer.rewind
  73. 9 begin
  74. 80 yield
  75. ensure
  76. 80 @buffer.pos = current_pos
  77. end
  78. end
  79. end
  80. end

lib/httpx/selector.rb

92.52% lines covered

107 relevant lines. 99 lines covered and 8 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "io/wait"
  3. 26 module HTTPX
  4. 26 class Selector
  5. 26 extend Forwardable
  6. 26 READABLE = %i[rw r].freeze
  7. 26 WRITABLE = %i[rw w].freeze
  8. 26 private_constant :READABLE
  9. 26 private_constant :WRITABLE
  10. 26 def_delegator :@timers, :after
  11. 26 def_delegator :@selectables, :empty?
  12. 26 def initialize
  13. 7410 @timers = Timers.new
  14. 7410 @selectables = []
  15. 7410 @is_timer_interval = false
  16. end
  17. 26 def each(&blk)
  18. @selectables.each(&blk)
  19. end
  20. 26 def next_tick
  21. 11408458 catch(:jump_tick) do
  22. 11408458 timeout = next_timeout
  23. 11408458 if timeout && timeout.negative?
  24. @timers.fire
  25. throw(:jump_tick)
  26. end
  27. 425115 begin
  28. 11408458 select(timeout, &:call)
  29. 11408332 @timers.fire
  30. rescue TimeoutError => e
  31. @timers.fire(e)
  32. end
  33. end
  34. rescue StandardError => e
  35. 22 emit_error(e)
  36. rescue Exception # rubocop:disable Lint/RescueException
  37. 104 each_connection do |conn|
  38. 96 conn.force_reset
  39. 96 conn.disconnect
  40. end
  41. 104 raise
  42. end
  43. 26 def terminate
  44. # array may change during iteration
  45. 6962 selectables = @selectables.reject(&:inflight?)
  46. 6962 selectables.each(&:terminate)
  47. 7261 until selectables.empty?
  48. 2756 next_tick
  49. 2449 selectables &= @selectables
  50. end
  51. end
  52. 26 def find_resolver(options)
  53. 6916 res = @selectables.find do |c|
  54. 53 c.is_a?(Resolver::Resolver) && options == c.options
  55. end
  56. 6916 res.multi if res
  57. end
  58. 26 def each_connection(&block)
  59. 32637 return enum_for(__method__) unless block
  60. 16598 @selectables.each do |c|
  61. 2147 if c.is_a?(Resolver::Resolver)
  62. 215 c.each_connection(&block)
  63. else
  64. 1932 yield c
  65. end
  66. end
  67. end
  68. 26 def find_connection(request_uri, options)
  69. 8937 each_connection.find do |connection|
  70. 1206 connection.match?(request_uri, options)
  71. end
  72. end
  73. 26 def find_mergeable_connection(connection)
  74. 7102 each_connection.find do |ch|
  75. 302 ch != connection && ch.mergeable?(connection)
  76. end
  77. end
  78. # deregisters +io+ from selectables.
  79. 26 def deregister(io)
  80. 8073 @selectables.delete(io)
  81. end
  82. # register +io+.
  83. 26 def register(io)
  84. 8486 return if @selectables.include?(io)
  85. 8093 @selectables << io
  86. end
  87. 26 private
  88. 26 def select(interval, &block)
  89. # do not cause an infinite loop here.
  90. #
  91. # this may happen if timeout calculation actually triggered an error which causes
  92. # the connections to be reaped (such as the total timeout error) before #select
  93. # gets called.
  94. 11408458 return if interval.nil? && @selectables.empty?
  95. 11405726 return select_one(interval, &block) if @selectables.size == 1
  96. 314 select_many(interval, &block)
  97. end
  98. 26 def select_many(interval, &block)
  99. 314 r, w = nil
  100. # first, we group IOs based on interest type. On call to #interests however,
  101. # things might already happen, and new IOs might be registered, so we might
  102. # have to start all over again. We do this until we group all selectables
  103. 314 @selectables.delete_if do |io|
  104. 374 interests = io.interests
  105. 374 (r ||= []) << io if READABLE.include?(interests)
  106. 374 (w ||= []) << io if WRITABLE.include?(interests)
  107. 374 io.state == :closed
  108. end
  109. # TODO: what to do if there are no selectables?
  110. 314 readers, writers = IO.select(r, w, nil, interval)
  111. 314 if readers.nil? && writers.nil? && interval
  112. 135 [*r, *w].each { |io| io.handle_socket_timeout(interval) }
  113. 120 return
  114. end
  115. 179 if writers
  116. 3 readers.each do |io|
  117. 102 yield io
  118. # so that we don't yield 2 times
  119. 102 writers.delete(io)
  120. 178 end if readers
  121. 179 writers.each(&block)
  122. else
  123. readers.each(&block) if readers
  124. end
  125. end
  126. 26 def select_one(interval)
  127. 11405412 io = @selectables.first
  128. 11405412 return unless io
  129. 11405412 interests = io.interests
  130. 11405412 result = case interests
  131. 13118 when :r then io.to_io.wait_readable(interval)
  132. 9552 when :w then io.to_io.wait_writable(interval)
  133. when :rw then io.to_io.wait(interval, :read_write)
  134. 11382742 when nil then return
  135. end
  136. 22670 unless result || interval.nil?
  137. 565 io.handle_socket_timeout(interval) unless @is_timer_interval
  138. 482 return
  139. end
  140. # raise TimeoutError.new(interval, "timed out while waiting on select")
  141. 22105 yield io
  142. # rescue IOError, SystemCallError
  143. # @selectables.reject!(&:closed?)
  144. # raise unless @selectables.empty?
  145. end
  146. 26 def next_timeout
  147. 11408458 @is_timer_interval = false
  148. 11408458 timer_interval = @timers.wait_interval
  149. 11408458 connection_interval = @selectables.filter_map(&:timeout).min
  150. 11408458 return connection_interval unless timer_interval
  151. 11375251 if connection_interval.nil? || timer_interval <= connection_interval
  152. 11375214 @is_timer_interval = true
  153. 10812550 return timer_interval
  154. end
  155. 37 connection_interval
  156. end
  157. 26 def emit_error(e)
  158. 22 @selectables.each do |c|
  159. next if c.is_a?(Resolver::Resolver)
  160. c.emit(:error, e)
  161. end
  162. end
  163. end
  164. end

lib/httpx/session.rb

94.24% lines covered

243 relevant lines. 229 lines covered and 14 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. # Class implementing the APIs being used publicly.
  4. #
  5. # HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
  6. # HTTPX.plugin(..).get(..) #=> creating an intermediate HTTPX::Session with plugin, then sending the GET request
  7. 26 class Session
  8. 26 include Loggable
  9. 26 include Chainable
  10. # initializes the session with a set of +options+, which will be shared by all
  11. # requests sent from it.
  12. #
  13. # When pass a block, it'll yield itself to it, then closes after the block is evaluated.
  14. 26 def initialize(options = EMPTY_HASH, &blk)
  15. 10743 @options = self.class.default_options.merge(options)
  16. 10743 @responses = {}
  17. 10743 @persistent = @options.persistent
  18. 10743 @pool = @options.pool_class.new(@options.pool_options)
  19. 10743 @wrapped = false
  20. 10743 @closing = false
  21. 10743 wrap(&blk) if blk
  22. end
  23. # Yields itself the block, then closes it after the block is evaluated.
  24. #
  25. # session.wrap do |http|
  26. # http.get("https://wikipedia.com")
  27. # end # wikipedia connection closes here
  28. 26 def wrap
  29. 527 prev_wrapped = @wrapped
  30. 527 @wrapped = true
  31. 527 was_initialized = false
  32. 527 current_selector = get_current_selector do
  33. 527 selector = Selector.new
  34. 527 set_current_selector(selector)
  35. 527 was_initialized = true
  36. 527 selector
  37. end
  38. 51 begin
  39. 527 yield self
  40. ensure
  41. 527 unless prev_wrapped
  42. 527 if @persistent
  43. 1 deactivate(current_selector)
  44. else
  45. 526 close(current_selector)
  46. end
  47. end
  48. 527 @wrapped = prev_wrapped
  49. 527 set_current_selector(nil) if was_initialized
  50. end
  51. end
  52. # closes all the active connections from the session.
  53. #
  54. # when called directly without specifying +selector+, all available connections
  55. # will be picked up from the connection pool and closed. Connections in use
  56. # by other sessions, or same session in a different thread, will not be reaped.
  57. 26 def close(selector = Selector.new)
  58. # throw resolvers away from the pool
  59. 6962 @pool.reset_resolvers
  60. # preparing to throw away connections
  61. 16139 while (connection = @pool.pop_connection)
  62. 4288 next if connection.state == :closed
  63. 201 select_connection(connection, selector)
  64. end
  65. 782 begin
  66. 6962 @closing = true
  67. 6962 selector.terminate
  68. ensure
  69. 6962 @closing = false
  70. end
  71. end
  72. # performs one, or multple requests; it accepts:
  73. #
  74. # 1. one or multiple HTTPX::Request objects;
  75. # 2. an HTTP verb, then a sequence of URIs or URI/options tuples;
  76. # 3. one or multiple HTTP verb / uri / (optional) options tuples;
  77. #
  78. # when present, the set of +options+ kwargs is applied to all of the
  79. # sent requests.
  80. #
  81. # respectively returns a single HTTPX::Response response, or all of them in an Array, in the same order.
  82. #
  83. # resp1 = session.request(req1)
  84. # resp1, resp2 = session.request(req1, req2)
  85. # resp1 = session.request("GET", "https://server.org/a")
  86. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"])
  87. # resp1, resp2 = session.request(["GET", "https://server.org/a"], ["GET", "https://server.org/b"])
  88. # resp1 = session.request("POST", "https://server.org/a", form: { "foo" => "bar" })
  89. # resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
  90. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
  91. #
  92. 26 def request(*args, **params)
  93. 7203 raise ArgumentError, "must perform at least one request" if args.empty?
  94. 7203 requests = args.first.is_a?(Request) ? args : build_requests(*args, params)
  95. 7166 responses = send_requests(*requests)
  96. 7048 return responses.first if responses.size == 1
  97. 191 responses
  98. end
  99. # returns a HTTP::Request instance built from the HTTP +verb+, the request +uri+, and
  100. # the optional set of request-specific +options+. This request **must** be sent through
  101. # the same session it was built from.
  102. #
  103. # req = session.build_request("GET", "https://server.com")
  104. # resp = session.request(req)
  105. 26 def build_request(verb, uri, params = EMPTY_HASH, options = @options)
  106. 8581 rklass = options.request_class
  107. 8581 request = rklass.new(verb, uri, options, params)
  108. 8544 request.persistent = @persistent
  109. 8544 set_request_callbacks(request)
  110. 8544 request
  111. end
  112. 26 def select_connection(connection, selector)
  113. 8486 pin_connection(connection, selector)
  114. 8486 selector.register(connection)
  115. end
  116. 26 def pin_connection(connection, selector)
  117. 8495 connection.current_session = self
  118. 8495 connection.current_selector = selector
  119. end
  120. 26 alias_method :select_resolver, :select_connection
  121. 26 def deselect_connection(connection, selector, cloned = false)
  122. 7726 selector.deregister(connection)
  123. # when connections coalesce
  124. 7726 return if connection.state == :idle
  125. 7689 return if cloned
  126. 7681 return if @closing && connection.state == :closed
  127. 7673 @pool.checkin_connection(connection)
  128. end
  129. 26 def deselect_resolver(resolver, selector)
  130. 347 selector.deregister(resolver)
  131. 347 return if @closing && resolver.closed?
  132. 347 @pool.checkin_resolver(resolver)
  133. end
  134. 26 def try_clone_connection(connection, selector, family)
  135. 445 connection.family ||= family
  136. 445 return connection if connection.family == family
  137. new_connection = connection.class.new(connection.origin, connection.options)
  138. new_connection.family = family
  139. connection.sibling = new_connection
  140. do_init_connection(new_connection, selector)
  141. new_connection
  142. end
  143. # returns the HTTPX::Connection through which the +request+ should be sent through.
  144. 26 def find_connection(request_uri, selector, options)
  145. 8937 if (connection = selector.find_connection(request_uri, options))
  146. 1087 return connection
  147. end
  148. 7785 connection = @pool.checkout_connection(request_uri, options)
  149. 6897 case connection.state
  150. when :idle
  151. 7082 do_init_connection(connection, selector)
  152. when :open
  153. 66 if options.io
  154. 66 select_connection(connection, selector)
  155. else
  156. pin_connection(connection, selector)
  157. end
  158. when :closed
  159. 604 connection.idling
  160. 604 select_connection(connection, selector)
  161. when :closing
  162. connection.once(:close) do
  163. connection.idling
  164. select_connection(connection, selector)
  165. end
  166. else
  167. 9 pin_connection(connection, selector)
  168. end
  169. 7697 connection
  170. end
  171. 26 private
  172. 26 def deactivate(selector)
  173. 455 selector.each_connection do |connection|
  174. 344 connection.deactivate
  175. 344 deselect_connection(connection, selector) if connection.state == :inactive
  176. end
  177. end
  178. # callback executed when a response for a given request has been received.
  179. 26 def on_response(request, response)
  180. 7927 @responses[request] = response
  181. end
  182. # callback executed when an HTTP/2 promise frame has been received.
  183. 26 def on_promise(_, stream)
  184. 8 log(level: 2) { "#{stream.id}: refusing stream!" }
  185. 8 stream.refuse
  186. end
  187. # returns the corresponding HTTP::Response to the given +request+ if it has been received.
  188. 26 def fetch_response(request, _selector, _options)
  189. 11413661 @responses.delete(request)
  190. end
  191. # sends the +request+ to the corresponding HTTPX::Connection
  192. 26 def send_request(request, selector, options = request.options)
  193. 1843 error = begin
  194. 8863 catch(:resolve_error) do
  195. 8863 connection = find_connection(request.uri, selector, options)
  196. 8751 connection.send(request)
  197. end
  198. rescue StandardError => e
  199. 32 e
  200. end
  201. 8857 return unless error && error.is_a?(Exception)
  202. 112 raise error unless error.is_a?(Error)
  203. 112 request.emit(:response, ErrorResponse.new(request, error))
  204. end
  205. # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
  206. 26 def build_requests(*args, params)
  207. 6617 requests = if args.size == 1
  208. 78 reqs = args.first
  209. 78 reqs.map do |verb, uri, ps = EMPTY_HASH|
  210. 156 request_params = params
  211. 156 request_params = request_params.merge(ps) unless ps.empty?
  212. 156 build_request(verb, uri, request_params)
  213. end
  214. else
  215. 6539 verb, uris = args
  216. 6539 if uris.respond_to?(:each)
  217. 6299 uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
  218. 7041 request_params = params
  219. 7041 request_params = request_params.merge(ps) unless ps.empty?
  220. 7041 build_request(verb, uri, request_params)
  221. end
  222. else
  223. 240 [build_request(verb, uris, params)]
  224. end
  225. end
  226. 6580 raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
  227. 6580 requests
  228. end
  229. 26 def set_request_callbacks(request)
  230. 8544 request.on(:response, &method(:on_response).curry(2)[request])
  231. 8544 request.on(:promise, &method(:on_promise))
  232. end
  233. 26 def do_init_connection(connection, selector)
  234. 7082 resolve_connection(connection, selector) unless connection.family
  235. end
  236. # sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
  237. 26 def send_requests(*requests)
  238. 13869 selector = get_current_selector { Selector.new }
  239. 798 begin
  240. 7264 _send_requests(requests, selector)
  241. 7258 receive_requests(requests, selector)
  242. ensure
  243. 7264 unless @wrapped
  244. 6612 if @persistent
  245. 454 deactivate(selector)
  246. else
  247. 6158 close(selector)
  248. end
  249. end
  250. end
  251. end
  252. # sends an array of HTTPX::Request objects
  253. 26 def _send_requests(requests, selector)
  254. 7264 requests.each do |request|
  255. 8069 send_request(request, selector)
  256. end
  257. end
  258. # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
  259. 26 def receive_requests(requests, selector)
  260. # @type var responses: Array[response]
  261. 7258 responses = []
  262. # guarantee ordered responses
  263. 7258 loop do
  264. 8071 request = requests.first
  265. 8071 return responses unless request
  266. 11976637 catch(:coalesced) { selector.next_tick } until (response = fetch_response(request, selector, request.options))
  267. 7959 request.emit(:complete, response)
  268. 7959 responses << response
  269. 7959 requests.shift
  270. 7959 break if requests.empty?
  271. 813 next unless selector.empty?
  272. # in some cases, the pool of connections might have been drained because there was some
  273. # handshake error, and the error responses have already been emitted, but there was no
  274. # opportunity to traverse the requests, hence we're returning only a fraction of the errors
  275. # we were supposed to. This effectively fetches the existing responses and return them.
  276. while (request = requests.shift)
  277. response = fetch_response(request, selector, request.options)
  278. request.emit(:complete, response) if response
  279. responses << response
  280. end
  281. break
  282. end
  283. 7146 responses
  284. end
  285. 26 def resolve_connection(connection, selector)
  286. 7106 if connection.addresses || connection.open?
  287. #
  288. # there are two cases in which we want to activate initialization of
  289. # connection immediately:
  290. #
  291. # 1. when the connection already has addresses, i.e. it doesn't need to
  292. # resolve a name (not the same as name being an IP, yet)
  293. # 2. when the connection is initialized with an external already open IO.
  294. #
  295. 190 on_resolver_connection(connection, selector)
  296. 188 return
  297. end
  298. 6916 resolver = find_resolver_for(connection, selector)
  299. 6916 resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
  300. end
  301. 26 def on_resolver_connection(connection, selector)
  302. 7102 from_pool = false
  303. 7102 found_connection = selector.find_mergeable_connection(connection) || begin
  304. 7075 from_pool = true
  305. 7075 @pool.checkout_mergeable_connection(connection)
  306. end
  307. 7102 return select_connection(connection, selector) unless found_connection
  308. 27 if found_connection.open?
  309. 26 coalesce_connections(found_connection, connection, selector, from_pool)
  310. else
  311. 1 found_connection.once(:open) do
  312. 1 coalesce_connections(found_connection, connection, selector, from_pool)
  313. end
  314. end
  315. end
  316. 26 def on_resolver_close(resolver, selector)
  317. 347 return if resolver.closed?
  318. 347 deselect_resolver(resolver, selector)
  319. 347 resolver.close unless resolver.closed?
  320. end
  321. 26 def find_resolver_for(connection, selector)
  322. 6916 resolver = selector.find_resolver(connection.options)
  323. 6916 unless resolver
  324. 6914 resolver = @pool.checkout_resolver(connection.options)
  325. 6914 resolver.current_session = self
  326. 6914 resolver.current_selector = selector
  327. end
  328. 6916 resolver
  329. end
  330. # coalesces +conn2+ into +conn1+. if +conn1+ was loaded from the connection pool
  331. # (it is known via +from_pool+), then it adds its to the +selector+.
  332. 26 def coalesce_connections(conn1, conn2, selector, from_pool)
  333. 27 unless conn1.coalescable?(conn2)
  334. 14 select_connection(conn2, selector)
  335. 14 @pool.checkin_connection(conn1) if from_pool
  336. 14 return false
  337. end
  338. 13 conn2.coalesced_connection = conn1
  339. 13 select_connection(conn1, selector) if from_pool
  340. 13 deselect_connection(conn2, selector)
  341. 13 true
  342. end
  343. 26 def get_current_selector
  344. 7791 selector_store[self] || (yield if block_given?)
  345. end
  346. 26 def set_current_selector(selector)
  347. 1502 if selector
  348. 883 selector_store[self] = selector
  349. else
  350. 527 selector_store.delete(self)
  351. end
  352. end
  353. 26 def selector_store
  354. 9293 th_current = Thread.current
  355. 9293 th_current.thread_variable_get(:httpx_persistent_selector_store) || begin
  356. 112 {}.compare_by_identity.tap do |store|
  357. 112 th_current.thread_variable_set(:httpx_persistent_selector_store, store)
  358. end
  359. end
  360. end
  361. 26 @default_options = Options.new
  362. 26 @default_options.freeze
  363. 26 @plugins = []
  364. 26 class << self
  365. 26 attr_reader :default_options
  366. 26 def inherited(klass)
  367. 5379 super
  368. 5379 klass.instance_variable_set(:@default_options, @default_options)
  369. 5379 klass.instance_variable_set(:@plugins, @plugins.dup)
  370. 5379 klass.instance_variable_set(:@callbacks, @callbacks.dup)
  371. end
  372. # returns a new HTTPX::Session instance, with the plugin pointed by +pl+ loaded.
  373. #
  374. # session_with_retries = session.plugin(:retries)
  375. # session_with_custom = session.plugin(CustomPlugin)
  376. #
  377. 26 def plugin(pl, options = nil, &block)
  378. # raise Error, "Cannot add a plugin to a frozen config" if frozen?
  379. 7324 pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
  380. 7324 if !@plugins.include?(pl)
  381. 7094 @plugins << pl
  382. 7094 pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
  383. 7094 @default_options = @default_options.dup
  384. 7094 include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
  385. 7094 extend(pl::ClassMethods) if defined?(pl::ClassMethods)
  386. 7094 opts = @default_options
  387. 7094 opts.extend_with_plugin_classes(pl)
  388. 7094 if defined?(pl::OptionsMethods)
  389. 2916 (pl::OptionsMethods.instance_methods - Object.instance_methods).each do |meth|
  390. 8474 opts.options_class.method_added(meth)
  391. end
  392. 2916 @default_options = opts.options_class.new(opts)
  393. end
  394. 7094 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  395. 7094 @default_options = @default_options.merge(options) if options
  396. 7094 pl.configure(self, &block) if pl.respond_to?(:configure)
  397. 7094 @default_options.freeze
  398. 229 elsif options
  399. # this can happen when two plugins are loaded, an one of them calls the other under the hood,
  400. # albeit changing some default.
  401. 16 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  402. 16 @default_options = @default_options.merge(options) if options
  403. 16 @default_options.freeze
  404. end
  405. 7324 self
  406. end
  407. end
  408. end
  409. # session may be overridden by certain adapters.
  410. 26 S = Session
  411. end

lib/httpx/session_extensions.rb

100.0% lines covered

14 relevant lines. 14 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 unless ENV.keys.grep(/\Ahttps?_proxy\z/i).empty?
  4. 1 proxy_session = plugin(:proxy)
  5. 1 remove_const(:Session)
  6. 1 const_set(:Session, proxy_session.class)
  7. # redefine the default options static var, which needs to
  8. # refresh options_class
  9. 1 options = proxy_session.class.default_options.to_hash
  10. 1 original_verbosity = $VERBOSE
  11. 1 $VERBOSE = nil
  12. 1 const_set(:Options, proxy_session.class.default_options.options_class)
  13. 1 options[:options_class] = Class.new(options[:options_class])
  14. 1 options.freeze
  15. 1 Options.send(:const_set, :DEFAULT_OPTIONS, options)
  16. 1 Session.instance_variable_set(:@default_options, Options.new(options))
  17. 1 $VERBOSE = original_verbosity
  18. end
  19. skipped # :nocov:
  20. skipped if Session.default_options.debug_level > 2
  21. skipped proxy_session = plugin(:internal_telemetry)
  22. skipped remove_const(:Session)
  23. skipped const_set(:Session, proxy_session.class)
  24. skipped end
  25. skipped # :nocov:
  26. end

lib/httpx/timers.rb

95.38% lines covered

65 relevant lines. 62 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 class Timers
  4. 26 def initialize
  5. 7410 @intervals = []
  6. end
  7. 26 def after(interval_in_secs, cb = nil, &blk)
  8. 37970 return unless interval_in_secs
  9. 37970 callback = cb || blk
  10. # I'm assuming here that most requests will have the same
  11. # request timeout, as in most cases they share common set of
  12. # options. A user setting different request timeouts for 100s of
  13. # requests will already have a hard time dealing with that.
  14. 67923 unless (interval = @intervals.find { |t| t.interval == interval_in_secs })
  15. 9267 interval = Interval.new(interval_in_secs)
  16. 18451 interval.on_empty { @intervals.delete(interval) }
  17. 9267 @intervals << interval
  18. 9267 @intervals.sort!
  19. end
  20. 37970 interval << callback
  21. 37970 @next_interval_at = nil
  22. 37970 Timer.new(interval, callback)
  23. end
  24. 26 def wait_interval
  25. 11408458 return if @intervals.empty?
  26. 11375251 @next_interval_at = Utils.now
  27. 11375251 @intervals.first.interval
  28. end
  29. 26 def fire(error = nil)
  30. 11408332 raise error if error && error.timeout != @intervals.first
  31. 11408332 return if @intervals.empty? || !@next_interval_at
  32. 11368360 elapsed_time = Utils.elapsed_time(@next_interval_at)
  33. 22736736 @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
  34. 11368360 @next_interval_at = nil if @intervals.empty?
  35. end
  36. 26 class Timer
  37. 26 def initialize(interval, callback)
  38. 37970 @interval = interval
  39. 37970 @callback = callback
  40. end
  41. 26 def cancel
  42. 55861 @interval.delete(@callback)
  43. end
  44. end
  45. 26 class Interval
  46. 26 include Comparable
  47. 26 attr_reader :interval
  48. 26 def initialize(interval)
  49. 9267 @interval = interval
  50. 9267 @callbacks = []
  51. 9267 @on_empty = nil
  52. end
  53. 26 def on_empty(&blk)
  54. 9267 @on_empty = blk
  55. end
  56. 26 def cancel
  57. @on_empty.call
  58. end
  59. 26 def <=>(other)
  60. 722 @interval <=> other.interval
  61. end
  62. 26 def ==(other)
  63. 819 return @interval == other if other.is_a?(Numeric)
  64. 819 @interval == other.to_f # rubocop:disable Lint/FloatComparison
  65. end
  66. 26 def to_f
  67. 819 Float(@interval)
  68. end
  69. 26 def <<(callback)
  70. 37970 @callbacks << callback
  71. end
  72. 26 def delete(callback)
  73. 55861 @callbacks.delete(callback)
  74. 55861 @on_empty.call if @callbacks.empty?
  75. end
  76. 26 def no_callbacks?
  77. @callbacks.empty?
  78. end
  79. 26 def elapsed?
  80. @interval <= 0
  81. end
  82. 26 def elapse(elapsed)
  83. 10806240 @interval -= elapsed
  84. 11368376 if @interval <= 0
  85. 648 cb = @callbacks.dup
  86. 648 cb.each(&:call)
  87. end
  88. 11368376 @interval
  89. end
  90. end
  91. 26 private_constant :Interval
  92. end
  93. end

lib/httpx/transcoder.rb

100.0% lines covered

52 relevant lines. 52 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Transcoder
  4. 26 module_function
  5. 26 def normalize_keys(key, value, cond = nil, &block)
  6. 3415 if cond && cond.call(value)
  7. 1065 block.call(key.to_s, value)
  8. 2349 elsif value.respond_to?(:to_ary)
  9. 454 if value.empty?
  10. 128 block.call("#{key}[]")
  11. else
  12. 326 value.to_ary.each do |element|
  13. 524 normalize_keys("#{key}[]", element, cond, &block)
  14. end
  15. end
  16. 1895 elsif value.respond_to?(:to_hash)
  17. 512 value.to_hash.each do |child_key, child_value|
  18. 512 normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
  19. end
  20. else
  21. 1384 block.call(key.to_s, value)
  22. end
  23. end
  24. # based on https://github.com/rack/rack/blob/d15dd728440710cfc35ed155d66a98dc2c07ae42/lib/rack/query_parser.rb#L82
  25. 26 def normalize_query(params, name, v, depth)
  26. 184 raise Error, "params depth surpasses what's supported" if depth <= 0
  27. 184 name =~ /\A[\[\]]*([^\[\]]+)\]*/
  28. 184 k = Regexp.last_match(1) || ""
  29. 184 after = Regexp.last_match ? Regexp.last_match.post_match : ""
  30. 184 if k.empty?
  31. 16 return Array(v) if !v.empty? && name == "[]"
  32. 7 return
  33. end
  34. 147 case after
  35. when ""
  36. 49 params[k] = v
  37. when "["
  38. 7 params[name] = v
  39. when "[]"
  40. 16 params[k] ||= []
  41. 16 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  42. 16 params[k] << v
  43. when /^\[\]\[([^\[\]]+)\]$/, /^\[\](.+)$/
  44. 32 child_key = Regexp.last_match(1)
  45. 32 params[k] ||= []
  46. 32 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  47. 32 if params[k].last.is_a?(Hash) && !params_hash_has_key?(params[k].last, child_key)
  48. 8 normalize_query(params[k].last, child_key, v, depth - 1)
  49. else
  50. 24 params[k] << normalize_query({}, child_key, v, depth - 1)
  51. end
  52. else
  53. 56 params[k] ||= {}
  54. 56 raise Error, "expected Hash (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Hash)
  55. 49 params[k] = normalize_query(params[k], after, v, depth - 1)
  56. end
  57. 168 params
  58. end
  59. 26 def params_hash_has_key?(hash, key)
  60. 16 return false if key.include?("[]")
  61. 16 key.split(/[\[\]]+/).inject(hash) do |h, part|
  62. 16 next h if part == ""
  63. 16 return false unless h.is_a?(Hash) && h.key?(part)
  64. 8 h[part]
  65. end
  66. 8 true
  67. end
  68. end
  69. end
  70. 26 require "httpx/transcoder/body"
  71. 26 require "httpx/transcoder/form"
  72. 26 require "httpx/transcoder/json"
  73. 26 require "httpx/transcoder/chunker"
  74. 26 require "httpx/transcoder/deflate"
  75. 26 require "httpx/transcoder/gzip"

lib/httpx/transcoder/body.rb

100.0% lines covered

26 relevant lines. 26 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "delegate"
  3. 26 module HTTPX::Transcoder
  4. 26 module Body
  5. 26 class Error < HTTPX::Error; end
  6. 26 module_function
  7. 26 class Encoder < SimpleDelegator
  8. 26 def initialize(body)
  9. 1408 body = body.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && body.is_a?(Pathname)
  10. 1408 @body = body
  11. 1408 super(body)
  12. end
  13. 26 def bytesize
  14. 5568 if @body.respond_to?(:bytesize)
  15. 2486 @body.bytesize
  16. 3081 elsif @body.respond_to?(:to_ary)
  17. 926 @body.sum(&:bytesize)
  18. 2155 elsif @body.respond_to?(:size)
  19. 1500 @body.size || Float::INFINITY
  20. 655 elsif @body.respond_to?(:length)
  21. 360 @body.length || Float::INFINITY
  22. 295 elsif @body.respond_to?(:each)
  23. 288 Float::INFINITY
  24. else
  25. 8 raise Error, "cannot determine size of body: #{@body.inspect}"
  26. end
  27. end
  28. 26 def content_type
  29. 1346 "application/octet-stream"
  30. end
  31. end
  32. 26 def encode(body)
  33. 1408 Encoder.new(body)
  34. end
  35. end
  36. end

lib/httpx/transcoder/chunker.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "forwardable"
  3. 26 module HTTPX::Transcoder
  4. 26 module Chunker
  5. 26 class Error < HTTPX::Error; end
  6. 26 CRLF = "\r\n".b
  7. 26 class Encoder
  8. 26 extend Forwardable
  9. 26 def initialize(body)
  10. 96 @raw = body
  11. end
  12. 26 def each
  13. 96 return enum_for(__method__) unless block_given?
  14. 96 @raw.each do |chunk|
  15. 448 yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
  16. end
  17. 96 yield "0#{CRLF}"
  18. end
  19. 26 def respond_to_missing?(meth, *args)
  20. 108 @raw.respond_to?(meth, *args) || super
  21. end
  22. end
  23. 26 class Decoder
  24. 26 extend Forwardable
  25. 26 def_delegator :@buffer, :empty?
  26. 26 def_delegator :@buffer, :<<
  27. 26 def_delegator :@buffer, :clear
  28. 26 def initialize(buffer, trailers = false)
  29. 114 @buffer = buffer
  30. 114 @chunk_buffer = "".b
  31. 114 @finished = false
  32. 114 @state = :length
  33. 114 @trailers = trailers
  34. end
  35. 26 def to_s
  36. 106 @buffer
  37. end
  38. 26 def each
  39. 203 loop do
  40. 1043 case @state
  41. when :length
  42. 340 index = @buffer.index(CRLF)
  43. 340 return unless index && index.positive?
  44. # Read hex-length
  45. 340 hexlen = @buffer.byteslice(0, index)
  46. 340 @buffer = @buffer.byteslice(index..-1) || "".b
  47. 340 hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
  48. 340 @chunk_length = hexlen.hex
  49. # check if is last chunk
  50. 340 @finished = @chunk_length.zero?
  51. 340 nextstate(:crlf)
  52. when :crlf
  53. 566 crlf_size = @finished && !@trailers ? 4 : 2
  54. # consume CRLF
  55. 566 return if @buffer.bytesize < crlf_size
  56. 566 raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
  57. 566 @buffer = @buffer.byteslice(crlf_size..-1)
  58. 566 if @chunk_length.nil?
  59. 226 nextstate(:length)
  60. else
  61. 340 return if @finished
  62. 234 nextstate(:data)
  63. end
  64. when :data
  65. 284 chunk = @buffer.byteslice(0, @chunk_length)
  66. 284 @buffer = @buffer.byteslice(@chunk_length..-1) || "".b
  67. 284 @chunk_buffer << chunk
  68. 249 @chunk_length -= chunk.bytesize
  69. 284 if @chunk_length.zero?
  70. 234 yield @chunk_buffer unless @chunk_buffer.empty?
  71. 226 @chunk_buffer.clear
  72. 226 @chunk_length = nil
  73. 226 nextstate(:crlf)
  74. end
  75. end
  76. 1076 break if @buffer.empty?
  77. end
  78. end
  79. 26 def finished?
  80. 195 @finished
  81. end
  82. 26 private
  83. 26 def nextstate(state)
  84. 1026 @state = state
  85. end
  86. end
  87. 26 module_function
  88. 26 def encode(chunks)
  89. 96 Encoder.new(chunks)
  90. end
  91. end
  92. end

lib/httpx/transcoder/deflate.rb

100.0% lines covered

20 relevant lines. 20 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "zlib"
  3. 26 require_relative "utils/deflater"
  4. 26 module HTTPX
  5. 26 module Transcoder
  6. 26 module Deflate
  7. 26 class Deflater < Transcoder::Deflater
  8. 26 def deflate(chunk)
  9. 72 @deflater ||= Zlib::Deflate.new
  10. 72 if chunk.nil?
  11. 48 unless @deflater.closed?
  12. 24 last = @deflater.finish
  13. 24 @deflater.close
  14. 24 last.empty? ? nil : last
  15. end
  16. else
  17. 24 @deflater.deflate(chunk)
  18. end
  19. end
  20. end
  21. 26 module_function
  22. 26 def encode(body)
  23. 24 Deflater.new(body)
  24. end
  25. 26 def decode(response, bytesize: nil)
  26. 16 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  27. 16 GZIP::Inflater.new(bytesize)
  28. end
  29. end
  30. end
  31. end

lib/httpx/transcoder/form.rb

100.0% lines covered

42 relevant lines. 42 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "forwardable"
  3. 26 require "uri"
  4. 26 require_relative "multipart"
  5. 26 module HTTPX
  6. 26 module Transcoder
  7. 26 module Form
  8. 26 module_function
  9. 26 PARAM_DEPTH_LIMIT = 32
  10. 26 class Encoder
  11. 26 extend Forwardable
  12. 26 def_delegator :@raw, :to_s
  13. 26 def_delegator :@raw, :to_str
  14. 26 def_delegator :@raw, :bytesize
  15. 26 def_delegator :@raw, :==
  16. 26 def initialize(form)
  17. 706 @raw = form.each_with_object("".b) do |(key, val), buf|
  18. 1186 HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
  19. 1384 buf << "&" unless buf.empty?
  20. 1384 buf << URI.encode_www_form_component(k)
  21. 1384 buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
  22. end
  23. end
  24. end
  25. 26 def content_type
  26. 554 "application/x-www-form-urlencoded"
  27. end
  28. end
  29. 26 module Decoder
  30. 26 module_function
  31. 26 def call(response, *)
  32. 40 URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
  33. 96 HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
  34. end
  35. end
  36. end
  37. 26 def encode(form)
  38. 1657 if multipart?(form)
  39. 951 Multipart::Encoder.new(form)
  40. else
  41. 706 Encoder.new(form)
  42. end
  43. end
  44. 26 def decode(response)
  45. 64 content_type = response.content_type.mime_type
  46. 56 case content_type
  47. when "application/x-www-form-urlencoded"
  48. 40 Decoder
  49. when "multipart/form-data"
  50. 16 Multipart::Decoder.new(response)
  51. else
  52. 8 raise Error, "invalid form mime type (#{content_type})"
  53. end
  54. end
  55. 26 def multipart?(data)
  56. 1657 data.any? do |_, v|
  57. 2201 Multipart::MULTIPART_VALUE_COND.call(v) ||
  58. 1698 (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
  59. 2082 (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
  60. end
  61. end
  62. end
  63. end
  64. end

lib/httpx/transcoder/gzip.rb

100.0% lines covered

40 relevant lines. 40 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "zlib"
  3. 26 module HTTPX
  4. 26 module Transcoder
  5. 26 module GZIP
  6. 26 class Deflater < Transcoder::Deflater
  7. 26 def initialize(body)
  8. 48 @compressed_chunk = "".b
  9. 48 super
  10. end
  11. 26 def deflate(chunk)
  12. 96 @deflater ||= Zlib::GzipWriter.new(self)
  13. 96 if chunk.nil?
  14. 48 unless @deflater.closed?
  15. 48 @deflater.flush
  16. 48 @deflater.close
  17. 48 compressed_chunk
  18. end
  19. else
  20. 48 @deflater.write(chunk)
  21. 48 compressed_chunk
  22. end
  23. end
  24. 26 private
  25. 26 def write(chunk)
  26. 144 @compressed_chunk << chunk
  27. end
  28. 26 def compressed_chunk
  29. 96 @compressed_chunk.dup
  30. ensure
  31. 96 @compressed_chunk.clear
  32. end
  33. end
  34. 26 class Inflater
  35. 26 def initialize(bytesize)
  36. 171 @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
  37. 171 @bytesize = bytesize
  38. end
  39. 26 def call(chunk)
  40. 411 buffer = @inflater.inflate(chunk)
  41. 365 @bytesize -= chunk.bytesize
  42. 411 if @bytesize <= 0
  43. 108 buffer << @inflater.finish
  44. 108 @inflater.close
  45. end
  46. 411 buffer
  47. end
  48. end
  49. 26 module_function
  50. 26 def encode(body)
  51. 48 Deflater.new(body)
  52. end
  53. 26 def decode(response, bytesize: nil)
  54. 155 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  55. 155 Inflater.new(bytesize)
  56. end
  57. end
  58. end
  59. end

lib/httpx/transcoder/json.rb

100.0% lines covered

33 relevant lines. 33 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "forwardable"
  3. 26 module HTTPX::Transcoder
  4. 26 module JSON
  5. 26 module_function
  6. 26 JSON_REGEX = %r{
  7. \b
  8. application/
  9. # optional vendor specific type
  10. (?:
  11. # token as per https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6
  12. [!#$%&'*+\-.^_`|~0-9a-z]+
  13. # literal plus sign
  14. \+
  15. )?
  16. json
  17. \b
  18. }ix.freeze
  19. 26 class Encoder
  20. 26 extend Forwardable
  21. 26 def_delegator :@raw, :to_s
  22. 26 def_delegator :@raw, :bytesize
  23. 26 def_delegator :@raw, :==
  24. 26 def initialize(json)
  25. 83 @raw = JSON.json_dump(json)
  26. 83 @charset = @raw.encoding.name.downcase
  27. end
  28. 26 def content_type
  29. 83 "application/json; charset=#{@charset}"
  30. end
  31. end
  32. 26 def encode(json)
  33. 83 Encoder.new(json)
  34. end
  35. 26 def decode(response)
  36. 129 content_type = response.content_type.mime_type
  37. 129 raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
  38. 113 method(:json_load)
  39. end
  40. # rubocop:disable Style/SingleLineMethods
  41. 26 if defined?(MultiJson)
  42. 4 def json_load(*args); MultiJson.load(*args); end
  43. 2 def json_dump(*args); MultiJson.dump(*args); end
  44. 24 elsif defined?(Oj)
  45. 4 def json_load(response, *args); Oj.load(response.to_s, *args); end
  46. 2 def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
  47. 23 elsif defined?(Yajl)
  48. 4 def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
  49. 2 def json_dump(*args); Yajl::Encoder.encode(*args); end
  50. else
  51. 23 require "json"
  52. 114 def json_load(*args); ::JSON.parse(*args); end
  53. 93 def json_dump(*args); ::JSON.dump(*args); end
  54. end
  55. # rubocop:enable Style/SingleLineMethods
  56. end
  57. end

lib/httpx/transcoder/multipart.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require_relative "multipart/encoder"
  3. 26 require_relative "multipart/decoder"
  4. 26 require_relative "multipart/part"
  5. 26 require_relative "multipart/mime_type_detector"
  6. 26 module HTTPX::Transcoder
  7. 26 module Multipart
  8. 26 MULTIPART_VALUE_COND = lambda do |value|
  9. 4878 value.respond_to?(:read) ||
  10. 3506 (value.respond_to?(:to_hash) &&
  11. value.key?(:body) &&
  12. 644 (value.key?(:filename) || value.key?(:content_type)))
  13. end
  14. end
  15. end

lib/httpx/transcoder/multipart/decoder.rb

93.98% lines covered

83 relevant lines. 78 lines covered and 5 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "tempfile"
  3. 26 require "delegate"
  4. 26 module HTTPX
  5. 26 module Transcoder
  6. 26 module Multipart
  7. 26 class FilePart < SimpleDelegator
  8. 26 attr_reader :original_filename, :content_type
  9. 26 def initialize(filename, content_type)
  10. 32 @original_filename = filename
  11. 32 @content_type = content_type
  12. 32 @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  13. 32 super(@file)
  14. end
  15. end
  16. 26 class Decoder
  17. 26 include HTTPX::Utils
  18. 26 CRLF = "\r\n"
  19. 26 BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
  20. 26 MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
  21. 26 MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
  22. 26 MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
  23. 26 WINDOW_SIZE = 2 << 14
  24. 26 def initialize(response)
  25. 2 @boundary = begin
  26. 16 m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
  27. 16 raise Error, "no boundary declared in content-type header" unless m
  28. 16 m.strip
  29. end
  30. 16 @buffer = "".b
  31. 16 @parts = {}
  32. 16 @intermediate_boundary = "--#{@boundary}"
  33. 16 @state = :idle
  34. end
  35. 26 def call(response, *)
  36. 16 response.body.each do |chunk|
  37. 16 @buffer << chunk
  38. 16 parse
  39. end
  40. 16 raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
  41. 16 @parts
  42. end
  43. 26 private
  44. 26 def parse
  45. 14 case @state
  46. when :idle
  47. 16 raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
  48. 16 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
  49. 16 @state = :part_header
  50. when :part_header
  51. 48 idx = @buffer.index("#{CRLF}#{CRLF}")
  52. # raise Error, "couldn't parse part headers" unless idx
  53. 48 return unless idx
  54. 48 head = @buffer.byteslice(0..idx + 4 - 1)
  55. 48 @buffer = @buffer.byteslice(head.bytesize..-1)
  56. 48 content_type = head[MULTIPART_CONTENT_TYPE, 1]
  57. 84 if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
  58. 48 name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
  59. 48 name.gsub!(/\\(.)/, "\\1")
  60. 12 name
  61. else
  62. name = head[MULTIPART_CONTENT_ID, 1]
  63. end
  64. 48 filename = HTTPX::Utils.get_filename(head)
  65. 48 name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
  66. 48 @current = name
  67. 42 @parts[name] = if filename
  68. 32 FilePart.new(filename, content_type)
  69. else
  70. 16 "".b
  71. end
  72. 48 @state = :part_body
  73. when :part_body
  74. 48 part = @parts[@current]
  75. 48 body_separator = if part.is_a?(FilePart)
  76. 28 "#{CRLF}#{CRLF}"
  77. else
  78. 16 CRLF
  79. end
  80. 48 idx = @buffer.index(body_separator)
  81. 48 if idx
  82. 48 payload = @buffer.byteslice(0..idx - 1)
  83. 48 @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
  84. 48 part << payload
  85. 48 part.rewind if part.respond_to?(:rewind)
  86. 48 @state = :parse_boundary
  87. else
  88. part << @buffer
  89. @buffer.clear
  90. end
  91. when :parse_boundary
  92. 48 raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
  93. 48 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
  94. 48 if @buffer == "--"
  95. 16 @buffer.clear
  96. 16 @state = :done
  97. 16 return
  98. 31 elsif @buffer.start_with?(CRLF)
  99. 32 @buffer = @buffer.byteslice(2..-1)
  100. 32 @state = :part_header
  101. else
  102. return
  103. end
  104. when :done
  105. raise Error, "parsing should have been over by now"
  106. 20 end until @buffer.empty?
  107. end
  108. end
  109. end
  110. end
  111. end

lib/httpx/transcoder/multipart/encoder.rb

100.0% lines covered

70 relevant lines. 70 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Transcoder::Multipart
  4. 26 class Encoder
  5. 26 attr_reader :bytesize
  6. 26 def initialize(form)
  7. 951 @boundary = ("-" * 21) << SecureRandom.hex(21)
  8. 951 @part_index = 0
  9. 951 @buffer = "".b
  10. 951 @form = form
  11. 951 @bytesize = 0
  12. 951 @parts = to_parts(form)
  13. end
  14. 26 def content_type
  15. 951 "multipart/form-data; boundary=#{@boundary}"
  16. end
  17. 26 def to_s
  18. 18 read
  19. ensure
  20. 18 rewind
  21. end
  22. 26 def read(length = nil, outbuf = nil)
  23. 3518 data = String(outbuf).clear.force_encoding(Encoding::BINARY) if outbuf
  24. 3518 data ||= "".b
  25. 3518 read_chunks(data, length)
  26. 3518 data unless length && data.empty?
  27. end
  28. 26 def rewind
  29. 50 form = @form.each_with_object([]) do |(key, val), aux|
  30. 50 if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
  31. 50 val = val.reopen(val.path, File::RDONLY)
  32. end
  33. 50 val.rewind if val.respond_to?(:rewind)
  34. 50 aux << [key, val]
  35. end
  36. 50 @form = form
  37. 50 @bytesize = 0
  38. 50 @parts = to_parts(form)
  39. 50 @part_index = 0
  40. end
  41. 26 private
  42. 26 def to_parts(form)
  43. 1001 params = form.each_with_object([]) do |(key, val), aux|
  44. 1193 Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
  45. 1193 next if v.nil?
  46. 1193 value, content_type, filename = Part.call(v)
  47. 1193 header = header_part(k, content_type, filename)
  48. 1049 @bytesize += header.size
  49. 1193 aux << header
  50. 1049 @bytesize += value.size
  51. 1193 aux << value
  52. 1193 delimiter = StringIO.new("\r\n")
  53. 1049 @bytesize += delimiter.size
  54. 1193 aux << delimiter
  55. end
  56. end
  57. 1001 final_delimiter = StringIO.new("--#{@boundary}--\r\n")
  58. 881 @bytesize += final_delimiter.size
  59. 1001 params << final_delimiter
  60. 1001 params
  61. end
  62. 26 def header_part(key, content_type, filename)
  63. 1193 header = "--#{@boundary}\r\n".b
  64. 1193 header << "Content-Disposition: form-data; name=#{key.inspect}".b
  65. 1193 header << "; filename=#{filename.inspect}" if filename
  66. 1193 header << "\r\nContent-Type: #{content_type}\r\n\r\n"
  67. 1193 StringIO.new(header)
  68. end
  69. 26 def read_chunks(buffer, length = nil)
  70. 4618 while @part_index < @parts.size
  71. 10584 chunk = read_from_part(length)
  72. 10584 next unless chunk
  73. 6052 buffer << chunk.force_encoding(Encoding::BINARY)
  74. 6052 next unless length
  75. 5246 length -= chunk.bytesize
  76. 5986 break if length.zero?
  77. end
  78. end
  79. # if there's a current part to read from, tries to read a chunk.
  80. 26 def read_from_part(max_length = nil)
  81. 10584 part = @parts[@part_index]
  82. 10584 chunk = part.read(max_length, @buffer)
  83. 10584 return chunk if chunk && !chunk.empty?
  84. 4532 part.close if part.respond_to?(:close)
  85. 3980 @part_index += 1
  86. 1692 nil
  87. end
  88. end
  89. end
  90. end

lib/httpx/transcoder/multipart/mime_type_detector.rb

92.11% lines covered

38 relevant lines. 35 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Transcoder::Multipart
  4. 26 module MimeTypeDetector
  5. 26 module_function
  6. 26 DEFAULT_MIMETYPE = "application/octet-stream"
  7. # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
  8. 26 if defined?(FileMagic)
  9. 1 MAGIC_NUMBER = 256 * 1024
  10. 1 def call(file, _)
  11. 1 return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
  12. 1 mime = FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
  13. 1 filemagic.buffer(file.read(MAGIC_NUMBER))
  14. end
  15. 1 file.rewind
  16. 1 mime
  17. end
  18. 24 elsif defined?(Marcel)
  19. 1 def call(file, filename)
  20. 1 return nil if file.eof? # marcel returns "application/octet-stream" for empty files
  21. 1 Marcel::MimeType.for(file, name: filename)
  22. end
  23. 23 elsif defined?(MimeMagic)
  24. 1 def call(file, _)
  25. 1 mime = MimeMagic.by_magic(file)
  26. 1 mime.type if mime
  27. end
  28. 22 elsif system("which file", out: File::NULL)
  29. 23 require "open3"
  30. 23 def call(file, _)
  31. 677 return if file.eof? # file command returns "application/x-empty" for empty files
  32. 633 Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
  33. 75 begin
  34. 633 ::IO.copy_stream(file, stdin.binmode)
  35. rescue Errno::EPIPE
  36. end
  37. 633 file.rewind
  38. 633 stdin.close
  39. 633 status = thread.value
  40. # call to file command failed
  41. 633 if status.nil? || !status.success?
  42. $stderr.print(stderr.read)
  43. else
  44. 633 output = stdout.read.strip
  45. 633 if output.include?("cannot open")
  46. $stderr.print(output)
  47. else
  48. 633 output
  49. end
  50. end
  51. end
  52. end
  53. else
  54. def call(_, _); end
  55. end
  56. end
  57. end
  58. end

lib/httpx/transcoder/multipart/part.rb

100.0% lines covered

18 relevant lines. 18 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Transcoder::Multipart
  4. 26 module Part
  5. 26 module_function
  6. 26 def call(value)
  7. # take out specialized objects of the way
  8. 1193 if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
  9. 112 return value, value.content_type, value.filename
  10. end
  11. 1065 content_type = filename = nil
  12. 1065 if value.is_a?(Hash)
  13. 322 content_type = value[:content_type]
  14. 322 filename = value[:filename]
  15. 322 value = value[:body]
  16. end
  17. 1065 value = value.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
  18. 1065 if value.respond_to?(:path) && value.respond_to?(:read)
  19. # either a File, a Tempfile, or something else which has to quack like a file
  20. 681 filename ||= File.basename(value.path)
  21. 681 content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
  22. 681 [value, content_type, filename]
  23. else
  24. 384 [StringIO.new(value.to_s), content_type || "text/plain", filename]
  25. end
  26. end
  27. end
  28. end
  29. end

lib/httpx/transcoder/utils/body_reader.rb

92.0% lines covered

25 relevant lines. 23 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require "stringio"
  3. 26 module HTTPX
  4. 26 module Transcoder
  5. 26 class BodyReader
  6. 26 def initialize(body)
  7. 210 @body = if body.respond_to?(:read)
  8. 20 body.rewind if body.respond_to?(:rewind)
  9. 20 body
  10. 189 elsif body.respond_to?(:each)
  11. 36 body.enum_for(:each)
  12. else
  13. 154 StringIO.new(body.to_s)
  14. end
  15. end
  16. 26 def bytesize
  17. 402 return @body.bytesize if @body.respond_to?(:bytesize)
  18. 366 Float::INFINITY
  19. end
  20. 26 def read(length = nil, outbuf = nil)
  21. 456 return @body.read(length, outbuf) if @body.respond_to?(:read)
  22. begin
  23. 84 chunk = @body.next
  24. 48 if outbuf
  25. outbuf.clear.force_encoding(Encoding::BINARY)
  26. outbuf << chunk
  27. else
  28. 48 outbuf = chunk
  29. end
  30. 48 outbuf unless length && outbuf.empty?
  31. 24 rescue StopIteration
  32. end
  33. end
  34. 26 def close
  35. 48 @body.close if @body.respond_to?(:close)
  36. end
  37. end
  38. end
  39. end

lib/httpx/transcoder/utils/deflater.rb

100.0% lines covered

37 relevant lines. 37 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 require_relative "body_reader"
  3. 26 module HTTPX
  4. 26 module Transcoder
  5. 26 class Deflater
  6. 26 attr_reader :content_type
  7. 26 def initialize(body)
  8. 84 @content_type = body.content_type
  9. 84 @body = BodyReader.new(body)
  10. 84 @closed = false
  11. end
  12. 26 def bytesize
  13. 324 buffer_deflate!
  14. 324 @buffer.size
  15. end
  16. 26 def read(length = nil, outbuf = nil)
  17. 416 return @buffer.read(length, outbuf) if @buffer
  18. 240 return if @closed
  19. 192 chunk = @body.read(length)
  20. 192 compressed_chunk = deflate(chunk)
  21. 192 return unless compressed_chunk
  22. 156 if outbuf
  23. 144 outbuf.clear.force_encoding(Encoding::BINARY)
  24. 144 outbuf << compressed_chunk
  25. else
  26. 12 compressed_chunk
  27. end
  28. end
  29. 26 def close
  30. 48 return if @closed
  31. 48 @buffer.close if @buffer
  32. 48 @body.close
  33. 48 @closed = true
  34. end
  35. 26 def rewind
  36. 28 return unless @buffer
  37. 16 @buffer.rewind
  38. end
  39. 26 private
  40. # rubocop:disable Naming/MemoizedInstanceVariableName
  41. 26 def buffer_deflate!
  42. 324 return @buffer if defined?(@buffer)
  43. 84 buffer = Response::Buffer.new(
  44. threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
  45. )
  46. 84 ::IO.copy_stream(self, buffer)
  47. 84 buffer.rewind if buffer.respond_to?(:rewind)
  48. 84 @buffer = buffer
  49. end
  50. # rubocop:enable Naming/MemoizedInstanceVariableName
  51. end
  52. end
  53. end

lib/httpx/utils.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 26 module HTTPX
  3. 26 module Utils
  4. 26 using URIExtensions
  5. 26 TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
  6. 26 VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
  7. 26 FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
  8. 26 FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
  9. 26 module_function
  10. 26 def now
  11. 11414939 Process.clock_gettime(Process::CLOCK_MONOTONIC)
  12. end
  13. 26 def elapsed_time(monotonic_timestamp)
  14. 11368825 Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
  15. end
  16. # The value of this field can be either an HTTP-date or a number of
  17. # seconds to delay after the response is received.
  18. 26 def parse_retry_after(retry_after)
  19. # first: bet on it being an integer
  20. 63 Integer(retry_after)
  21. rescue ArgumentError
  22. # Then it's a datetime
  23. 16 time = Time.httpdate(retry_after)
  24. 16 time - Time.now
  25. end
  26. 26 def get_filename(header, _prefix_regex = nil)
  27. 88 filename = nil
  28. 77 case header
  29. when FILENAME_REGEX
  30. 56 filename = Regexp.last_match(1)
  31. 56 filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
  32. when FILENAME_EXTENSION_REGEX
  33. 16 filename = Regexp.last_match(1)
  34. 16 encoding, _, filename = filename.split("'", 3)
  35. end
  36. 88 return unless filename
  37. 136 filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
  38. 72 filename.scrub!
  39. 72 filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
  40. 72 filename.force_encoding ::Encoding.find(encoding) if encoding
  41. 72 filename
  42. end
  43. 26 URIParser = URI::RFC2396_Parser.new
  44. 26 def to_uri(uri)
  45. 17136 return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
  46. 33 uri = URI(URIParser.escape(uri))
  47. 33 non_ascii_hostname = URIParser.unescape(uri.host)
  48. 33 non_ascii_hostname.force_encoding(Encoding::UTF_8)
  49. 33 idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
  50. 33 uri.host = idna_hostname
  51. 32 uri.non_ascii_hostname = non_ascii_hostname
  52. 32 uri
  53. end
  54. end
  55. end