loading
Generated 2025-06-10T06:35:31+00:00

All Files ( 96.12% covered at 95443.59 hits/line )

105 files in total.
7700 relevant lines, 7401 lines covered and 299 lines missed. ( 96.12% )
File % covered Lines Relevant Lines Lines covered Lines missed Avg. Hits / Line
lib/httpx.rb 100.00 % 66 39 39 0 807.49
lib/httpx/adapters/datadog.rb 87.04 % 345 162 141 21 40.52
lib/httpx/adapters/faraday.rb 98.13 % 298 160 157 3 133.82
lib/httpx/adapters/sentry.rb 100.00 % 121 62 62 0 98.13
lib/httpx/adapters/webmock.rb 100.00 % 175 93 93 0 95.96
lib/httpx/altsvc.rb 96.39 % 163 83 80 3 243.11
lib/httpx/buffer.rb 100.00 % 61 27 27 0 402114.26
lib/httpx/callbacks.rb 100.00 % 35 19 19 0 167440.68
lib/httpx/chainable.rb 95.65 % 106 46 44 2 1083.83
lib/httpx/connection.rb 92.18 % 957 486 448 38 182698.19
lib/httpx/connection/http1.rb 89.14 % 400 221 197 24 3868.03
lib/httpx/connection/http2.rb 95.09 % 452 265 252 13 285129.10
lib/httpx/domain_name.rb 95.56 % 145 45 43 2 247.22
lib/httpx/errors.rb 97.62 % 109 42 41 1 87.74
lib/httpx/extensions.rb 67.86 % 59 28 19 9 420.25
lib/httpx/headers.rb 100.00 % 176 71 71 0 18726.44
lib/httpx/io.rb 100.00 % 11 5 5 0 27.00
lib/httpx/io/ssl.rb 96.55 % 166 87 84 3 2373.20
lib/httpx/io/tcp.rb 91.15 % 208 113 103 10 8281.80
lib/httpx/io/udp.rb 100.00 % 62 35 35 0 370.09
lib/httpx/io/unix.rb 94.29 % 70 35 33 2 20.49
lib/httpx/loggable.rb 100.00 % 53 20 20 0 581988.95
lib/httpx/options.rb 98.79 % 375 165 163 2 18242.35
lib/httpx/parser/http1.rb 100.00 % 186 109 109 0 7825.29
lib/httpx/plugins/auth.rb 100.00 % 25 9 9 0 24.00
lib/httpx/plugins/auth/basic.rb 100.00 % 20 10 10 0 86.60
lib/httpx/plugins/auth/digest.rb 100.00 % 102 61 61 0 127.85
lib/httpx/plugins/auth/ntlm.rb 100.00 % 35 19 19 0 2.95
lib/httpx/plugins/auth/socks5.rb 100.00 % 22 11 11 0 23.82
lib/httpx/plugins/aws_sdk_authentication.rb 100.00 % 109 43 43 0 13.21
lib/httpx/plugins/aws_sigv4.rb 100.00 % 237 121 121 0 101.93
lib/httpx/plugins/basic_auth.rb 100.00 % 29 12 12 0 35.33
lib/httpx/plugins/brotli.rb 100.00 % 50 25 25 0 10.80
lib/httpx/plugins/callbacks.rb 100.00 % 115 55 55 0 132.45
lib/httpx/plugins/circuit_breaker.rb 100.00 % 145 66 66 0 74.82
lib/httpx/plugins/circuit_breaker/circuit.rb 100.00 % 100 47 47 0 58.21
lib/httpx/plugins/circuit_breaker/circuit_store.rb 100.00 % 53 23 23 0 98.48
lib/httpx/plugins/content_digest.rb 100.00 % 202 102 102 0 79.35
lib/httpx/plugins/cookies.rb 100.00 % 107 52 52 0 122.63
lib/httpx/plugins/cookies/cookie.rb 100.00 % 174 77 77 0 318.26
lib/httpx/plugins/cookies/jar.rb 100.00 % 95 46 46 0 260.85
lib/httpx/plugins/cookies/set_cookie_parser.rb 100.00 % 143 72 72 0 156.88
lib/httpx/plugins/digest_auth.rb 100.00 % 65 29 29 0 99.24
lib/httpx/plugins/expect.rb 100.00 % 118 56 56 0 87.91
lib/httpx/plugins/follow_redirects.rb 100.00 % 231 108 108 0 196576.34
lib/httpx/plugins/grpc.rb 100.00 % 280 133 133 0 117.79
lib/httpx/plugins/grpc/call.rb 90.91 % 63 33 30 3 40.18
lib/httpx/plugins/grpc/grpc_encoding.rb 97.87 % 90 47 46 1 76.85
lib/httpx/plugins/grpc/message.rb 95.83 % 55 24 23 1 39.50
lib/httpx/plugins/h2c.rb 94.92 % 117 59 56 3 14.34
lib/httpx/plugins/ntlm_auth.rb 100.00 % 60 30 30 0 3.97
lib/httpx/plugins/oauth.rb 100.00 % 175 87 87 0 70.23
lib/httpx/plugins/persistent.rb 100.00 % 73 28 28 0 214.14
lib/httpx/plugins/proxy.rb 98.01 % 308 151 148 3 325.01
lib/httpx/plugins/proxy/http.rb 100.00 % 184 104 104 0 186.51
lib/httpx/plugins/proxy/socks4.rb 97.47 % 135 79 77 2 178.85
lib/httpx/plugins/proxy/socks5.rb 99.12 % 194 113 112 1 279.94
lib/httpx/plugins/proxy/ssh.rb 92.31 % 92 52 48 4 8.19
lib/httpx/plugins/push_promise.rb 100.00 % 81 41 41 0 10.49
lib/httpx/plugins/query.rb 100.00 % 35 15 15 0 10.07
lib/httpx/plugins/rate_limiter.rb 100.00 % 55 18 18 0 41.72
lib/httpx/plugins/response_cache.rb 100.00 % 331 143 143 0 187.63
lib/httpx/plugins/response_cache/file_store.rb 100.00 % 140 72 72 0 166.58
lib/httpx/plugins/response_cache/store.rb 100.00 % 33 16 16 0 153.94
lib/httpx/plugins/retries.rb 96.91 % 228 97 94 3 198699.23
lib/httpx/plugins/ssrf_filter.rb 100.00 % 145 61 61 0 153.89
lib/httpx/plugins/stream.rb 97.78 % 183 90 88 2 115.18
lib/httpx/plugins/stream_bidi.rb 99.30 % 315 142 141 1 89.68
lib/httpx/plugins/upgrade.rb 100.00 % 78 34 34 0 44.29
lib/httpx/plugins/upgrade/h2.rb 91.67 % 54 24 22 2 7.79
lib/httpx/plugins/webdav.rb 100.00 % 86 39 39 0 23.10
lib/httpx/plugins/xml.rb 100.00 % 76 34 34 0 84.35
lib/httpx/pmatch_extensions.rb 100.00 % 33 17 17 0 28.71
lib/httpx/pool.rb 100.00 % 185 89 89 0 4873.11
lib/httpx/punycode.rb 100.00 % 22 9 9 0 18.78
lib/httpx/request.rb 100.00 % 317 132 132 0 6724.05
lib/httpx/request/body.rb 100.00 % 153 66 66 0 3223.18
lib/httpx/resolver.rb 100.00 % 161 82 82 0 1870.98
lib/httpx/resolver/https.rb 86.11 % 254 144 124 20 34.19
lib/httpx/resolver/multi.rb 88.24 % 93 51 45 6 3640.75
lib/httpx/resolver/native.rb 89.00 % 538 309 275 34 1251.90
lib/httpx/resolver/resolver.rb 84.52 % 169 84 71 13 1810.86
lib/httpx/resolver/system.rb 78.99 % 253 138 109 29 17.30
lib/httpx/response.rb 100.00 % 304 115 115 0 1938.23
lib/httpx/response/body.rb 100.00 % 242 109 109 0 2979.05
lib/httpx/response/buffer.rb 92.19 % 120 64 59 5 1577.27
lib/httpx/selector.rb 93.64 % 224 110 103 7 1948170.67
lib/httpx/session.rb 95.60 % 553 273 261 12 122605.23
lib/httpx/session_extensions.rb 100.00 % 29 14 14 0 6.64
lib/httpx/timers.rb 93.94 % 133 66 62 4 3396958.50
lib/httpx/transcoder.rb 100.00 % 91 52 52 0 281.75
lib/httpx/transcoder/body.rb 100.00 % 43 26 26 0 965.31
lib/httpx/transcoder/chunker.rb 100.00 % 115 66 66 0 211.85
lib/httpx/transcoder/deflate.rb 100.00 % 37 20 20 0 30.70
lib/httpx/transcoder/form.rb 100.00 % 80 42 42 0 439.62
lib/httpx/transcoder/gzip.rb 100.00 % 71 40 40 0 98.45
lib/httpx/transcoder/json.rb 100.00 % 71 33 33 0 42.61
lib/httpx/transcoder/multipart.rb 100.00 % 17 10 10 0 921.70
lib/httpx/transcoder/multipart/decoder.rb 94.05 % 141 84 79 5 29.20
lib/httpx/transcoder/multipart/encoder.rb 100.00 % 120 70 70 0 1945.89
lib/httpx/transcoder/multipart/mime_type_detector.rb 92.11 % 78 38 35 3 177.47
lib/httpx/transcoder/multipart/part.rb 100.00 % 35 18 18 0 505.17
lib/httpx/transcoder/utils/body_reader.rb 92.00 % 46 25 23 2 99.88
lib/httpx/transcoder/utils/deflater.rb 100.00 % 75 37 37 0 102.92
lib/httpx/utils.rb 100.00 % 75 39 39 0 548261.72

lib/httpx.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "httpx/version"
  3. # Top-Level Namespace
  4. #
  5. 27 module HTTPX
  6. 27 EMPTY = [].freeze
  7. 27 EMPTY_HASH = {}.freeze
  8. # All plugins should be stored under this module/namespace. Can register and load
  9. # plugins.
  10. #
  11. 27 module Plugins
  12. 27 @plugins = {}
  13. 27 @plugins_mutex = Thread::Mutex.new
  14. # Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
  15. # it from the load path under "httpx/plugins/" directory.
  16. #
  17. 27 def self.load_plugin(name)
  18. 5712 h = @plugins
  19. 5712 m = @plugins_mutex
  20. 11424 unless (plugin = m.synchronize { h[name] })
  21. 270 require "httpx/plugins/#{name}"
  22. 540 raise "Plugin #{name} hasn't been registered" unless (plugin = m.synchronize { h[name] })
  23. end
  24. 5712 plugin
  25. end
  26. # Registers a plugin (+mod+) in the central store indexed by +name+.
  27. #
  28. 27 def self.register_plugin(name, mod)
  29. 336 h = @plugins
  30. 336 m = @plugins_mutex
  31. 640 m.synchronize { h[name] = mod }
  32. end
  33. end
  34. end
  35. 27 require "httpx/extensions"
  36. 27 require "httpx/errors"
  37. 27 require "httpx/utils"
  38. 27 require "httpx/punycode"
  39. 27 require "httpx/domain_name"
  40. 27 require "httpx/altsvc"
  41. 27 require "httpx/callbacks"
  42. 27 require "httpx/loggable"
  43. 27 require "httpx/transcoder"
  44. 27 require "httpx/timers"
  45. 27 require "httpx/pool"
  46. 27 require "httpx/headers"
  47. 27 require "httpx/request"
  48. 27 require "httpx/response"
  49. 27 require "httpx/options"
  50. 27 require "httpx/chainable"
  51. 27 require "httpx/session"
  52. 27 require "httpx/session_extensions"
  53. # load integrations when possible
  54. 27 require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog::Tracing)
  55. 27 require "httpx/adapters/sentry" if defined?(Sentry)
  56. 27 require "httpx/adapters/webmock" if defined?(WebMock)

lib/httpx/adapters/datadog.rb

87.04% lines covered

162 relevant lines. 141 lines covered and 21 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "datadog/tracing/contrib/integration"
  3. 6 require "datadog/tracing/contrib/configuration/settings"
  4. 6 require "datadog/tracing/contrib/patcher"
  5. 6 module Datadog::Tracing
  6. 6 module Contrib
  7. 6 module HTTPX
  8. 6 DATADOG_VERSION = defined?(::DDTrace) ? ::DDTrace::VERSION : ::Datadog::VERSION
  9. 6 METADATA_MODULE = Datadog::Tracing::Metadata
  10. 6 TYPE_OUTBOUND = Datadog::Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
  11. 6 TAG_BASE_SERVICE = Datadog::Tracing::Contrib::Ext::Metadata::TAG_BASE_SERVICE
  12. 6 TAG_PEER_HOSTNAME = Datadog::Tracing::Metadata::Ext::TAG_PEER_HOSTNAME
  13. 6 TAG_KIND = Datadog::Tracing::Metadata::Ext::TAG_KIND
  14. 6 TAG_CLIENT = Datadog::Tracing::Metadata::Ext::SpanKind::TAG_CLIENT
  15. 6 TAG_COMPONENT = Datadog::Tracing::Metadata::Ext::TAG_COMPONENT
  16. 6 TAG_OPERATION = Datadog::Tracing::Metadata::Ext::TAG_OPERATION
  17. 6 TAG_URL = Datadog::Tracing::Metadata::Ext::HTTP::TAG_URL
  18. 6 TAG_METHOD = Datadog::Tracing::Metadata::Ext::HTTP::TAG_METHOD
  19. 6 TAG_TARGET_HOST = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_HOST
  20. 6 TAG_TARGET_PORT = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_PORT
  21. 6 TAG_STATUS_CODE = Datadog::Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE
  22. # HTTPX Datadog Plugin
  23. #
  24. # Enables tracing for httpx requests.
  25. #
  26. # A span will be created for each request transaction; the span is created lazily only when
  27. # buffering a request, and it is fed the start time stored inside the tracer object.
  28. #
  29. 6 module Plugin
  30. 6 module RequestTracer
  31. 6 extend Contrib::HttpAnnotationHelper
  32. 6 module_function
  33. 6 SPAN_REQUEST = "httpx.request"
  34. # initializes tracing on the +request+.
  35. 6 def call(request)
  36. 249 return unless configuration(request).enabled
  37. 84 span = nil
  38. # request objects are reused, when already buffered requests get rerouted to a different
  39. # connection due to connection issues, or when they already got a response, but need to
  40. # be retried. In such situations, the original span needs to be extended for the former,
  41. # while a new is required for the latter.
  42. 84 request.on(:idle) do
  43. 18 span = nil
  44. end
  45. # the span is initialized when the request is buffered in the parser, which is the closest
  46. # one gets to actually sending the request.
  47. 84 request.on(:headers) do
  48. 96 next if span
  49. 96 span = initialize_span(request, now)
  50. end
  51. 84 request.on(:response) do |response|
  52. 96 unless span
  53. 6 next unless response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
  54. # handles the case when the +error+ happened during name resolution, which means
  55. # that the tracing start point hasn't been triggered yet; in such cases, the approximate
  56. # initial resolving time is collected from the connection, and used as span start time,
  57. # and the tracing object in inserted before the on response callback is called.
  58. 6 span = initialize_span(request, response.error.connection.init_time)
  59. end
  60. 96 finish(response, span)
  61. end
  62. end
  63. 6 def finish(response, span)
  64. 96 if response.is_a?(::HTTPX::ErrorResponse)
  65. 6 span.set_error(response.error)
  66. else
  67. 90 span.set_tag(TAG_STATUS_CODE, response.status.to_s)
  68. 90 span.set_error(::HTTPX::HTTPError.new(response)) if response.status >= 400 && response.status <= 599
  69. 90 span.set_tags(
  70. Datadog.configuration.tracing.header_tags.response_tags(response.headers.to_h)
  71. )
  72. end
  73. 96 span.finish
  74. end
  75. # return a span initialized with the +@request+ state.
  76. 6 def initialize_span(request, start_time)
  77. 102 verb = request.verb
  78. 102 uri = request.uri
  79. 102 config = configuration(request)
  80. 102 span = create_span(request, config, start_time)
  81. 102 span.resource = verb
  82. # Tag original global service name if not used
  83. 102 span.set_tag(TAG_BASE_SERVICE, Datadog.configuration.service) if span.service != Datadog.configuration.service
  84. 102 span.set_tag(TAG_KIND, TAG_CLIENT)
  85. 102 span.set_tag(TAG_COMPONENT, "httpx")
  86. 102 span.set_tag(TAG_OPERATION, "request")
  87. 102 span.set_tag(TAG_URL, request.path)
  88. 102 span.set_tag(TAG_METHOD, verb)
  89. 102 span.set_tag(TAG_TARGET_HOST, uri.host)
  90. 102 span.set_tag(TAG_TARGET_PORT, uri.port)
  91. 102 span.set_tag(TAG_PEER_HOSTNAME, uri.host)
  92. # Tag as an external peer service
  93. # span.set_tag(TAG_PEER_SERVICE, span.service)
  94. 102 if config[:distributed_tracing]
  95. 96 propagate_trace_http(
  96. Datadog::Tracing.active_trace,
  97. request.headers
  98. )
  99. end
  100. # Set analytics sample rate
  101. 102 if Contrib::Analytics.enabled?(config[:analytics_enabled])
  102. 12 Contrib::Analytics.set_sample_rate(span, config[:analytics_sample_rate])
  103. end
  104. 102 span.set_tags(
  105. Datadog.configuration.tracing.header_tags.request_tags(request.headers.to_h)
  106. )
  107. 102 span
  108. rescue StandardError => e
  109. Datadog.logger.error("error preparing span for http request: #{e}")
  110. Datadog.logger.error(e.backtrace)
  111. end
  112. 6 def now
  113. 96 ::Datadog::Core::Utils::Time.now.utc
  114. end
  115. 6 def configuration(request)
  116. 351 Datadog.configuration.tracing[:httpx, request.uri.host]
  117. end
  118. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
  119. 3 def propagate_trace_http(trace, headers)
  120. 48 Datadog::Tracing::Contrib::HTTP.inject(trace, headers)
  121. end
  122. 3 def create_span(request, configuration, start_time)
  123. 51 Datadog::Tracing.trace(
  124. SPAN_REQUEST,
  125. service: service_name(request.uri.host, configuration),
  126. type: TYPE_OUTBOUND,
  127. start_time: start_time
  128. )
  129. end
  130. else
  131. 3 def propagate_trace_http(trace, headers)
  132. 48 Datadog::Tracing::Propagation::HTTP.inject!(trace.to_digest, headers)
  133. end
  134. 3 def create_span(request, configuration, start_time)
  135. 51 Datadog::Tracing.trace(
  136. SPAN_REQUEST,
  137. service: service_name(request.uri.host, configuration),
  138. span_type: TYPE_OUTBOUND,
  139. start_time: start_time
  140. )
  141. end
  142. end
  143. end
  144. 6 module RequestMethods
  145. # intercepts request initialization to inject the tracing logic.
  146. 6 def initialize(*)
  147. 249 super
  148. 249 return unless Datadog::Tracing.enabled?
  149. 249 RequestTracer.call(self)
  150. end
  151. end
  152. 6 module ConnectionMethods
  153. 6 attr_reader :init_time
  154. 6 def initialize(*)
  155. 236 super
  156. 236 @init_time = ::Datadog::Core::Utils::Time.now.utc
  157. end
  158. end
  159. end
  160. 6 module Configuration
  161. # Default settings for httpx
  162. #
  163. 6 class Settings < Datadog::Tracing::Contrib::Configuration::Settings
  164. 6 DEFAULT_ERROR_HANDLER = lambda do |response|
  165. Datadog::Ext::HTTP::ERROR_RANGE.cover?(response.status)
  166. end
  167. 6 option :service_name, default: "httpx"
  168. 6 option :distributed_tracing, default: true
  169. 6 option :split_by_domain, default: false
  170. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  171. 6 option :enabled do |o|
  172. 6 o.type :bool
  173. 6 o.env "DD_TRACE_HTTPX_ENABLED"
  174. 6 o.default true
  175. end
  176. 6 option :analytics_enabled do |o|
  177. 6 o.type :bool
  178. 6 o.env "DD_TRACE_HTTPX_ANALYTICS_ENABLED"
  179. 6 o.default false
  180. end
  181. 6 option :analytics_sample_rate do |o|
  182. 6 o.type :float
  183. 6 o.env "DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE"
  184. 6 o.default 1.0
  185. end
  186. else
  187. option :enabled do |o|
  188. o.default { env_to_bool("DD_TRACE_HTTPX_ENABLED", true) }
  189. o.lazy
  190. end
  191. option :analytics_enabled do |o|
  192. o.default { env_to_bool(%w[DD_TRACE_HTTPX_ANALYTICS_ENABLED DD_HTTPX_ANALYTICS_ENABLED], false) }
  193. o.lazy
  194. end
  195. option :analytics_sample_rate do |o|
  196. o.default { env_to_float(%w[DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE DD_HTTPX_ANALYTICS_SAMPLE_RATE], 1.0) }
  197. o.lazy
  198. end
  199. end
  200. 6 if defined?(Datadog::Tracing::Contrib::SpanAttributeSchema)
  201. 6 option :service_name do |o|
  202. 6 o.default do
  203. 66 Datadog::Tracing::Contrib::SpanAttributeSchema.fetch_service_name(
  204. "DD_TRACE_HTTPX_SERVICE_NAME",
  205. "httpx"
  206. )
  207. end
  208. 6 o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  209. end
  210. else
  211. option :service_name do |o|
  212. o.default do
  213. ENV.fetch("DD_TRACE_HTTPX_SERVICE_NAME", "httpx")
  214. end
  215. o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  216. end
  217. end
  218. 6 option :distributed_tracing, default: true
  219. 6 if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.15.0")
  220. 6 option :error_handler do |o|
  221. 6 o.type :proc
  222. 6 o.default_proc(&DEFAULT_ERROR_HANDLER)
  223. end
  224. elsif Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
  225. option :error_handler do |o|
  226. o.type :proc
  227. o.experimental_default_proc(&DEFAULT_ERROR_HANDLER)
  228. end
  229. else
  230. option :error_handler, default: DEFAULT_ERROR_HANDLER
  231. end
  232. end
  233. end
  234. # Patcher enables patching of 'httpx' with datadog components.
  235. #
  236. 6 module Patcher
  237. 6 include Datadog::Tracing::Contrib::Patcher
  238. 6 module_function
  239. 6 def target_version
  240. 12 Integration.version
  241. end
  242. # loads a session instannce with the datadog plugin, and replaces the
  243. # base HTTPX::Session with the patched session class.
  244. 6 def patch
  245. 6 datadog_session = ::HTTPX.plugin(Plugin)
  246. 6 ::HTTPX.send(:remove_const, :Session)
  247. 6 ::HTTPX.send(:const_set, :Session, datadog_session.class)
  248. end
  249. end
  250. # Datadog Integration for HTTPX.
  251. #
  252. 6 class Integration
  253. 6 include Contrib::Integration
  254. 6 MINIMUM_VERSION = Gem::Version.new("0.10.2")
  255. 6 register_as :httpx
  256. 6 def self.version
  257. 246 Gem.loaded_specs["httpx"] && Gem.loaded_specs["httpx"].version
  258. end
  259. 6 def self.loaded?
  260. 78 defined?(::HTTPX::Request)
  261. end
  262. 6 def self.compatible?
  263. 78 super && version >= MINIMUM_VERSION
  264. end
  265. 6 def new_configuration
  266. 156 Configuration::Settings.new
  267. end
  268. 6 def patcher
  269. 156 Patcher
  270. end
  271. end
  272. end
  273. end
  274. end

lib/httpx/adapters/faraday.rb

98.13% lines covered

160 relevant lines. 157 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 15 require "delegate"
  3. 15 require "httpx"
  4. 15 require "faraday"
  5. 15 module Faraday
  6. 15 class Adapter
  7. 15 class HTTPX < Faraday::Adapter
  8. 15 module RequestMixin
  9. 15 def build_connection(env)
  10. 287 return @connection if defined?(@connection)
  11. 287 @connection = ::HTTPX.plugin(:persistent).plugin(ReasonPlugin)
  12. 287 @connection = @connection.with(@connection_options) unless @connection_options.empty?
  13. 287 connection_opts = options_from_env(env)
  14. 287 if (bind = env.request.bind)
  15. 8 @bind = TCPSocket.new(bind[:host], bind[:port])
  16. 7 connection_opts[:io] = @bind
  17. end
  18. 287 @connection = @connection.with(connection_opts)
  19. 287 if (proxy = env.request.proxy)
  20. 8 proxy_options = { uri: proxy.uri }
  21. 8 proxy_options[:username] = proxy.user if proxy.user
  22. 8 proxy_options[:password] = proxy.password if proxy.password
  23. 8 @connection = @connection.plugin(:proxy).with(proxy: proxy_options)
  24. end
  25. 287 @connection = @connection.plugin(OnDataPlugin) if env.request.stream_response?
  26. 287 @connection = @config_block.call(@connection) || @connection if @config_block
  27. 287 @connection
  28. end
  29. 15 def close
  30. 294 @connection.close if @connection
  31. 294 @bind.close if @bind
  32. end
  33. 15 private
  34. 15 def connect(env, &blk)
  35. 287 connection(env, &blk)
  36. rescue ::HTTPX::TLSError => e
  37. 8 raise Faraday::SSLError, e
  38. rescue Errno::ECONNABORTED,
  39. Errno::ECONNREFUSED,
  40. Errno::ECONNRESET,
  41. Errno::EHOSTUNREACH,
  42. Errno::EINVAL,
  43. Errno::ENETUNREACH,
  44. Errno::EPIPE,
  45. ::HTTPX::ConnectionError => e
  46. 8 raise Faraday::ConnectionFailed, e
  47. end
  48. 15 def build_request(env)
  49. 295 meth = env[:method]
  50. 30 request_options = {
  51. 264 headers: env.request_headers,
  52. body: env.body,
  53. **options_from_env(env),
  54. }
  55. 295 [meth.to_s.upcase, env.url, request_options]
  56. end
  57. 15 def options_from_env(env)
  58. 582 timeout_options = {}
  59. 582 req_opts = env.request
  60. 582 if (sec = request_timeout(:read, req_opts))
  61. 28 timeout_options[:read_timeout] = sec
  62. end
  63. 582 if (sec = request_timeout(:write, req_opts))
  64. 14 timeout_options[:write_timeout] = sec
  65. end
  66. 582 if (sec = request_timeout(:open, req_opts))
  67. 14 timeout_options[:connect_timeout] = sec
  68. end
  69. 59 {
  70. 522 ssl: ssl_options_from_env(env),
  71. timeout: timeout_options,
  72. }
  73. end
  74. 15 if defined?(::OpenSSL)
  75. 15 def ssl_options_from_env(env)
  76. 582 ssl_options = {}
  77. 582 unless env.ssl.verify.nil?
  78. 28 ssl_options[:verify_mode] = env.ssl.verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
  79. end
  80. 582 ssl_options[:ca_file] = env.ssl.ca_file if env.ssl.ca_file
  81. 582 ssl_options[:ca_path] = env.ssl.ca_path if env.ssl.ca_path
  82. 582 ssl_options[:cert_store] = env.ssl.cert_store if env.ssl.cert_store
  83. 582 ssl_options[:cert] = env.ssl.client_cert if env.ssl.client_cert
  84. 582 ssl_options[:key] = env.ssl.client_key if env.ssl.client_key
  85. 582 ssl_options[:ssl_version] = env.ssl.version if env.ssl.version
  86. 582 ssl_options[:verify_depth] = env.ssl.verify_depth if env.ssl.verify_depth
  87. 582 ssl_options[:min_version] = env.ssl.min_version if env.ssl.min_version
  88. 582 ssl_options[:max_version] = env.ssl.max_version if env.ssl.max_version
  89. 582 ssl_options
  90. end
  91. else
  92. skipped # :nocov:
  93. skipped def ssl_options_from_env(*)
  94. skipped {}
  95. skipped end
  96. skipped # :nocov:
  97. end
  98. end
  99. 15 include RequestMixin
  100. 15 module OnDataPlugin
  101. 15 module RequestMethods
  102. 15 attr_writer :response_on_data
  103. 15 def response=(response)
  104. 16 super
  105. 16 return if response.is_a?(::HTTPX::ErrorResponse)
  106. 16 response.body.on_data = @response_on_data
  107. end
  108. end
  109. 15 module ResponseBodyMethods
  110. 15 attr_writer :on_data
  111. 15 def write(chunk)
  112. 41 return super unless @on_data
  113. 41 @on_data.call(chunk, chunk.bytesize)
  114. end
  115. end
  116. end
  117. 15 module ReasonPlugin
  118. 15 def self.load_dependencies(*)
  119. 287 require "net/http/status"
  120. end
  121. 15 module ResponseMethods
  122. 15 def reason
  123. 241 Net::HTTP::STATUS_CODES.fetch(@status, "Non-Standard status code")
  124. end
  125. end
  126. end
  127. 15 class ParallelManager
  128. 15 class ResponseHandler < SimpleDelegator
  129. 15 attr_reader :env
  130. 15 def initialize(env)
  131. 32 @env = env
  132. 32 super
  133. end
  134. 15 def on_response(&blk)
  135. 64 if blk
  136. 32 @on_response = ->(response) do
  137. 32 blk.call(response)
  138. end
  139. 32 self
  140. else
  141. 32 @on_response
  142. end
  143. end
  144. 15 def on_complete(&blk)
  145. 32 if blk
  146. @on_complete = blk
  147. self
  148. else
  149. 32 @on_complete
  150. end
  151. end
  152. end
  153. 15 include RequestMixin
  154. 15 def initialize(options)
  155. 32 @handlers = []
  156. 32 @connection_options = options
  157. end
  158. 15 def enqueue(request)
  159. 32 handler = ResponseHandler.new(request)
  160. 32 @handlers << handler
  161. 32 handler
  162. end
  163. 15 def run
  164. 32 return unless @handlers.last
  165. 24 env = @handlers.last.env
  166. 24 connect(env) do |session|
  167. 56 requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
  168. 24 if env.request.stream_response?
  169. 8 requests.each do |request|
  170. 8 request.response_on_data = env.request.on_data
  171. end
  172. end
  173. 24 responses = session.request(*requests)
  174. 24 Array(responses).each_with_index do |response, index|
  175. 32 handler = @handlers[index]
  176. 32 handler.on_response.call(response)
  177. 32 handler.on_complete.call(handler.env) if handler.on_complete
  178. end
  179. end
  180. rescue ::HTTPX::TimeoutError => e
  181. raise Faraday::TimeoutError, e
  182. end
  183. # from Faraday::Adapter#connection
  184. 15 def connection(env)
  185. 24 conn = build_connection(env)
  186. 24 return conn unless block_given?
  187. 24 yield conn
  188. end
  189. 15 private
  190. # from Faraday::Adapter#request_timeout
  191. 15 def request_timeout(type, options)
  192. 168 key = Faraday::Adapter::TIMEOUT_KEYS[type]
  193. 168 options[key] || options[:timeout]
  194. end
  195. end
  196. 15 self.supports_parallel = true
  197. 15 class << self
  198. 15 def setup_parallel_manager(options = {})
  199. 32 ParallelManager.new(options)
  200. end
  201. end
  202. 15 def call(env)
  203. 295 super
  204. 295 if parallel?(env)
  205. 32 handler = env[:parallel_manager].enqueue(env)
  206. 32 handler.on_response do |response|
  207. 32 if response.is_a?(::HTTPX::Response)
  208. 24 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  209. 24 response_headers.merge!(response.headers)
  210. end
  211. else
  212. 7 env[:error] = response.error
  213. 8 save_response(env, 0, "", {}, nil)
  214. end
  215. end
  216. 28 return handler
  217. end
  218. 263 response = connect_and_request(env)
  219. 217 save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
  220. 217 response_headers.merge!(response.headers)
  221. end
  222. 217 @app.call(env)
  223. end
  224. 15 private
  225. 15 def connect_and_request(env)
  226. 263 connect(env) do |session|
  227. 263 request = session.build_request(*build_request(env))
  228. 263 request.response_on_data = env.request.on_data if env.request.stream_response?
  229. 263 response = session.request(request)
  230. # do not call #raise_for_status for HTTP 4xx or 5xx, as faraday has a middleware for that.
  231. 263 response.raise_for_status unless response.is_a?(::HTTPX::Response)
  232. 217 response
  233. end
  234. rescue ::HTTPX::TimeoutError => e
  235. 24 raise Faraday::TimeoutError, e
  236. end
  237. 15 def parallel?(env)
  238. 295 env[:parallel_manager]
  239. end
  240. end
  241. 15 register_middleware httpx: HTTPX
  242. end
  243. end

lib/httpx/adapters/sentry.rb

100.0% lines covered

62 relevant lines. 62 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "sentry-ruby"
  3. 6 module HTTPX::Plugins
  4. 6 module Sentry
  5. 6 module Tracer
  6. 6 module_function
  7. 6 def call(request)
  8. 129 sentry_span = start_sentry_span
  9. 129 return unless sentry_span
  10. 129 set_sentry_trace_header(request, sentry_span)
  11. 129 request.on(:response, &method(:finish_sentry_span).curry(3)[sentry_span, request])
  12. end
  13. 6 def start_sentry_span
  14. 129 return unless ::Sentry.initialized? && (span = ::Sentry.get_current_scope.get_span)
  15. 129 return if span.sampled == false
  16. 129 span.start_child(op: "httpx.client", start_timestamp: ::Sentry.utc_now.to_f)
  17. end
  18. 6 def set_sentry_trace_header(request, sentry_span)
  19. 129 return unless sentry_span
  20. 129 config = ::Sentry.configuration
  21. 129 url = request.uri.to_s
  22. 258 return unless config.propagate_traces && config.trace_propagation_targets.any? { |target| url.match?(target) }
  23. 129 trace = ::Sentry.get_current_client.generate_sentry_trace(sentry_span)
  24. 129 request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
  25. end
  26. 6 def finish_sentry_span(span, request, response)
  27. 135 return unless ::Sentry.initialized?
  28. 135 record_sentry_breadcrumb(request, response)
  29. 135 record_sentry_span(request, response, span)
  30. end
  31. 6 def record_sentry_breadcrumb(req, res)
  32. 135 return unless ::Sentry.configuration.breadcrumbs_logger.include?(:http_logger)
  33. 135 request_info = extract_request_info(req)
  34. 135 data = if res.is_a?(HTTPX::ErrorResponse)
  35. 13 { error: res.error.message, **request_info }
  36. else
  37. 122 { status: res.status, **request_info }
  38. end
  39. 135 crumb = ::Sentry::Breadcrumb.new(
  40. level: :info,
  41. category: "httpx",
  42. type: :info,
  43. data: data
  44. )
  45. 135 ::Sentry.add_breadcrumb(crumb)
  46. end
  47. 6 def record_sentry_span(req, res, sentry_span)
  48. 135 return unless sentry_span
  49. 135 request_info = extract_request_info(req)
  50. 135 sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
  51. 135 if res.is_a?(HTTPX::ErrorResponse)
  52. 13 sentry_span.set_data(:error, res.error.message)
  53. else
  54. 122 sentry_span.set_data(:status, res.status)
  55. end
  56. 135 sentry_span.set_timestamp(::Sentry.utc_now.to_f)
  57. end
  58. 6 def extract_request_info(req)
  59. 270 uri = req.uri
  60. result = {
  61. 270 method: req.verb,
  62. }
  63. 270 if ::Sentry.configuration.send_default_pii
  64. 24 uri += "?#{req.query}" unless req.query.empty?
  65. 24 result[:body] = req.body.to_s unless req.body.empty? || req.body.unbounded_body?
  66. end
  67. 270 result[:url] = uri.to_s
  68. 270 result
  69. end
  70. end
  71. 6 module RequestMethods
  72. 6 def __sentry_enable_trace!
  73. 135 return if @__sentry_enable_trace
  74. 129 Tracer.call(self)
  75. 129 @__sentry_enable_trace = true
  76. end
  77. end
  78. 6 module ConnectionMethods
  79. 6 def send(request)
  80. 135 request.__sentry_enable_trace!
  81. 135 super
  82. end
  83. end
  84. end
  85. end
  86. 6 Sentry.register_patch(:httpx) do
  87. 30 sentry_session = HTTPX.plugin(HTTPX::Plugins::Sentry)
  88. 30 HTTPX.send(:remove_const, :Session)
  89. 30 HTTPX.send(:const_set, :Session, sentry_session.class)
  90. end

lib/httpx/adapters/webmock.rb

100.0% lines covered

93 relevant lines. 93 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module WebMock
  3. 8 module HttpLibAdapters
  4. 8 require "net/http/status"
  5. 8 HTTP_REASONS = Net::HTTP::STATUS_CODES
  6. #
  7. # HTTPX plugin for webmock.
  8. #
  9. # Requests are "hijacked" at the session, before they're distributed to a connection.
  10. #
  11. 8 module Plugin
  12. 8 class << self
  13. 8 def build_webmock_request_signature(request)
  14. 188 uri = WebMock::Util::URI.heuristic_parse(request.uri)
  15. 188 uri.query = request.query
  16. 188 uri.path = uri.normalized_path.gsub("[^:]//", "/")
  17. 188 WebMock::RequestSignature.new(
  18. request.verb.downcase.to_sym,
  19. uri.to_s,
  20. body: request.body.to_s,
  21. headers: request.headers.to_h
  22. )
  23. end
  24. 8 def build_webmock_response(_request, response)
  25. 6 webmock_response = WebMock::Response.new
  26. 6 webmock_response.status = [response.status, HTTP_REASONS[response.status]]
  27. 6 webmock_response.body = response.body.to_s
  28. 6 webmock_response.headers = response.headers.to_h
  29. 6 webmock_response
  30. end
  31. 8 def build_from_webmock_response(request, webmock_response)
  32. 158 return build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
  33. 140 return build_error_response(request, webmock_response.exception) if webmock_response.exception
  34. 133 request.options.response_class.new(request,
  35. webmock_response.status[0],
  36. "2.0",
  37. webmock_response.headers).tap do |res|
  38. 133 res.mocked = true
  39. end
  40. end
  41. 8 def build_error_response(request, exception)
  42. 25 HTTPX::ErrorResponse.new(request, exception)
  43. end
  44. end
  45. 8 module InstanceMethods
  46. 8 private
  47. 8 def do_init_connection(connection, selector)
  48. 170 super
  49. 170 connection.once(:unmock_connection) do
  50. 24 next unless connection.current_session == self
  51. 24 unless connection.addresses
  52. # reset Happy Eyeballs, fail early
  53. 24 connection.sibling = nil
  54. 24 deselect_connection(connection, selector)
  55. end
  56. 24 resolve_connection(connection, selector)
  57. end
  58. end
  59. end
  60. 8 module ResponseMethods
  61. 8 attr_accessor :mocked
  62. 8 def initialize(*)
  63. 157 super
  64. 157 @mocked = false
  65. end
  66. end
  67. 8 module ResponseBodyMethods
  68. 8 def decode_chunk(chunk)
  69. 96 return chunk if @response.mocked
  70. 42 super
  71. end
  72. end
  73. 8 module ConnectionMethods
  74. 8 def initialize(*)
  75. 170 super
  76. 170 @mocked = true
  77. end
  78. 8 def open?
  79. 194 return true if @mocked
  80. 24 super
  81. end
  82. 8 def interests
  83. 273 return if @mocked
  84. 243 super
  85. end
  86. 8 def terminate
  87. 145 force_reset
  88. end
  89. 8 def send(request)
  90. 188 request_signature = Plugin.build_webmock_request_signature(request)
  91. 188 WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
  92. 188 if (mock_response = WebMock::StubRegistry.instance.response_for_request(request_signature))
  93. 158 response = Plugin.build_from_webmock_response(request, mock_response)
  94. 158 WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, request_signature, mock_response)
  95. 158 log { "mocking #{request.uri} with #{mock_response.inspect}" }
  96. 158 request.transition(:headers)
  97. 158 request.transition(:body)
  98. 158 request.transition(:trailers)
  99. 158 request.transition(:done)
  100. 158 response.finish!
  101. 158 request.response = response
  102. 158 request.emit(:response, response)
  103. 158 request_signature.headers = request.headers.to_h
  104. 158 response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)
  105. 30 elsif WebMock.net_connect_allowed?(request_signature.uri)
  106. 24 if WebMock::CallbackRegistry.any_callbacks?
  107. 6 request.on(:response) do |resp|
  108. 6 unless resp.is_a?(HTTPX::ErrorResponse)
  109. 6 webmock_response = Plugin.build_webmock_response(request, resp)
  110. 6 WebMock::CallbackRegistry.invoke_callbacks(
  111. { lib: :httpx, real_request: true }, request_signature,
  112. webmock_response
  113. )
  114. end
  115. end
  116. end
  117. 24 @mocked = false
  118. 24 emit(:unmock_connection, self)
  119. 24 super
  120. else
  121. 6 raise WebMock::NetConnectNotAllowedError, request_signature
  122. end
  123. end
  124. end
  125. end
  126. 8 class HttpxAdapter < HttpLibAdapter
  127. 8 adapter_for :httpx
  128. 8 class << self
  129. 8 def enable!
  130. 370 @original_session ||= HTTPX::Session
  131. 370 webmock_session = HTTPX.plugin(Plugin)
  132. 370 HTTPX.send(:remove_const, :Session)
  133. 370 HTTPX.send(:const_set, :Session, webmock_session.class)
  134. end
  135. 8 def disable!
  136. 370 return unless @original_session
  137. 362 HTTPX.send(:remove_const, :Session)
  138. 362 HTTPX.send(:const_set, :Session, @original_session)
  139. end
  140. end
  141. end
  142. end
  143. end

lib/httpx/altsvc.rb

96.39% lines covered

83 relevant lines. 80 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "strscan"
  3. 27 module HTTPX
  4. 27 module AltSvc
  5. # makes connections able to accept requests destined to primary service.
  6. 27 module ConnectionMixin
  7. 27 using URIExtensions
  8. 27 def send(request)
  9. 8 request.headers["alt-used"] = @origin.authority if @parser && !@write_buffer.full? && match_altsvcs?(request.uri)
  10. 8 super
  11. end
  12. 27 def match?(uri, options)
  13. 8 return false if !used? && (@state == :closing || @state == :closed)
  14. 8 match_altsvcs?(uri) && match_altsvc_options?(uri, options)
  15. end
  16. 27 private
  17. # checks if this is connection is an alternative service of
  18. # +uri+
  19. 27 def match_altsvcs?(uri)
  20. 24 @origins.any? { |origin| altsvc_match?(uri, origin) } ||
  21. AltSvc.cached_altsvc(@origin).any? do |altsvc|
  22. origin = altsvc["origin"]
  23. altsvc_match?(origin, uri.origin)
  24. end
  25. end
  26. 27 def match_altsvc_options?(uri, options)
  27. 8 return @options == options unless @options.ssl.all? do |k, v|
  28. 8 v == (k == :hostname ? uri.host : options.ssl[k])
  29. end
  30. 8 @options.options_equals?(options, Options::REQUEST_BODY_IVARS + %i[@ssl])
  31. end
  32. 27 def altsvc_match?(uri, other_uri)
  33. 16 other_uri = URI(other_uri)
  34. 16 uri.origin == other_uri.origin || begin
  35. 7 case uri.scheme
  36. when "h2"
  37. (other_uri.scheme == "https" || other_uri.scheme == "h2") &&
  38. uri.host == other_uri.host &&
  39. uri.port == other_uri.port
  40. else
  41. 8 false
  42. end
  43. end
  44. end
  45. end
  46. 27 @altsvc_mutex = Thread::Mutex.new
  47. 48 @altsvcs = Hash.new { |h, k| h[k] = [] }
  48. 27 module_function
  49. 27 def cached_altsvc(origin)
  50. 40 now = Utils.now
  51. 40 @altsvc_mutex.synchronize do
  52. 40 lookup(origin, now)
  53. end
  54. end
  55. 27 def cached_altsvc_set(origin, entry)
  56. 24 now = Utils.now
  57. 24 @altsvc_mutex.synchronize do
  58. 24 return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
  59. 24 entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
  60. 24 @altsvcs[origin] << entry
  61. 24 entry
  62. end
  63. end
  64. 27 def lookup(origin, ttl)
  65. 40 return [] unless @altsvcs.key?(origin)
  66. 28 @altsvcs[origin] = @altsvcs[origin].select do |entry|
  67. 24 !entry.key?("TTL") || entry["TTL"] > ttl
  68. end
  69. 48 @altsvcs[origin].reject { |entry| entry["noop"] }
  70. end
  71. 27 def emit(request, response)
  72. 8006 return unless response.respond_to?(:headers)
  73. # Alt-Svc
  74. 7968 return unless response.headers.key?("alt-svc")
  75. 79 origin = request.origin
  76. 79 host = request.uri.host
  77. 79 altsvc = response.headers["alt-svc"]
  78. # https://datatracker.ietf.org/doc/html/rfc7838#section-3
  79. # A field value containing the special value "clear" indicates that the
  80. # origin requests all alternatives for that origin to be invalidated
  81. # (including those specified in the same response, in case of an
  82. # invalid reply containing both "clear" and alternative services).
  83. 79 if altsvc == "clear"
  84. 8 @altsvc_mutex.synchronize do
  85. 8 @altsvcs[origin].clear
  86. end
  87. 7 return
  88. end
  89. 71 parse(altsvc) do |alt_origin, alt_params|
  90. 8 alt_origin.host ||= host
  91. 8 yield(alt_origin, origin, alt_params)
  92. end
  93. end
  94. 27 def parse(altsvc)
  95. 183 return enum_for(__method__, altsvc) unless block_given?
  96. 127 scanner = StringScanner.new(altsvc)
  97. 133 until scanner.eos?
  98. 127 alt_service = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  99. 127 alt_params = []
  100. 127 loop do
  101. 151 alt_param = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
  102. 151 alt_params << alt_param.strip if alt_param
  103. 151 scanner.skip(/;/)
  104. 151 break if scanner.eos? || scanner.scan(/ *, */)
  105. end
  106. 254 alt_params = Hash[alt_params.map { |field| field.split("=", 2) }]
  107. 127 alt_proto, alt_authority = alt_service.split("=", 2)
  108. 127 alt_origin = parse_altsvc_origin(alt_proto, alt_authority)
  109. 127 return unless alt_origin
  110. 48 yield(alt_origin, alt_params.merge("proto" => alt_proto))
  111. end
  112. end
  113. 27 def parse_altsvc_scheme(alt_proto)
  114. 134 case alt_proto
  115. when "h2c"
  116. 8 "http"
  117. when "h2"
  118. 56 "https"
  119. end
  120. end
  121. 27 def parse_altsvc_origin(alt_proto, alt_origin)
  122. 127 alt_scheme = parse_altsvc_scheme(alt_proto)
  123. 127 return unless alt_scheme
  124. 48 alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
  125. 48 URI.parse("#{alt_scheme}://#{alt_origin}")
  126. end
  127. end
  128. end

lib/httpx/buffer.rb

100.0% lines covered

27 relevant lines. 27 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 module HTTPX
  4. # Internal class to abstract a string buffer, by wrapping a string and providing the
  5. # minimum possible API and functionality required.
  6. #
  7. # buffer = Buffer.new(640)
  8. # buffer.full? #=> false
  9. # buffer << "aa"
  10. # buffer.capacity #=> 638
  11. #
  12. 27 class Buffer
  13. 27 extend Forwardable
  14. 27 def_delegator :@buffer, :to_s
  15. 27 def_delegator :@buffer, :to_str
  16. 27 def_delegator :@buffer, :empty?
  17. 27 def_delegator :@buffer, :bytesize
  18. 27 def_delegator :@buffer, :clear
  19. 27 def_delegator :@buffer, :replace
  20. 27 attr_reader :limit
  21. 27 if RUBY_VERSION >= "3.4.0"
  22. 15 def initialize(limit)
  23. 2920 @buffer = String.new("", encoding: Encoding::BINARY, capacity: limit)
  24. 2920 @limit = limit
  25. end
  26. 15 def <<(chunk)
  27. 12241 @buffer.append_as_bytes(chunk)
  28. end
  29. else
  30. 12 def initialize(limit)
  31. 18990 @buffer = "".b
  32. 18990 @limit = limit
  33. end
  34. 12 def_delegator :@buffer, :<<
  35. end
  36. 27 def full?
  37. 10778471 @buffer.bytesize >= @limit
  38. end
  39. 27 def capacity
  40. 12 @limit - @buffer.bytesize
  41. end
  42. 27 def shift!(fin)
  43. 22082 @buffer = @buffer.byteslice(fin..-1) || "".b
  44. end
  45. end
  46. end

lib/httpx/callbacks.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Callbacks
  4. 27 def on(type, &action)
  5. 354423 callbacks(type) << action
  6. 354423 action
  7. end
  8. 27 def once(type, &block)
  9. 192536 on(type) do |*args, &callback|
  10. 99145 block.call(*args, &callback)
  11. 99081 :delete
  12. end
  13. end
  14. 27 def emit(type, *args)
  15. 129569 log { "emit #{type.inspect} callbacks" } if respond_to?(:log)
  16. 271405 callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
  17. end
  18. 27 def callbacks_for?(type)
  19. 3234 @callbacks.key?(type) && @callbacks[type].any?
  20. end
  21. 27 protected
  22. 27 def callbacks(type = nil)
  23. 487347 return @callbacks unless type
  24. 702734 @callbacks ||= Hash.new { |h, k| h[k] = [] }
  25. 487260 @callbacks[type]
  26. end
  27. end
  28. end

lib/httpx/chainable.rb

95.65% lines covered

46 relevant lines. 44 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Session mixin, implements most of the APIs that the users call.
  4. # delegates to a default session when extended.
  5. 27 module Chainable
  6. 27 %w[head get post put delete trace options connect patch].each do |meth|
  7. 234 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  8. 9 def #{meth}(*uri, **options) # def get(*uri, **options)
  9. 18 request("#{meth.upcase}", uri, **options) # request("GET", uri, **options)
  10. end # end
  11. MOD
  12. end
  13. # delegates to the default session (see HTTPX::Session#request).
  14. 27 def request(*args, **options)
  15. 2670 branch(default_options).request(*args, **options)
  16. end
  17. 27 def accept(type)
  18. 16 with(headers: { "accept" => String(type) })
  19. end
  20. # delegates to the default session (see HTTPX::Session#wrap).
  21. 27 def wrap(&blk)
  22. 89 branch(default_options).wrap(&blk)
  23. end
  24. # returns a new instance loaded with the +pl+ plugin and +options+.
  25. 27 def plugin(pl, options = nil, &blk)
  26. 5271 klass = is_a?(S) ? self.class : Session
  27. 5271 klass = Class.new(klass)
  28. 5271 klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
  29. 5271 klass.plugin(pl, options, &blk).new
  30. end
  31. # returns a new instance loaded with +options+.
  32. 27 def with(options, &blk)
  33. 2761 branch(default_options.merge(options), &blk)
  34. end
  35. 27 private
  36. # returns default instance of HTTPX::Options.
  37. 27 def default_options
  38. 10847 @options || Session.default_options
  39. end
  40. # returns a default instance of HTTPX::Session.
  41. 27 def branch(options, &blk)
  42. 5504 return self.class.new(options, &blk) if is_a?(S)
  43. 3211 Session.new(options, &blk)
  44. end
  45. 27 def method_missing(meth, *args, **options, &blk)
  46. 652 case meth
  47. when /\Awith_(.+)/
  48. 729 option = Regexp.last_match(1)
  49. 729 return super unless option
  50. 729 with(option.to_sym => args.first || options)
  51. when /\Aon_(.+)/
  52. 9 callback = Regexp.last_match(1)
  53. 6 return super unless %w[
  54. connection_opened connection_closed
  55. request_error
  56. request_started request_body_chunk request_completed
  57. response_started response_body_chunk response_completed
  58. 2 ].include?(callback)
  59. 9 warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
  60. 1 "Use `HTTPX.plugin(:callbacks).#{meth}` instead."
  61. 9 plugin(:callbacks).__send__(meth, *args, **options, &blk)
  62. else
  63. super
  64. end
  65. end
  66. 27 def respond_to_missing?(meth, *)
  67. 49 case meth
  68. when /\Awith_(.+)/
  69. 40 option = Regexp.last_match(1)
  70. 40 default_options.respond_to?(option) || super
  71. when /\Aon_(.+)/
  72. 16 callback = Regexp.last_match(1)
  73. 12 %w[
  74. connection_opened connection_closed
  75. request_error
  76. request_started request_body_chunk request_completed
  77. response_started response_body_chunk response_completed
  78. 3 ].include?(callback) || super
  79. else
  80. super
  81. end
  82. end
  83. end
  84. 27 extend Chainable
  85. end

lib/httpx/connection.rb

92.18% lines covered

486 relevant lines. 448 lines covered and 38 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "forwardable"
  4. 27 require "httpx/io"
  5. 27 require "httpx/buffer"
  6. 27 module HTTPX
  7. # The Connection can be watched for IO events.
  8. #
  9. # It contains the +io+ object to read/write from, and knows what to do when it can.
  10. #
  11. # It defers connecting until absolutely necessary. Connection should be triggered from
  12. # the IO selector (until then, any request will be queued).
  13. #
  14. # A connection boots up its parser after connection is established. All pending requests
  15. # will be redirected there after connection.
  16. #
  17. # A connection can be prevented from closing by the parser, that is, if there are pending
  18. # requests. This will signal that the connection was prematurely closed, due to a possible
  19. # number of conditions:
  20. #
  21. # * Remote peer closed the connection ("Connection: close");
  22. # * Remote peer doesn't support pipelining;
  23. #
  24. # A connection may also route requests for a different host for which the +io+ was connected
  25. # to, provided that the IP is the same and the port and scheme as well. This will allow to
  26. # share the same socket to send HTTP/2 requests to different hosts.
  27. #
  28. 27 class Connection
  29. 27 extend Forwardable
  30. 27 include Loggable
  31. 27 include Callbacks
  32. 27 using URIExtensions
  33. 27 require "httpx/connection/http2"
  34. 27 require "httpx/connection/http1"
  35. 27 def_delegator :@io, :closed?
  36. 27 def_delegator :@write_buffer, :empty?
  37. 27 attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
  38. 27 attr_writer :current_selector
  39. 27 attr_accessor :current_session, :family
  40. 27 protected :sibling
  41. 27 def initialize(uri, options)
  42. 7425 @current_session = @current_selector = @sibling = @coalesced_connection = nil
  43. 7425 @exhausted = @cloned = @main_sibling = false
  44. 7425 @options = Options.new(options)
  45. 7425 @type = initialize_type(uri, @options)
  46. 7425 @origins = [uri.origin]
  47. 7425 @origin = Utils.to_uri(uri.origin)
  48. 7425 @window_size = @options.window_size
  49. 7425 @read_buffer = Buffer.new(@options.buffer_size)
  50. 7425 @write_buffer = Buffer.new(@options.buffer_size)
  51. 7425 @pending = []
  52. 7425 on(:error, &method(:on_error))
  53. 7425 if @options.io
  54. # if there's an already open IO, get its
  55. # peer address, and force-initiate the parser
  56. 66 transition(:already_open)
  57. 66 @io = build_socket
  58. 66 parser
  59. else
  60. 7359 transition(:idle)
  61. end
  62. 7425 on(:close) do
  63. 7583 next if @exhausted # it'll reset
  64. # may be called after ":close" above, so after the connection has been checked back in.
  65. # next unless @current_session
  66. 7575 next unless @current_session
  67. 7575 @current_session.deselect_connection(self, @current_selector, @cloned)
  68. end
  69. 7425 on(:terminate) do
  70. 2819 next if @exhausted # it'll reset
  71. 2811 current_session = @current_session
  72. 2811 current_selector = @current_selector
  73. # may be called after ":close" above, so after the connection has been checked back in.
  74. 2811 next unless current_session && current_selector
  75. 17 current_session.deselect_connection(self, current_selector)
  76. end
  77. 7425 on(:altsvc) do |alt_origin, origin, alt_params|
  78. 8 build_altsvc_connection(alt_origin, origin, alt_params)
  79. end
  80. 7425 @inflight = 0
  81. 7425 @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
  82. 7425 self.addresses = @options.addresses if @options.addresses
  83. end
  84. 27 def peer
  85. 15068 @origin
  86. end
  87. # this is a semi-private method, to be used by the resolver
  88. # to initiate the io object.
  89. 27 def addresses=(addrs)
  90. 7212 if @io
  91. 258 @io.add_addresses(addrs)
  92. else
  93. 6954 @io = build_socket(addrs)
  94. end
  95. end
  96. 27 def addresses
  97. 14825 @io && @io.addresses
  98. end
  99. 27 def match?(uri, options)
  100. 2092 return false if !used? && (@state == :closing || @state == :closed)
  101. 310 (
  102. 1820 @origins.include?(uri.origin) &&
  103. # if there is more than one origin to match, it means that this connection
  104. # was the result of coalescing. To prevent blind trust in the case where the
  105. # origin came from an ORIGIN frame, we're going to verify the hostname with the
  106. # SSL certificate
  107. 1886 (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
  108. ) && @options == options
  109. end
  110. 27 def expired?
  111. return false unless @io
  112. @io.expired?
  113. end
  114. 27 def mergeable?(connection)
  115. 318 return false if @state == :closing || @state == :closed || !@io
  116. 64 return false unless connection.addresses
  117. 2 (
  118. 64 (open? && @origin == connection.origin) ||
  119. 64 !(@io.addresses & (connection.addresses || [])).empty?
  120. ) && @options == connection.options
  121. end
  122. # coalesces +self+ into +connection+.
  123. 27 def coalesce!(connection)
  124. 13 @coalesced_connection = connection
  125. 13 close_sibling
  126. 13 connection.merge(self)
  127. end
  128. # coalescable connections need to be mergeable!
  129. # but internally, #mergeable? is called before #coalescable?
  130. 27 def coalescable?(connection)
  131. 27 if @io.protocol == "h2" &&
  132. @origin.scheme == "https" &&
  133. connection.origin.scheme == "https" &&
  134. @io.can_verify_peer?
  135. 13 @io.verify_hostname(connection.origin.host)
  136. else
  137. 14 @origin == connection.origin
  138. end
  139. end
  140. 27 def create_idle(options = {})
  141. self.class.new(@origin, @options.merge(options))
  142. end
  143. 27 def merge(connection)
  144. 34 @origins |= connection.instance_variable_get(:@origins)
  145. 37 if connection.ssl_session
  146. 14 @ssl_session = connection.ssl_session
  147. 2 @io.session_new_cb do |sess|
  148. 21 @ssl_session = sess
  149. 14 end if @io
  150. end
  151. 37 connection.purge_pending do |req|
  152. 7 send(req)
  153. end
  154. end
  155. 27 def purge_pending(&block)
  156. 37 pendings = []
  157. 37 if @parser
  158. 21 @inflight -= @parser.pending.size
  159. 24 pendings << @parser.pending
  160. end
  161. 37 pendings << @pending
  162. 37 pendings.each do |pending|
  163. 61 pending.reject!(&block)
  164. end
  165. end
  166. 27 def io_connected?
  167. return @coalesced_connection.io_connected? if @coalesced_connection
  168. @io && @io.state == :connected
  169. end
  170. 27 def connecting?
  171. 10813746 @state == :idle
  172. end
  173. 27 def inflight?
  174. 2931 @parser && (
  175. # parser may be dealing with other requests (possibly started from a different fiber)
  176. 2461 !@parser.empty? ||
  177. # connection may be doing connection termination handshake
  178. !@write_buffer.empty?
  179. )
  180. end
  181. 27 def interests
  182. # connecting
  183. 10802025 if connecting?
  184. 11407 connect
  185. 11405 return @io.interests if connecting?
  186. end
  187. # if the write buffer is full, we drain it
  188. 10791230 return :w unless @write_buffer.empty?
  189. 10749410 return @parser.interests if @parser
  190. 22 nil
  191. rescue StandardError => e
  192. emit(:error, e)
  193. nil
  194. end
  195. 27 def to_io
  196. 23216 @io.to_io
  197. end
  198. 27 def call
  199. 20314 case @state
  200. when :idle
  201. 10640 connect
  202. 10626 consume
  203. when :closed
  204. return
  205. when :closing
  206. consume
  207. transition(:closed)
  208. when :open
  209. 11347 consume
  210. end
  211. 8629 nil
  212. rescue StandardError => e
  213. 21 @write_buffer.clear
  214. 21 emit(:error, e)
  215. 21 raise e
  216. end
  217. 27 def close
  218. 2769 transition(:active) if @state == :inactive
  219. 2769 @parser.close if @parser
  220. end
  221. 27 def terminate
  222. 2445 case @state
  223. when :idle
  224. purge_after_closed
  225. emit(:terminate)
  226. when :closed
  227. 8 @connected_at = nil
  228. end
  229. 2769 close
  230. end
  231. # bypasses the state machine to force closing of connections still connecting.
  232. # **only** used for Happy Eyeballs v2.
  233. 27 def force_reset(cloned = false)
  234. 292 @state = :closing
  235. 292 @cloned = cloned
  236. 292 transition(:closed)
  237. end
  238. 27 def reset
  239. 7499 return if @state == :closing || @state == :closed
  240. 7440 transition(:closing)
  241. 7440 transition(:closed)
  242. end
  243. 27 def send(request)
  244. 9017 return @coalesced_connection.send(request) if @coalesced_connection
  245. 9011 if @parser && !@write_buffer.full?
  246. 442 if @response_received_at && @keep_alive_timeout &&
  247. Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
  248. # when pushing a request into an existing connection, we have to check whether there
  249. # is the possibility that the connection might have extended the keep alive timeout.
  250. # for such cases, we want to ping for availability before deciding to shovel requests.
  251. 20 log(level: 3) { "keep alive timeout expired, pinging connection..." }
  252. 20 @pending << request
  253. 20 transition(:active) if @state == :inactive
  254. 20 parser.ping
  255. 20 request.ping!
  256. 18 return
  257. end
  258. 422 send_request_to_parser(request)
  259. else
  260. 8569 @pending << request
  261. end
  262. end
  263. 27 def timeout
  264. 10676116 return if @state == :closed || @state == :inactive
  265. 10676116 return @timeout if @timeout
  266. 10663208 return @options.timeout[:connect_timeout] if @state == :idle
  267. 10663208 @options.timeout[:operation_timeout]
  268. end
  269. 27 def idling
  270. 810 purge_after_closed
  271. 810 @write_buffer.clear
  272. 810 transition(:idle)
  273. 810 @parser = nil if @parser
  274. end
  275. 27 def used?
  276. 2311 @connected_at
  277. end
  278. 27 def deactivate
  279. 369 transition(:inactive)
  280. end
  281. 27 def open?
  282. 7241 @state == :open || @state == :inactive
  283. end
  284. 27 def handle_socket_timeout(interval)
  285. 32 error = OperationTimeoutError.new(interval, "timed out while waiting on select")
  286. 32 error.set_backtrace(caller)
  287. 32 on_error(error)
  288. end
  289. 27 def sibling=(connection)
  290. 24 @sibling = connection
  291. 24 return unless connection
  292. @main_sibling = connection.sibling.nil?
  293. return unless @main_sibling
  294. connection.sibling = self
  295. end
  296. 27 def handle_connect_error(error)
  297. 284 @connect_error = error
  298. 284 return handle_error(error) unless @sibling && @sibling.connecting?
  299. @sibling.merge(self)
  300. force_reset(true)
  301. end
  302. 27 def disconnect
  303. 7718 return unless @current_session && @current_selector
  304. 7583 emit(:close)
  305. 7567 @current_session = nil
  306. 7567 @current_selector = nil
  307. end
  308. skipped # :nocov:
  309. skipped def inspect
  310. skipped "#<#{self.class}:#{object_id} " \
  311. skipped "@origin=#{@origin} " \
  312. skipped "@state=#{@state} " \
  313. skipped "@pending=#{@pending.size} " \
  314. skipped "@io=#{@io}>"
  315. skipped end
  316. skipped # :nocov:
  317. 27 private
  318. 27 def connect
  319. 21039 transition(:open)
  320. end
  321. 27 def consume
  322. 25202 return unless @io
  323. 25202 catch(:called) do
  324. 25202 epiped = false
  325. 25202 loop do
  326. # connection may have
  327. 45632 return if @state == :idle
  328. 42321 parser.consume
  329. # we exit if there's no more requests to process
  330. #
  331. # this condition takes into account:
  332. #
  333. # * the number of inflight requests
  334. # * the number of pending requests
  335. # * whether the write buffer has bytes (i.e. for close handshake)
  336. 42305 if @pending.empty? && @inflight.zero? && @write_buffer.empty?
  337. 2902 log(level: 3) { "NO MORE REQUESTS..." }
  338. 2886 return
  339. end
  340. 39419 @timeout = @current_timeout
  341. 39419 read_drained = false
  342. 39419 write_drained = nil
  343. #
  344. # tight read loop.
  345. #
  346. # read as much of the socket as possible.
  347. #
  348. # this tight loop reads all the data it can from the socket and pipes it to
  349. # its parser.
  350. #
  351. 7597 loop do
  352. 51117 siz = @io.read(@window_size, @read_buffer)
  353. 51247 log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
  354. 51117 unless siz
  355. 22 @write_buffer.clear
  356. 22 ex = EOFError.new("descriptor closed")
  357. 22 ex.set_backtrace(caller)
  358. 22 on_error(ex)
  359. 22 return
  360. end
  361. # socket has been drained. mark and exit the read loop.
  362. 51095 if siz.zero?
  363. 10765 read_drained = @read_buffer.empty?
  364. 10765 epiped = false
  365. 10765 break
  366. end
  367. 40330 parser << @read_buffer.to_s
  368. # continue reading if possible.
  369. 35909 break if interests == :w && !epiped
  370. # exit the read loop if connection is preparing to be closed
  371. 28438 break if @state == :closing || @state == :closed
  372. # exit #consume altogether if all outstanding requests have been dealt with
  373. 28423 return if @pending.empty? && @inflight.zero?
  374. 39419 end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
  375. #
  376. # tight write loop.
  377. #
  378. # flush as many bytes as the sockets allow.
  379. #
  380. 6171 loop do
  381. # buffer has been drainned, mark and exit the write loop.
  382. 24169 if @write_buffer.empty?
  383. # we only mark as drained on the first loop
  384. 2844 write_drained = write_drained.nil? && @inflight.positive?
  385. 2844 break
  386. end
  387. 2086 begin
  388. 21325 siz = @io.write(@write_buffer)
  389. rescue Errno::EPIPE
  390. # this can happen if we still have bytes in the buffer to send to the server, but
  391. # the server wants to respond immediately with some message, or an error. An example is
  392. # when one's uploading a big file to an unintended endpoint, and the server stops the
  393. # consumption, and responds immediately with an authorization of even method not allowed error.
  394. # at this point, we have to let the connection switch to read-mode.
  395. 17 log(level: 2) { "pipe broken, could not flush buffer..." }
  396. 17 epiped = true
  397. 17 read_drained = false
  398. 17 break
  399. end
  400. 21396 log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
  401. 21301 unless siz
  402. @write_buffer.clear
  403. ex = EOFError.new("descriptor closed")
  404. ex.set_backtrace(caller)
  405. on_error(ex)
  406. return
  407. end
  408. # socket closed for writing. mark and exit the write loop.
  409. 21301 if siz.zero?
  410. 16 write_drained = !@write_buffer.empty?
  411. 16 break
  412. end
  413. # exit write loop if marked to consume from peer, or is closing.
  414. 21285 break if interests == :r || @state == :closing || @state == :closed
  415. 3077 write_drained = false
  416. 31856 end unless (ints = interests) == :r
  417. 31849 send_pending if @state == :open
  418. # return if socket is drained
  419. 31849 next unless (ints != :r || read_drained) && (ints != :w || write_drained)
  420. # gotta go back to the event loop. It happens when:
  421. #
  422. # * the socket is drained of bytes or it's not the interest of the conn to read;
  423. # * theres nothing more to write, or it's not in the interest of the conn to write;
  424. 11462 log(level: 3) { "(#{ints}): WAITING FOR EVENTS..." }
  425. 11419 return
  426. end
  427. end
  428. end
  429. 27 def send_pending
  430. 82252 while !@write_buffer.full? && (request = @pending.shift)
  431. 18723 send_request_to_parser(request)
  432. end
  433. end
  434. 27 def parser
  435. 110465 @parser ||= build_parser
  436. end
  437. 27 def send_request_to_parser(request)
  438. 18217 @inflight += 1
  439. 19145 request.peer_address = @io.ip
  440. 19145 set_request_timeouts(request)
  441. 19145 parser.send(request)
  442. 19145 return unless @state == :inactive
  443. 9 transition(:active)
  444. # mark request as ping, as this inactive connection may have been
  445. # closed by the server, and we don't want that to influence retry
  446. # bookkeeping.
  447. 9 request.ping!
  448. end
  449. 27 def build_parser(protocol = @io.protocol)
  450. 7487 parser = parser_type(protocol).new(@write_buffer, @options)
  451. 7487 set_parser_callbacks(parser)
  452. 7487 parser
  453. end
  454. 27 def set_parser_callbacks(parser)
  455. 7597 parser.on(:response) do |request, response|
  456. 7998 AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
  457. 8 emit(:altsvc, alt_origin, origin, alt_params)
  458. end
  459. 7998 @response_received_at = Utils.now
  460. 7148 @inflight -= 1
  461. 7998 response.finish!
  462. 7998 request.emit(:response, response)
  463. end
  464. 7597 parser.on(:altsvc) do |alt_origin, origin, alt_params|
  465. emit(:altsvc, alt_origin, origin, alt_params)
  466. end
  467. 7597 parser.on(:pong, &method(:send_pending))
  468. 7597 parser.on(:promise) do |request, stream|
  469. 24 request.emit(:promise, parser, stream)
  470. end
  471. 7597 parser.on(:exhausted) do
  472. 8 @exhausted = true
  473. 8 current_session = @current_session
  474. 8 current_selector = @current_selector
  475. begin
  476. 8 parser.close
  477. 8 @pending.concat(parser.pending)
  478. ensure
  479. 8 @current_session = current_session
  480. 8 @current_selector = current_selector
  481. end
  482. 7 case @state
  483. when :closed
  484. 8 idling
  485. 8 @exhausted = false
  486. when :closing
  487. once(:closed) do
  488. idling
  489. @exhausted = false
  490. end
  491. end
  492. end
  493. 7597 parser.on(:origin) do |origin|
  494. @origins |= [origin]
  495. end
  496. 7597 parser.on(:close) do |force|
  497. 2819 if force
  498. 2819 reset
  499. 2811 emit(:terminate)
  500. end
  501. end
  502. 7597 parser.on(:close_handshake) do
  503. 8 consume
  504. end
  505. 7597 parser.on(:reset) do
  506. 4039 @pending.concat(parser.pending) unless parser.empty?
  507. 4039 current_session = @current_session
  508. 4039 current_selector = @current_selector
  509. 4039 reset
  510. 4031 unless @pending.empty?
  511. 179 idling
  512. 179 @current_session = current_session
  513. 179 @current_selector = current_selector
  514. end
  515. end
  516. 7597 parser.on(:current_timeout) do
  517. 3210 @current_timeout = @timeout = parser.timeout
  518. end
  519. 7597 parser.on(:timeout) do |tout|
  520. 2751 @timeout = tout
  521. end
  522. 7597 parser.on(:error) do |request, error|
  523. 60 case error
  524. when :http_1_1_required
  525. 16 current_session = @current_session
  526. 16 current_selector = @current_selector
  527. 16 parser.close
  528. 16 other_connection = current_session.find_connection(@origin, current_selector,
  529. @options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
  530. 16 other_connection.merge(self)
  531. 16 request.transition(:idle)
  532. 16 other_connection.send(request)
  533. 16 next
  534. when OperationTimeoutError
  535. # request level timeouts should take precedence
  536. next unless request.active_timeouts.empty?
  537. end
  538. 46 @inflight -= 1
  539. 51 response = ErrorResponse.new(request, error)
  540. 51 request.response = response
  541. 51 request.emit(:response, response)
  542. end
  543. end
  544. 27 def transition(nextstate)
  545. 46431 handle_transition(nextstate)
  546. rescue Errno::ECONNABORTED,
  547. Errno::ECONNREFUSED,
  548. Errno::ECONNRESET,
  549. Errno::EADDRNOTAVAIL,
  550. Errno::EHOSTUNREACH,
  551. Errno::EINVAL,
  552. Errno::ENETUNREACH,
  553. Errno::EPIPE,
  554. Errno::ENOENT,
  555. SocketError,
  556. IOError => e
  557. # connect errors, exit gracefully
  558. 81 error = ConnectionError.new(e.message)
  559. 81 error.set_backtrace(e.backtrace)
  560. 81 handle_connect_error(error) if connecting?
  561. 81 @state = :closed
  562. 81 purge_after_closed
  563. 81 disconnect
  564. rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
  565. # connect errors, exit gracefully
  566. 24 handle_error(e)
  567. 24 handle_connect_error(e) if connecting?
  568. 24 @state = :closed
  569. 24 purge_after_closed
  570. 24 disconnect
  571. end
  572. 27 def handle_transition(nextstate)
  573. 41056 case nextstate
  574. when :idle
  575. 8184 @timeout = @current_timeout = @options.timeout[:connect_timeout]
  576. 8184 @connected_at = @response_received_at = nil
  577. when :open
  578. 21335 return if @state == :closed
  579. 21335 @io.connect
  580. 21230 close_sibling if @io.state == :connected
  581. 21230 return unless @io.connected?
  582. 7493 @connected_at = Utils.now
  583. 7493 send_pending
  584. 7493 @timeout = @current_timeout = parser.timeout
  585. 7493 emit(:open)
  586. when :inactive
  587. 369 return unless @state == :open
  588. # do not deactivate connection in use
  589. 368 return if @inflight.positive?
  590. when :closing
  591. 7448 return unless @state == :idle || @state == :open
  592. 7448 unless @write_buffer.empty?
  593. # preset state before handshake, as error callbacks
  594. # may take it back here.
  595. 2759 @state = nextstate
  596. # handshakes, try sending
  597. 2759 consume
  598. 2759 @write_buffer.clear
  599. 2759 return
  600. end
  601. when :closed
  602. 7740 return unless @state == :closing
  603. 7740 return unless @write_buffer.empty?
  604. 7715 purge_after_closed
  605. 7715 disconnect if @pending.empty?
  606. when :already_open
  607. 66 nextstate = :open
  608. # the first check for given io readiness must still use a timeout.
  609. # connect is the reasonable choice in such a case.
  610. 66 @timeout = @options.timeout[:connect_timeout]
  611. 66 send_pending
  612. when :active
  613. 200 return unless @state == :inactive
  614. 200 nextstate = :open
  615. # activate
  616. 200 @current_session.select_connection(self, @current_selector)
  617. end
  618. 29268 @state = nextstate
  619. end
  620. 27 def close_sibling
  621. 9851 return unless @sibling
  622. if @sibling.io_connected?
  623. reset
  624. # TODO: transition connection to closed
  625. end
  626. unless @sibling.state == :closed
  627. merge(@sibling) unless @main_sibling
  628. @sibling.force_reset(true)
  629. end
  630. @sibling = nil
  631. end
  632. 27 def purge_after_closed
  633. 8637 @io.close if @io
  634. 8637 @read_buffer.clear
  635. 8637 @timeout = nil
  636. end
  637. 27 def initialize_type(uri, options)
  638. 7059 options.transport || begin
  639. 6255 case uri.scheme
  640. when "http"
  641. 4021 "tcp"
  642. when "https"
  643. 3010 "ssl"
  644. else
  645. raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
  646. end
  647. end
  648. end
  649. # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
  650. 27 def build_altsvc_connection(alt_origin, origin, alt_params)
  651. # do not allow security downgrades on altsvc negotiation
  652. 8 return if @origin.scheme == "https" && alt_origin.scheme != "https"
  653. 8 altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
  654. # altsvc already exists, somehow it wasn't advertised, probably noop
  655. 8 return unless altsvc
  656. 8 alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
  657. 8 connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
  658. # advertised altsvc is the same origin being used, ignore
  659. 8 return if connection == self
  660. 8 connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
  661. 8 log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
  662. 8 connection.merge(self)
  663. 8 terminate
  664. rescue UnsupportedSchemeError
  665. altsvc["noop"] = true
  666. nil
  667. end
  668. 27 def build_socket(addrs = nil)
  669. 6216 case @type
  670. when "tcp"
  671. 4077 TCP.new(peer, addrs, @options)
  672. when "ssl"
  673. 2915 SSL.new(peer, addrs, @options) do |sock|
  674. 2893 sock.ssl_session = @ssl_session
  675. 2893 sock.session_new_cb do |sess|
  676. 5013 @ssl_session = sess
  677. 5013 sock.ssl_session = sess
  678. end
  679. end
  680. when "unix"
  681. 28 path = Array(addrs).first
  682. 28 path = String(path) if path
  683. 28 UNIX.new(peer, path, @options)
  684. else
  685. raise Error, "unsupported transport (#{@type})"
  686. end
  687. end
  688. 27 def on_error(error, request = nil)
  689. 617 if error.is_a?(OperationTimeoutError)
  690. # inactive connections do not contribute to the select loop, therefore
  691. # they should not fail due to such errors.
  692. 32 return if @state == :inactive
  693. 32 if @timeout
  694. 28 @timeout -= error.timeout
  695. 32 return unless @timeout <= 0
  696. end
  697. 32 error = error.to_connection_error if connecting?
  698. end
  699. 617 handle_error(error, request)
  700. 617 reset
  701. end
  702. 27 def handle_error(error, request = nil)
  703. 925 parser.handle_error(error, request) if @parser && parser.respond_to?(:handle_error)
  704. 2041 while (req = @pending.shift)
  705. 436 next if request && req == request
  706. 436 response = ErrorResponse.new(req, error)
  707. 436 req.response = response
  708. 420 req.emit(:response, response)
  709. end
  710. 909 return unless request
  711. 363 @inflight -= 1
  712. 414 response = ErrorResponse.new(request, error)
  713. 414 request.response = response
  714. 414 request.emit(:response, response)
  715. end
  716. 27 def set_request_timeouts(request)
  717. 19145 set_request_write_timeout(request)
  718. 19145 set_request_read_timeout(request)
  719. 19145 set_request_request_timeout(request)
  720. end
  721. 27 def set_request_read_timeout(request)
  722. 19145 read_timeout = request.read_timeout
  723. 19145 return if read_timeout.nil? || read_timeout.infinite?
  724. 18831 set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
  725. 16 read_timeout_callback(request, read_timeout)
  726. end
  727. end
  728. 27 def set_request_write_timeout(request)
  729. 19145 write_timeout = request.write_timeout
  730. 19145 return if write_timeout.nil? || write_timeout.infinite?
  731. 19145 set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
  732. 16 write_timeout_callback(request, write_timeout)
  733. end
  734. end
  735. 27 def set_request_request_timeout(request)
  736. 18861 request_timeout = request.request_timeout
  737. 18861 return if request_timeout.nil? || request_timeout.infinite?
  738. 549 set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
  739. 382 read_timeout_callback(request, request_timeout, RequestTimeoutError)
  740. end
  741. end
  742. 27 def write_timeout_callback(request, write_timeout)
  743. 16 return if request.state == :done
  744. 16 @write_buffer.clear
  745. 16 error = WriteTimeoutError.new(request, nil, write_timeout)
  746. 16 on_error(error, request)
  747. end
  748. 27 def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
  749. 398 response = request.response
  750. 398 return if response && response.finished?
  751. 398 @write_buffer.clear
  752. 398 error = error_type.new(request, request.response, read_timeout)
  753. 398 on_error(error, request)
  754. end
  755. 27 def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
  756. 38605 request.set_timeout_callback(start_event) do
  757. 38405 timer = @current_selector.after(timeout, callback)
  758. 38405 request.active_timeouts << label
  759. 38405 Array(finish_events).each do |event|
  760. # clean up request timeouts if the connection errors out
  761. 57542 request.set_timeout_callback(event) do
  762. 57084 timer.cancel
  763. 57084 request.active_timeouts.delete(label)
  764. end
  765. end
  766. end
  767. end
  768. 27 def parser_type(protocol)
  769. 6747 case protocol
  770. 3209 when "h2" then HTTP2
  771. 4406 when "http/1.1" then HTTP1
  772. else
  773. raise Error, "unsupported protocol (##{protocol})"
  774. end
  775. end
  776. end
  777. end

lib/httpx/connection/http1.rb

89.14% lines covered

221 relevant lines. 197 lines covered and 24 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "httpx/parser/http1"
  3. 27 module HTTPX
  4. 27 class Connection::HTTP1
  5. 27 include Callbacks
  6. 27 include Loggable
  7. 27 MAX_REQUESTS = 200
  8. 27 CRLF = "\r\n"
  9. 27 attr_reader :pending, :requests
  10. 27 attr_accessor :max_concurrent_requests
  11. 27 def initialize(buffer, options)
  12. 4406 @options = options
  13. 4406 @max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
  14. 4406 @max_requests = @options.max_requests
  15. 4406 @parser = Parser::HTTP1.new(self)
  16. 4406 @buffer = buffer
  17. 4406 @version = [1, 1]
  18. 4406 @pending = []
  19. 4406 @requests = []
  20. 4406 @handshake_completed = false
  21. end
  22. 27 def timeout
  23. 4289 @options.timeout[:operation_timeout]
  24. end
  25. 27 def interests
  26. # this means we're processing incoming response already
  27. 30326 return :r if @request
  28. 25258 return if @requests.empty?
  29. 25237 request = @requests.first
  30. 25237 return unless request
  31. 25237 return :w if request.interests == :w || !@buffer.empty?
  32. 22307 :r
  33. end
  34. 27 def reset
  35. 4168 @max_requests = @options.max_requests || MAX_REQUESTS
  36. 4168 @parser.reset!
  37. 4168 @handshake_completed = false
  38. 4168 @pending.concat(@requests) unless @requests.empty?
  39. end
  40. 27 def close
  41. 82 reset
  42. 82 emit(:close, true)
  43. end
  44. 27 def exhausted?
  45. 529 !@max_requests.positive?
  46. end
  47. 27 def empty?
  48. # this means that for every request there's an available
  49. # partial response, so there are no in-flight requests waiting.
  50. 4088 @requests.empty? || (
  51. # checking all responses can be time-consuming. Alas, as in HTTP/1, responses
  52. # do not come out of order, we can get away with checking first and last.
  53. 185 !@requests.first.response.nil? &&
  54. (@requests.size == 1 || !@requests.last.response.nil?)
  55. )
  56. end
  57. 27 def <<(data)
  58. 7099 @parser << data
  59. end
  60. 27 def send(request)
  61. 15503 unless @max_requests.positive?
  62. @pending << request
  63. return
  64. end
  65. 15503 return if @requests.include?(request)
  66. 15503 @requests << request
  67. 15503 @pipelining = true if @requests.size > 1
  68. end
  69. 27 def consume
  70. 16077 requests_limit = [@max_requests, @requests.size].min
  71. 16077 concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
  72. 16077 @requests.each_with_index do |request, idx|
  73. 18654 break if idx >= concurrent_requests_limit
  74. 16051 next unless request.can_buffer?
  75. 6140 handle(request)
  76. end
  77. end
  78. # HTTP Parser callbacks
  79. #
  80. # must be public methods, or else they won't be reachable
  81. 27 def on_start
  82. 4800 log(level: 2) { "parsing begins" }
  83. end
  84. 27 def on_headers(h)
  85. 4768 @request = @requests.first
  86. 4768 return if @request.response
  87. 4800 log(level: 2) { "headers received" }
  88. 4768 headers = @request.options.headers_class.new(h)
  89. 4768 response = @request.options.response_class.new(@request,
  90. @parser.status_code,
  91. @parser.http_version.join("."),
  92. headers)
  93. 4800 log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
  94. 5056 log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{log_redact(v)}" }.join("\n") }
  95. 4768 @request.response = response
  96. 4760 on_complete if response.finished?
  97. end
  98. 27 def on_trailers(h)
  99. 8 return unless @request
  100. 8 response = @request.response
  101. 8 log(level: 2) { "trailer headers received" }
  102. 8 log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{log_redact(v.join(", "))}" }.join("\n") }
  103. 8 response.merge_headers(h)
  104. end
  105. 27 def on_data(chunk)
  106. 5539 request = @request
  107. 5539 return unless request
  108. 5571 log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
  109. 5571 log(level: 2, color: :green) { "-> #{log_redact(chunk.inspect)}" }
  110. 5539 response = request.response
  111. 5539 response << chunk
  112. rescue StandardError => e
  113. 14 error_response = ErrorResponse.new(request, e)
  114. 14 request.response = error_response
  115. 14 dispatch
  116. end
  117. 27 def on_complete
  118. 4730 request = @request
  119. 4730 return unless request
  120. 4762 log(level: 2) { "parsing complete" }
  121. 4730 dispatch
  122. end
  123. 27 def dispatch
  124. 4744 request = @request
  125. 4744 if request.expects?
  126. 72 @parser.reset!
  127. 63 return handle(request)
  128. end
  129. 4672 @request = nil
  130. 4672 @requests.shift
  131. 4672 response = request.response
  132. 4672 emit(:response, request, response)
  133. 4615 if @parser.upgrade?
  134. 31 response << @parser.upgrade_data
  135. 31 throw(:called)
  136. end
  137. 4584 @parser.reset!
  138. 4115 @max_requests -= 1
  139. 4584 if response.is_a?(ErrorResponse)
  140. 14 disable
  141. else
  142. 4570 manage_connection(request, response)
  143. end
  144. 529 if exhausted?
  145. @pending.concat(@requests)
  146. @requests.clear
  147. emit(:exhausted)
  148. else
  149. 529 send(@pending.shift) unless @pending.empty?
  150. end
  151. end
  152. 27 def handle_error(ex, request = nil)
  153. 218 if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request && @request.response &&
  154. !@request.response.headers.key?("content-length") &&
  155. !@request.response.headers.key?("transfer-encoding")
  156. # if the response does not contain a content-length header, the server closing the
  157. # connnection is the indicator of response consumed.
  158. # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
  159. 32 catch(:called) { on_complete }
  160. 14 return
  161. end
  162. 202 if @pipelining
  163. catch(:called) { disable }
  164. else
  165. 202 @requests.each do |req|
  166. 190 next if request && request == req
  167. emit(:error, req, ex)
  168. end
  169. 202 @pending.each do |req|
  170. next if request && request == req
  171. emit(:error, req, ex)
  172. end
  173. end
  174. end
  175. 27 def ping
  176. reset
  177. emit(:reset)
  178. emit(:exhausted)
  179. end
  180. 27 private
  181. 27 def manage_connection(request, response)
  182. 4570 connection = response.headers["connection"]
  183. 4102 case connection
  184. when /keep-alive/i
  185. 529 if @handshake_completed
  186. if @max_requests.zero?
  187. @pending.concat(@requests)
  188. @requests.clear
  189. emit(:exhausted)
  190. end
  191. return
  192. end
  193. 529 keep_alive = response.headers["keep-alive"]
  194. 529 return unless keep_alive
  195. 108 parameters = Hash[keep_alive.split(/ *, */).map do |pair|
  196. 108 pair.split(/ *= */, 2)
  197. end]
  198. 108 @max_requests = parameters["max"].to_i - 1 if parameters.key?("max")
  199. 108 if parameters.key?("timeout")
  200. keep_alive_timeout = parameters["timeout"].to_i
  201. emit(:timeout, keep_alive_timeout)
  202. end
  203. 108 @handshake_completed = true
  204. when /close/i
  205. 4041 disable
  206. when nil
  207. # In HTTP/1.1, it's keep alive by default
  208. return if response.version == "1.1" && request.headers["connection"] != "close"
  209. disable
  210. end
  211. end
  212. 27 def disable
  213. 4055 disable_pipelining
  214. 4055 reset
  215. 4055 emit(:reset)
  216. 4047 throw(:called)
  217. end
  218. 27 def disable_pipelining
  219. 4055 return if @requests.empty?
  220. # do not disable pipelining if already set to 1 request at a time
  221. 187 return if @max_concurrent_requests == 1
  222. 29 @requests.each do |r|
  223. 29 r.transition(:idle)
  224. # when we disable pipelining, we still want to try keep-alive.
  225. # only when keep-alive with one request fails, do we fallback to
  226. # connection: close.
  227. 29 r.headers["connection"] = "close" if @max_concurrent_requests == 1
  228. end
  229. # server doesn't handle pipelining, and probably
  230. # doesn't support keep-alive. Fallback to send only
  231. # 1 keep alive request.
  232. 29 @max_concurrent_requests = 1
  233. 29 @pipelining = false
  234. end
  235. 27 def set_protocol_headers(request)
  236. 4915 if !request.headers.key?("content-length") &&
  237. request.body.bytesize == Float::INFINITY
  238. 32 request.body.chunk!
  239. end
  240. 4915 extra_headers = {}
  241. 4915 unless request.headers.key?("connection")
  242. 4891 connection_value = if request.persistent?
  243. # when in a persistent connection, the request can't be at
  244. # the edge of a renegotiation
  245. 97 if @requests.index(request) + 1 < @max_requests
  246. 97 "keep-alive"
  247. else
  248. "close"
  249. end
  250. else
  251. # when it's not a persistent connection, it sets "Connection: close" always
  252. # on the last request of the possible batch (either allowed max requests,
  253. # or if smaller, the size of the batch itself)
  254. 4794 requests_limit = [@max_requests, @requests.size].min
  255. 4794 if request == @requests[requests_limit - 1]
  256. 4229 "close"
  257. else
  258. 565 "keep-alive"
  259. end
  260. end
  261. 4386 extra_headers["connection"] = connection_value
  262. end
  263. 4915 extra_headers["host"] = request.authority unless request.headers.key?("host")
  264. 4915 extra_headers
  265. end
  266. 27 def handle(request)
  267. 6212 catch(:buffer_full) do
  268. 6212 request.transition(:headers)
  269. 6204 join_headers(request) if request.state == :headers
  270. 6204 request.transition(:body)
  271. 6204 join_body(request) if request.state == :body
  272. 5112 request.transition(:trailers)
  273. # HTTP/1.1 trailers should only work for chunked encoding
  274. 5112 join_trailers(request) if request.body.chunked? && request.state == :trailers
  275. 5112 request.transition(:done)
  276. end
  277. end
  278. 27 def join_headline(request)
  279. 4337 "#{request.verb} #{request.path} HTTP/#{@version.join(".")}"
  280. end
  281. 27 def join_headers(request)
  282. 4915 headline = join_headline(request)
  283. 4915 @buffer << headline << CRLF
  284. 4947 log(color: :yellow) { "<- HEADLINE: #{headline.chomp.inspect}" }
  285. 4915 extra_headers = set_protocol_headers(request)
  286. 4915 join_headers2(request.headers.each(extra_headers))
  287. 4947 log { "<- " }
  288. 4915 @buffer << CRLF
  289. end
  290. 27 def join_body(request)
  291. 5991 return if request.body.empty?
  292. 6565 while (chunk = request.drain_body)
  293. 3280 log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
  294. 3280 log(level: 2, color: :green) { "<- #{log_redact(chunk.inspect)}" }
  295. 3280 @buffer << chunk
  296. 3280 throw(:buffer_full, request) if @buffer.full?
  297. end
  298. 1556 return unless (error = request.drain_error)
  299. raise error
  300. end
  301. 27 def join_trailers(request)
  302. 96 return unless request.trailers? && request.callbacks_for?(:trailers)
  303. 32 join_headers2(request.trailers)
  304. 32 log { "<- " }
  305. 32 @buffer << CRLF
  306. end
  307. 27 def join_headers2(headers)
  308. 4947 headers.each do |field, value|
  309. 30354 field = capitalized(field)
  310. 30514 log(color: :yellow) { "<- HEADER: #{[field, log_redact(value)].join(": ")}" }
  311. 30354 @buffer << "#{field}: #{value}#{CRLF}"
  312. end
  313. end
  314. 27 UPCASED = {
  315. "www-authenticate" => "WWW-Authenticate",
  316. "http2-settings" => "HTTP2-Settings",
  317. "content-md5" => "Content-MD5",
  318. }.freeze
  319. 27 def capitalized(field)
  320. 30354 UPCASED[field] || field.split("-").map(&:capitalize).join("-")
  321. end
  322. end
  323. end

lib/httpx/connection/http2.rb

95.09% lines covered

265 relevant lines. 252 lines covered and 13 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "securerandom"
  3. 27 require "http/2"
  4. 27 module HTTPX
  5. 27 class Connection::HTTP2
  6. 27 include Callbacks
  7. 27 include Loggable
  8. 27 MAX_CONCURRENT_REQUESTS = ::HTTP2::DEFAULT_MAX_CONCURRENT_STREAMS
  9. 27 class Error < Error
  10. 27 def initialize(id, error)
  11. 50 super("stream #{id} closed with error: #{error}")
  12. end
  13. end
  14. 27 class PingError < Error
  15. 27 def initialize
  16. super(0, :ping_error)
  17. end
  18. end
  19. 27 class GoawayError < Error
  20. 27 def initialize
  21. 19 super(0, :no_error)
  22. end
  23. end
  24. 27 attr_reader :streams, :pending
  25. 27 def initialize(buffer, options)
  26. 3248 @options = options
  27. 3248 @settings = @options.http2_settings
  28. 3248 @pending = []
  29. 3248 @streams = {}
  30. 3248 @drains = {}
  31. 3248 @pings = []
  32. 3248 @buffer = buffer
  33. 3248 @handshake_completed = false
  34. 3248 @wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
  35. 3248 @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
  36. 3248 @max_requests = @options.max_requests
  37. 3248 init_connection
  38. end
  39. 27 def timeout
  40. 6414 return @options.timeout[:operation_timeout] if @handshake_completed
  41. 3204 @options.timeout[:settings_timeout]
  42. end
  43. 27 def interests
  44. # waiting for WINDOW_UPDATE frames
  45. 10719001 return :r if @buffer.full?
  46. 10719001 if @connection.state == :closed
  47. 2907 return unless @handshake_completed
  48. 2519 return :w
  49. end
  50. 10716094 unless @connection.state == :connected && @handshake_completed
  51. 12380 return @buffer.empty? ? :r : :rw
  52. end
  53. 10702243 return :w if !@pending.empty? && can_buffer_more_requests?
  54. 10702243 return :w unless @drains.empty?
  55. 10701265 if @buffer.empty?
  56. 10701265 return if @streams.empty? && @pings.empty?
  57. 39812 return :r
  58. end
  59. :rw
  60. end
  61. 27 def close
  62. 2760 unless @connection.state == :closed
  63. 2751 @connection.goaway
  64. 2751 emit(:timeout, @options.timeout[:close_handshake_timeout])
  65. end
  66. 2760 emit(:close, true)
  67. end
  68. 27 def empty?
  69. 2753 @connection.state == :closed || @streams.empty?
  70. end
  71. 27 def exhausted?
  72. 3253 !@max_requests.positive?
  73. end
  74. 27 def <<(data)
  75. 32927 @connection << data
  76. end
  77. 27 def send(request, head = false)
  78. 7100 unless can_buffer_more_requests?
  79. 3441 head ? @pending.unshift(request) : @pending << request
  80. 3441 return false
  81. end
  82. 3659 unless (stream = @streams[request])
  83. 3659 stream = @connection.new_stream
  84. 3659 handle_stream(stream, request)
  85. 3249 @streams[request] = stream
  86. 3249 @max_requests -= 1
  87. end
  88. 3659 handle(request, stream)
  89. 3643 true
  90. rescue ::HTTP2::Error::StreamLimitExceeded
  91. @pending.unshift(request)
  92. false
  93. end
  94. 27 def consume
  95. 25379 @streams.each do |request, stream|
  96. 10040 next unless request.can_buffer?
  97. 1117 handle(request, stream)
  98. end
  99. end
  100. 27 def handle_error(ex, request = nil)
  101. 273 if ex.is_a?(OperationTimeoutError) && !@handshake_completed && @connection.state != :closed
  102. 8 @connection.goaway(:settings_timeout, "closing due to settings timeout")
  103. 8 emit(:close_handshake)
  104. 8 settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
  105. 8 settings_ex.set_backtrace(ex.backtrace)
  106. 8 ex = settings_ex
  107. end
  108. 273 @streams.each_key do |req|
  109. 225 next if request && request == req
  110. 16 emit(:error, req, ex)
  111. end
  112. 516 while (req = @pending.shift)
  113. 35 next if request && request == req
  114. 35 emit(:error, req, ex)
  115. end
  116. end
  117. 27 def ping
  118. 20 ping = SecureRandom.gen_random(8)
  119. 20 @connection.ping(ping.dup)
  120. ensure
  121. 20 @pings << ping
  122. end
  123. 27 private
  124. 27 def can_buffer_more_requests?
  125. 7601 (@handshake_completed || !@wait_for_handshake) &&
  126. @streams.size < @max_concurrent_requests &&
  127. @streams.size < @max_requests
  128. end
  129. 27 def send_pending
  130. 8565 while (request = @pending.shift)
  131. 3316 break unless send(request, true)
  132. end
  133. end
  134. 27 def handle(request, stream)
  135. 4920 catch(:buffer_full) do
  136. 4920 request.transition(:headers)
  137. 4912 join_headers(stream, request) if request.state == :headers
  138. 4912 request.transition(:body)
  139. 4912 join_body(stream, request) if request.state == :body
  140. 3822 request.transition(:trailers)
  141. 3822 join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
  142. 3822 request.transition(:done)
  143. end
  144. end
  145. 27 def init_connection
  146. 3248 @connection = ::HTTP2::Client.new(@settings)
  147. 3248 @connection.on(:frame, &method(:on_frame))
  148. 3248 @connection.on(:frame_sent, &method(:on_frame_sent))
  149. 3248 @connection.on(:frame_received, &method(:on_frame_received))
  150. 3248 @connection.on(:origin, &method(:on_origin))
  151. 3248 @connection.on(:promise, &method(:on_promise))
  152. 3248 @connection.on(:altsvc) { |frame| on_altsvc(frame[:origin], frame) }
  153. 3248 @connection.on(:settings_ack, &method(:on_settings))
  154. 3248 @connection.on(:ack, &method(:on_pong))
  155. 3248 @connection.on(:goaway, &method(:on_close))
  156. #
  157. # Some servers initiate HTTP/2 negotiation right away, some don't.
  158. # As such, we have to check the socket buffer. If there is something
  159. # to read, the server initiated the negotiation. If not, we have to
  160. # initiate it.
  161. #
  162. 3248 @connection.send_connection_preface
  163. end
  164. 27 alias_method :reset, :init_connection
  165. 27 public :reset
  166. 27 def handle_stream(stream, request)
  167. 3675 request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
  168. 3675 stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
  169. 3675 stream.on(:half_close) do
  170. 3641 log(level: 2) { "#{stream.id}: waiting for response..." }
  171. end
  172. 3675 stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
  173. 3675 stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
  174. 3675 stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
  175. end
  176. 27 def set_protocol_headers(request)
  177. 409 {
  178. 3241 ":scheme" => request.scheme,
  179. ":method" => request.verb,
  180. ":path" => request.path,
  181. ":authority" => request.authority,
  182. }
  183. end
  184. 27 def join_headers(stream, request)
  185. 3651 extra_headers = set_protocol_headers(request)
  186. 3651 if request.headers.key?("host")
  187. 8 log { "forbidden \"host\" header found (#{log_redact(request.headers["host"])}), will use it as authority..." }
  188. 7 extra_headers[":authority"] = request.headers["host"]
  189. end
  190. 3651 log(level: 1, color: :yellow) do
  191. 128 request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact(v)}" }.join("\n")
  192. end
  193. 3651 stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
  194. end
  195. 27 def join_trailers(stream, request)
  196. 1529 unless request.trailers?
  197. 1521 stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
  198. 1358 return
  199. end
  200. 8 log(level: 1, color: :yellow) do
  201. 15 request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact(v)}" }.join("\n")
  202. end
  203. 8 stream.headers(request.trailers.each, end_stream: true)
  204. end
  205. 27 def join_body(stream, request)
  206. 4727 return if request.body.empty?
  207. 2621 chunk = @drains.delete(request) || request.drain_body
  208. 2815 while chunk
  209. 2835 next_chunk = request.drain_body
  210. 2835 send_chunk(request, stream, chunk, next_chunk)
  211. 2739 if next_chunk && (@buffer.full? || request.body.unbounded_body?)
  212. 874 @drains[request] = next_chunk
  213. 994 throw(:buffer_full)
  214. end
  215. 1745 chunk = next_chunk
  216. end
  217. 1531 return unless (error = request.drain_error)
  218. 24 on_stream_refuse(stream, request, error)
  219. end
  220. 27 def send_chunk(request, stream, chunk, next_chunk)
  221. 2856 log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
  222. 2856 log(level: 2, color: :green) { "#{stream.id}: -> #{log_redact(chunk.inspect)}" }
  223. 2835 stream.data(chunk, end_stream: end_stream?(request, next_chunk))
  224. end
  225. 27 def end_stream?(request, next_chunk)
  226. 2739 !(next_chunk || request.trailers? || request.callbacks_for?(:trailers))
  227. end
  228. ######
  229. # HTTP/2 Callbacks
  230. ######
  231. 27 def on_stream_headers(stream, request, h)
  232. 3607 response = request.response
  233. 3607 if response.is_a?(Response) && response.version == "2.0"
  234. 114 on_stream_trailers(stream, response, h)
  235. 114 return
  236. end
  237. 3493 log(color: :yellow) do
  238. 128 h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{log_redact(v)}" }.join("\n")
  239. end
  240. 3493 _, status = h.shift
  241. 3493 headers = request.options.headers_class.new(h)
  242. 3493 response = request.options.response_class.new(request, status, "2.0", headers)
  243. 3493 request.response = response
  244. 3093 @streams[request] = stream
  245. 3485 handle(request, stream) if request.expects?
  246. end
  247. 27 def on_stream_trailers(stream, response, h)
  248. 114 log(color: :yellow) do
  249. h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{log_redact(v)}" }.join("\n")
  250. end
  251. 114 response.merge_headers(h)
  252. end
  253. 27 def on_stream_data(stream, request, data)
  254. 6470 log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
  255. 6470 log(level: 2, color: :green) { "#{stream.id}: <- #{log_redact(data.inspect)}" }
  256. 6452 request.response << data
  257. end
  258. 27 def on_stream_refuse(stream, request, error)
  259. 24 on_stream_close(stream, request, error)
  260. 24 stream.close
  261. end
  262. 27 def on_stream_close(stream, request, error)
  263. 3423 return if error == :stream_closed && !@streams.key?(request)
  264. 3413 log(level: 2) { "#{stream.id}: closing stream" }
  265. 3399 @drains.delete(request)
  266. 3399 @streams.delete(request)
  267. 3399 if error
  268. 24 case error
  269. when :http_1_1_required
  270. emit(:error, request, error)
  271. else
  272. 24 ex = Error.new(stream.id, error)
  273. 24 ex.set_backtrace(caller)
  274. 24 response = ErrorResponse.new(request, ex)
  275. 24 request.response = response
  276. 24 emit(:response, request, response)
  277. end
  278. else
  279. 3375 response = request.response
  280. 3375 if response && response.is_a?(Response) && response.status == 421
  281. 8 emit(:error, request, :http_1_1_required)
  282. else
  283. 3367 emit(:response, request, response)
  284. end
  285. end
  286. 3391 send(@pending.shift) unless @pending.empty?
  287. 3391 return unless @streams.empty? && exhausted?
  288. 8 if @pending.empty?
  289. close
  290. else
  291. 8 emit(:exhausted)
  292. end
  293. end
  294. 27 def on_frame(bytes)
  295. 20156 @buffer << bytes
  296. end
  297. 27 def on_settings(*)
  298. 3210 @handshake_completed = true
  299. 3210 emit(:current_timeout)
  300. 3210 @max_concurrent_requests = [@max_concurrent_requests, @connection.remote_settings[:settings_max_concurrent_streams]].min
  301. 3210 send_pending
  302. end
  303. 27 def on_close(_last_frame, error, _payload)
  304. 34 is_connection_closed = @connection.state == :closed
  305. 34 if error
  306. 34 @buffer.clear if is_connection_closed
  307. 30 case error
  308. when :http_1_1_required
  309. 21 while (request = @pending.shift)
  310. 8 emit(:error, request, error)
  311. end
  312. when :no_error
  313. 19 ex = GoawayError.new
  314. 19 @pending.unshift(*@streams.keys)
  315. 19 @drains.clear
  316. 19 @streams.clear
  317. else
  318. 7 ex = Error.new(0, error)
  319. end
  320. 34 if ex
  321. 26 ex.set_backtrace(caller)
  322. 26 handle_error(ex)
  323. end
  324. end
  325. 34 return unless is_connection_closed && @streams.empty?
  326. 34 emit(:close, is_connection_closed)
  327. end
  328. 27 def on_frame_sent(frame)
  329. 16983 log(level: 2) { "#{frame[:stream]}: frame was sent!" }
  330. 16899 log(level: 2, color: :blue) do
  331. 12 payload =
  332. 83 case frame[:type]
  333. when :data
  334. 24 frame.merge(payload: frame[:payload].bytesize)
  335. when :headers, :ping
  336. 24 frame.merge(payload: log_redact(frame[:payload]))
  337. else
  338. 48 frame
  339. end
  340. 84 "#{frame[:stream]}: #{payload}"
  341. end
  342. end
  343. 27 def on_frame_received(frame)
  344. 17875 log(level: 2) { "#{frame[:stream]}: frame was received!" }
  345. 17815 log(level: 2, color: :magenta) do
  346. 9 payload =
  347. 59 case frame[:type]
  348. when :data
  349. 21 frame.merge(payload: frame[:payload].bytesize)
  350. when :headers, :ping
  351. 16 frame.merge(payload: log_redact(frame[:payload]))
  352. else
  353. 32 frame
  354. end
  355. 60 "#{frame[:stream]}: #{payload}"
  356. end
  357. end
  358. 27 def on_altsvc(origin, frame)
  359. log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
  360. log(level: 2) { "#{frame[:stream]}: #{log_redact(frame.inspect)}" }
  361. alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
  362. params = { "ma" => frame[:max_age] }
  363. emit(:altsvc, origin, alt_origin, origin, params)
  364. end
  365. 27 def on_promise(stream)
  366. 24 emit(:promise, @streams.key(stream.parent), stream)
  367. end
  368. 27 def on_origin(origin)
  369. emit(:origin, origin)
  370. end
  371. 27 def on_pong(ping)
  372. 8 raise PingError unless @pings.delete(ping.to_s)
  373. 8 emit(:pong)
  374. end
  375. end
  376. end

lib/httpx/domain_name.rb

95.56% lines covered

45 relevant lines. 43 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. #
  3. # domain_name.rb - Domain Name manipulation library for Ruby
  4. #
  5. # Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
  6. #
  7. # Redistribution and use in source and binary forms, with or without
  8. # modification, are permitted provided that the following conditions
  9. # are met:
  10. # 1. Redistributions of source code must retain the above copyright
  11. # notice, this list of conditions and the following disclaimer.
  12. # 2. Redistributions in binary form must reproduce the above copyright
  13. # notice, this list of conditions and the following disclaimer in the
  14. # documentation and/or other materials provided with the distribution.
  15. #
  16. # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17. # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19. # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20. # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24. # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25. # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26. # SUCH DAMAGE.
  27. 27 require "ipaddr"
  28. 27 module HTTPX
  29. # Represents a domain name ready for extracting its registered domain
  30. # and TLD.
  31. 27 class DomainName
  32. 27 include Comparable
  33. # The full host name normalized, ASCII-ized and downcased using the
  34. # Unicode NFC rules and the Punycode algorithm. If initialized with
  35. # an IP address, the string representation of the IP address
  36. # suitable for opening a connection to.
  37. 27 attr_reader :hostname
  38. # The Unicode representation of the #hostname property.
  39. #
  40. # :attr_reader: hostname_idn
  41. # The least "universally original" domain part of this domain name.
  42. # For example, "example.co.uk" for "www.sub.example.co.uk". This
  43. # may be nil if the hostname does not have one, like when it is an
  44. # IP address, an effective TLD or higher itself, or of a
  45. # non-canonical domain.
  46. 27 attr_reader :domain
  47. 27 class << self
  48. 27 def new(domain)
  49. 856 return domain if domain.is_a?(self)
  50. 792 super(domain)
  51. end
  52. # Normalizes a _domain_ using the Punycode algorithm as necessary.
  53. # The result will be a downcased, ASCII-only string.
  54. 27 def normalize(domain)
  55. 760 unless domain.ascii_only?
  56. domain = domain.chomp(".").unicode_normalize(:nfc)
  57. domain = Punycode.encode_hostname(domain)
  58. end
  59. 760 domain.downcase
  60. end
  61. end
  62. # Parses _hostname_ into a DomainName object. An IP address is also
  63. # accepted. An IPv6 address may be enclosed in square brackets.
  64. 27 def initialize(hostname)
  65. 792 hostname = String(hostname)
  66. 792 raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
  67. 98 begin
  68. 792 @ipaddr = IPAddr.new(hostname)
  69. 32 @hostname = @ipaddr.to_s
  70. 32 return
  71. rescue IPAddr::Error
  72. 760 nil
  73. end
  74. 760 @hostname = DomainName.normalize(hostname)
  75. 760 tld = if (last_dot = @hostname.rindex("."))
  76. 184 @hostname[(last_dot + 1)..-1]
  77. else
  78. 576 @hostname
  79. end
  80. # unknown/local TLD
  81. 760 @domain = if last_dot
  82. # fallback - accept cookies down to second level
  83. # cf. http://www.dkim-reputation.org/regdom-libs/
  84. 184 if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
  85. 48 @hostname[(penultimate_dot + 1)..-1]
  86. else
  87. 136 @hostname
  88. end
  89. else
  90. # no domain part - must be a local hostname
  91. 576 tld
  92. end
  93. end
  94. # Checks if the server represented by this domain is qualified to
  95. # send and receive cookies with a domain attribute value of
  96. # _domain_. A true value given as the second argument represents
  97. # cookies without a domain attribute value, in which case only
  98. # hostname equality is checked.
  99. 27 def cookie_domain?(domain, host_only = false)
  100. # RFC 6265 #5.3
  101. # When the user agent "receives a cookie":
  102. 32 return self == @domain if host_only
  103. 32 domain = DomainName.new(domain)
  104. # RFC 6265 #5.1.3
  105. # Do not perform subdomain matching against IP addresses.
  106. 32 @hostname == domain.hostname if @ipaddr
  107. # RFC 6265 #4.1.1
  108. # Domain-value must be a subdomain.
  109. 32 @domain && self <= domain && domain <= @domain
  110. end
  111. 27 def <=>(other)
  112. 48 other = DomainName.new(other)
  113. 48 othername = other.hostname
  114. 48 if othername == @hostname
  115. 16 0
  116. 31 elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
  117. # The other is higher
  118. 16 -1
  119. else
  120. # The other is lower
  121. 16 1
  122. end
  123. end
  124. end
  125. end

lib/httpx/errors.rb

97.62% lines covered

42 relevant lines. 41 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # the default exception class for exceptions raised by HTTPX.
  4. 27 class Error < StandardError; end
  5. 27 class UnsupportedSchemeError < Error; end
  6. 27 class ConnectionError < Error; end
  7. # Error raised when there was a timeout. Its subclasses allow for finer-grained
  8. # control of which timeout happened.
  9. 27 class TimeoutError < Error
  10. # The timeout value which caused this error to be raised.
  11. 27 attr_reader :timeout
  12. # initializes the timeout exception with the +timeout+ causing the error, and the
  13. # error +message+ for it.
  14. 27 def initialize(timeout, message)
  15. 525 @timeout = timeout
  16. 525 super(message)
  17. end
  18. # clones this error into a HTTPX::ConnectionTimeoutError.
  19. 27 def to_connection_error
  20. 24 ex = ConnectTimeoutError.new(@timeout, message)
  21. 24 ex.set_backtrace(backtrace)
  22. 24 ex
  23. end
  24. end
  25. # Raise when it can't acquire a connection from the pool.
  26. 27 class PoolTimeoutError < TimeoutError; end
  27. # Error raised when there was a timeout establishing the connection to a server.
  28. # This may be raised due to timeouts during TCP and TLS (when applicable) connection
  29. # establishment.
  30. 27 class ConnectTimeoutError < TimeoutError; end
  31. # Error raised when there was a timeout while sending a request, or receiving a response
  32. # from the server.
  33. 27 class RequestTimeoutError < TimeoutError
  34. # The HTTPX::Request request object this exception refers to.
  35. 27 attr_reader :request
  36. # initializes the exception with the +request+ and +response+ it refers to, and the
  37. # +timeout+ causing the error, and the
  38. 27 def initialize(request, response, timeout)
  39. 414 @request = request
  40. 414 @response = response
  41. 414 super(timeout, "Timed out after #{timeout} seconds")
  42. end
  43. 27 def marshal_dump
  44. [message]
  45. end
  46. end
  47. # Error raised when there was a timeout while receiving a response from the server.
  48. 27 class ReadTimeoutError < RequestTimeoutError; end
  49. # Error raised when there was a timeout while sending a request from the server.
  50. 27 class WriteTimeoutError < RequestTimeoutError; end
  51. # Error raised when there was a timeout while waiting for the HTTP/2 settings frame from the server.
  52. 27 class SettingsTimeoutError < TimeoutError; end
  53. # Error raised when there was a timeout while resolving a domain to an IP.
  54. 27 class ResolveTimeoutError < TimeoutError; end
  55. # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
  56. 27 class OperationTimeoutError < TimeoutError; end
  57. # Error raised when there was an error while resolving a domain to an IP.
  58. 27 class ResolveError < Error; end
  59. # Error raised when there was an error while resolving a domain to an IP
  60. # using a HTTPX::Resolver::Native resolver.
  61. 27 class NativeResolveError < ResolveError
  62. 27 attr_reader :connection, :host
  63. # initializes the exception with the +connection+ it refers to, the +host+ domain
  64. # which failed to resolve, and the error +message+.
  65. 27 def initialize(connection, host, message = "Can't resolve #{host}")
  66. 136 @connection = connection
  67. 136 @host = host
  68. 136 super(message)
  69. end
  70. end
  71. # The exception class for HTTP responses with 4xx or 5xx status.
  72. 27 class HTTPError < Error
  73. # The HTTPX::Response response object this exception refers to.
  74. 27 attr_reader :response
  75. # Creates the instance and assigns the HTTPX::Response +response+.
  76. 27 def initialize(response)
  77. 84 @response = response
  78. 84 super("HTTP Error: #{@response.status} #{@response.headers}\n#{@response.body}")
  79. end
  80. # The HTTP response status.
  81. #
  82. # error.status #=> 404
  83. 27 def status
  84. 16 @response.status
  85. end
  86. end
  87. end

lib/httpx/extensions.rb

67.86% lines covered

28 relevant lines. 19 lines covered and 9 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "uri"
  3. 27 module HTTPX
  4. 27 module ArrayExtensions
  5. 27 module FilterMap
  6. refine Array do
  7. # Ruby 2.7 backport
  8. def filter_map
  9. return to_enum(:filter_map) unless block_given?
  10. each_with_object([]) do |item, res|
  11. processed = yield(item)
  12. res << processed if processed
  13. end
  14. end
  15. 26 end unless Array.method_defined?(:filter_map)
  16. end
  17. 27 module Intersect
  18. refine Array do
  19. # Ruby 3.1 backport
  20. 4 def intersect?(arr)
  21. if size < arr.size
  22. smaller = self
  23. else
  24. smaller, arr = arr, self
  25. end
  26. (arr & smaller).size > 0
  27. end
  28. 26 end unless Array.method_defined?(:intersect?)
  29. end
  30. end
  31. 27 module URIExtensions
  32. # uri 0.11 backport, ships with ruby 3.1
  33. 27 refine URI::Generic do
  34. 27 def non_ascii_hostname
  35. 447 @non_ascii_hostname
  36. end
  37. 27 def non_ascii_hostname=(hostname)
  38. 32 @non_ascii_hostname = hostname
  39. end
  40. def authority
  41. 5695 return host if port == default_port
  42. 607 "#{host}:#{port}"
  43. 26 end unless URI::HTTP.method_defined?(:authority)
  44. def origin
  45. 4635 "#{scheme}://#{authority}"
  46. 26 end unless URI::HTTP.method_defined?(:origin)
  47. end
  48. end
  49. end

lib/httpx/headers.rb

100.0% lines covered

71 relevant lines. 71 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 class Headers
  4. 27 class << self
  5. 27 def new(headers = nil)
  6. 25876 return headers if headers.is_a?(self)
  7. 11843 super
  8. end
  9. end
  10. 27 def initialize(headers = nil)
  11. 11843 if headers.nil? || headers.empty?
  12. 1896 @headers = headers.to_h
  13. 1677 return
  14. end
  15. 9947 @headers = {}
  16. 9947 headers.each do |field, value|
  17. 61414 field = downcased(field)
  18. 61414 value = array_value(value)
  19. 61414 current = @headers[field]
  20. 61414 if current.nil?
  21. 54587 @headers[field] = value
  22. else
  23. 53 current.concat(value)
  24. end
  25. end
  26. end
  27. # cloned initialization
  28. 27 def initialize_clone(orig, **kwargs)
  29. 8 super
  30. 8 @headers = orig.instance_variable_get(:@headers).clone(**kwargs)
  31. end
  32. # dupped initialization
  33. 27 def initialize_dup(orig)
  34. 15451 super
  35. 15451 @headers = orig.instance_variable_get(:@headers).dup
  36. end
  37. # freezes the headers hash
  38. 27 def freeze
  39. 20152 @headers.freeze
  40. 20152 super
  41. end
  42. # merges headers with another header-quack.
  43. # the merge rule is, if the header already exists,
  44. # ignore what the +other+ headers has. Otherwise, set
  45. #
  46. 27 def merge(other)
  47. 4866 headers = dup
  48. 4866 other.each do |field, value|
  49. 4394 headers[downcased(field)] = value
  50. end
  51. 4866 headers
  52. end
  53. # returns the comma-separated values of the header field
  54. # identified by +field+, or nil otherwise.
  55. #
  56. 27 def [](field)
  57. 95014 a = @headers[downcased(field)] || return
  58. 28344 a.join(", ")
  59. end
  60. # sets +value+ (if not nil) as single value for the +field+ header.
  61. #
  62. 27 def []=(field, value)
  63. 41824 return unless value
  64. 37470 @headers[downcased(field)] = array_value(value)
  65. end
  66. # deletes all values associated with +field+ header.
  67. #
  68. 27 def delete(field)
  69. 274 canonical = downcased(field)
  70. 274 @headers.delete(canonical) if @headers.key?(canonical)
  71. end
  72. # adds additional +value+ to the existing, for header +field+.
  73. #
  74. 27 def add(field, value)
  75. 592 (@headers[downcased(field)] ||= []) << String(value)
  76. end
  77. # helper to be used when adding an header field as a value to another field
  78. #
  79. # h2_headers.add_header("vary", "accept-encoding")
  80. # h2_headers["vary"] #=> "accept-encoding"
  81. # h1_headers.add_header("vary", "accept-encoding")
  82. # h1_headers["vary"] #=> "Accept-Encoding"
  83. #
  84. 27 alias_method :add_header, :add
  85. # returns the enumerable headers store in pairs of header field + the values in
  86. # the comma-separated string format
  87. #
  88. 27 def each(extra_headers = nil)
  89. 67824 return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
  90. 36108 @headers.each do |field, value|
  91. 49039 yield(field, value.join(", ")) unless value.empty?
  92. end
  93. 5997 extra_headers.each do |field, value|
  94. 24497 yield(field, value) unless value.empty?
  95. 36107 end if extra_headers
  96. end
  97. 27 def ==(other)
  98. 21086 other == to_hash
  99. end
  100. 27 def empty?
  101. 312 @headers.empty?
  102. end
  103. # the headers store in Hash format
  104. 27 def to_hash
  105. 22911 Hash[to_a]
  106. end
  107. 27 alias_method :to_h, :to_hash
  108. # the headers store in array of pairs format
  109. 27 def to_a
  110. 22934 Array(each)
  111. end
  112. # headers as string
  113. 27 def to_s
  114. 1903 @headers.to_s
  115. end
  116. skipped # :nocov:
  117. skipped def inspect
  118. skipped "#<#{self.class}:#{object_id} " \
  119. skipped "#{to_hash.inspect}>"
  120. skipped end
  121. skipped # :nocov:
  122. # this is internal API and doesn't abide to other public API
  123. # guarantees, like downcasing strings.
  124. # Please do not use this outside of core!
  125. #
  126. 27 def key?(downcased_key)
  127. 64520 @headers.key?(downcased_key)
  128. end
  129. # returns the values for the +field+ header in array format.
  130. # This method is more internal, and for this reason doesn't try
  131. # to "correct" the user input, i.e. it doesn't downcase the key.
  132. #
  133. 27 def get(field)
  134. 263 @headers[field] || EMPTY
  135. end
  136. 27 private
  137. 27 def array_value(value)
  138. 103238 Array(value)
  139. end
  140. 27 def downcased(field)
  141. 203940 String(field).downcase
  142. end
  143. end
  144. end

lib/httpx/io.rb

100.0% lines covered

5 relevant lines. 5 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "socket"
  3. 27 require "httpx/io/udp"
  4. 27 require "httpx/io/tcp"
  5. 27 require "httpx/io/unix"
  6. begin
  7. 27 require "httpx/io/ssl"
  8. rescue LoadError
  9. end

lib/httpx/io/ssl.rb

96.55% lines covered

87 relevant lines. 84 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "openssl"
  3. 27 module HTTPX
  4. 27 TLSError = OpenSSL::SSL::SSLError
  5. 27 class SSL < TCP
  6. # rubocop:disable Style/MutableConstant
  7. 27 TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
  8. # https://github.com/jruby/jruby-openssl/issues/284
  9. # TODO: remove when dropping support for jruby-openssl < 0.15.4
  10. 27 TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby" && JOpenSSL::VERSION < "0.15.4"
  11. # rubocop:enable Style/MutableConstant
  12. 27 TLS_OPTIONS.freeze
  13. 27 attr_writer :ssl_session
  14. 27 def initialize(_, _, options)
  15. 3004 super
  16. 3004 ctx_options = TLS_OPTIONS.merge(options.ssl)
  17. 3004 @sni_hostname = ctx_options.delete(:hostname) || @hostname
  18. 3004 if @keep_open && @io.is_a?(OpenSSL::SSL::SSLSocket)
  19. # externally initiated ssl socket
  20. 22 @ctx = @io.context
  21. 22 @state = :negotiated
  22. else
  23. 2982 @ctx = OpenSSL::SSL::SSLContext.new
  24. 2982 @ctx.set_params(ctx_options) unless ctx_options.empty?
  25. 2982 unless @ctx.session_cache_mode.nil? # a dummy method on JRuby
  26. 2648 @ctx.session_cache_mode =
  27. OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
  28. end
  29. 2982 yield(self) if block_given?
  30. end
  31. 3004 @verify_hostname = @ctx.verify_hostname
  32. end
  33. 27 if OpenSSL::SSL::SSLContext.method_defined?(:session_new_cb=)
  34. 26 def session_new_cb(&pr)
  35. 7617 @ctx.session_new_cb = proc { |_, sess| pr.call(sess) }
  36. end
  37. else
  38. # session_new_cb not implemented under JRuby
  39. 1 def session_new_cb; end
  40. end
  41. 27 def protocol
  42. 2882 @io.alpn_protocol || super
  43. rescue StandardError
  44. 8 super
  45. end
  46. 27 if RUBY_ENGINE == "jruby"
  47. # in jruby, alpn_protocol may return ""
  48. # https://github.com/jruby/jruby-openssl/issues/287
  49. 1 def protocol
  50. 365 proto = @io.alpn_protocol
  51. 364 return super if proto.nil? || proto.empty?
  52. 363 proto
  53. rescue StandardError
  54. 1 super
  55. end
  56. end
  57. 27 def can_verify_peer?
  58. 13 @ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
  59. end
  60. 27 def verify_hostname(host)
  61. 15 return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
  62. 15 return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?
  63. 15 OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
  64. end
  65. 27 def connected?
  66. 12617 @state == :negotiated
  67. end
  68. 27 def expired?
  69. super || ssl_session_expired?
  70. end
  71. 27 def ssl_session_expired?
  72. 3207 @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
  73. end
  74. 27 def connect
  75. 12673 return if @state == :negotiated
  76. 12673 unless @state == :connected
  77. 7104 super
  78. 7072 return unless @state == :connected
  79. end
  80. 8687 unless @io.is_a?(OpenSSL::SSL::SSLSocket)
  81. 3207 if (hostname_is_ip = (@ip == @sni_hostname))
  82. # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
  83. 32 @sni_hostname = @ip.to_string
  84. # IP addresses in SNI is not valid per RFC 6066, section 3.
  85. 32 @ctx.verify_hostname = false
  86. end
  87. 3207 @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
  88. 3207 @io.hostname = @sni_hostname unless hostname_is_ip
  89. 3207 @io.session = @ssl_session unless ssl_session_expired?
  90. 3207 @io.sync_close = true
  91. end
  92. 8687 try_ssl_connect
  93. end
  94. 27 def try_ssl_connect
  95. 8687 ret = @io.connect_nonblock(exception: false)
  96. 8707 log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
  97. 7926 case ret
  98. when :wait_readable
  99. 5504 @interests = :r
  100. 5504 return
  101. when :wait_writable
  102. @interests = :w
  103. return
  104. end
  105. 3160 @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE && @verify_hostname
  106. 3159 transition(:negotiated)
  107. 3159 @interests = :w
  108. end
  109. 27 private
  110. 27 def transition(nextstate)
  111. 11061 case nextstate
  112. when :negotiated
  113. 3159 return unless @state == :connected
  114. when :closed
  115. 3082 return unless @state == :negotiated ||
  116. @state == :connected
  117. end
  118. 12509 do_transition(nextstate)
  119. end
  120. 27 def log_transition_state(nextstate)
  121. 70 return super unless nextstate == :negotiated
  122. 16 server_cert = @io.peer_cert
  123. 14 "#{super}\n\n" \
  124. 2 "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
  125. 2 "ALPN, server accepted to use #{protocol}\n" \
  126. "Server certificate:\n " \
  127. 2 "subject: #{server_cert.subject}\n " \
  128. 2 "start date: #{server_cert.not_before}\n " \
  129. 2 "expire date: #{server_cert.not_after}\n " \
  130. 2 "issuer: #{server_cert.issuer}\n " \
  131. "SSL certificate verify ok."
  132. end
  133. end
  134. end

lib/httpx/io/tcp.rb

91.15% lines covered

113 relevant lines. 103 lines covered and 10 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "ipaddr"
  4. 27 module HTTPX
  5. 27 class TCP
  6. 27 include Loggable
  7. 27 using URIExtensions
  8. 27 attr_reader :ip, :port, :addresses, :state, :interests
  9. 27 alias_method :host, :ip
  10. 27 def initialize(origin, addresses, options)
  11. 7099 @state = :idle
  12. 7099 @addresses = []
  13. 7099 @hostname = origin.host
  14. 7099 @options = options
  15. 7099 @fallback_protocol = @options.fallback_protocol
  16. 7099 @port = origin.port
  17. 7099 @interests = :w
  18. 7099 if @options.io
  19. 52 @io = case @options.io
  20. when Hash
  21. 16 @options.io[origin.authority]
  22. else
  23. 36 @options.io
  24. end
  25. 52 raise Error, "Given IO objects do not match the request authority" unless @io
  26. 52 _, _, _, @ip = @io.addr
  27. 52 @addresses << @ip
  28. 52 @keep_open = true
  29. 52 @state = :connected
  30. else
  31. 7047 add_addresses(addresses)
  32. end
  33. 7099 @ip_index = @addresses.size - 1
  34. end
  35. 27 def socket
  36. 195 @io
  37. end
  38. 27 def add_addresses(addrs)
  39. 7305 return if addrs.empty?
  40. 27311 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  41. 7305 ip_index = @ip_index || (@addresses.size - 1)
  42. 7305 if addrs.first.ipv6?
  43. # should be the next in line
  44. 268 @addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
  45. else
  46. 7037 @addresses.unshift(*addrs)
  47. 7037 @ip_index += addrs.size if @ip_index
  48. end
  49. end
  50. 27 def to_io
  51. 23346 @io.to_io
  52. end
  53. 27 def protocol
  54. 4458 @fallback_protocol
  55. end
  56. 27 def connect
  57. 29261 return unless closed?
  58. 29134 if !@io || @io.closed?
  59. 7763 transition(:idle)
  60. 7763 @io = build_socket
  61. end
  62. 29134 try_connect
  63. rescue Errno::ECONNREFUSED,
  64. Errno::EADDRNOTAVAIL,
  65. Errno::EHOSTUNREACH,
  66. SocketError,
  67. IOError => e
  68. 913 raise e if @ip_index <= 0
  69. 851 log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  70. 826 @ip_index -= 1
  71. 839 @io = build_socket
  72. 839 retry
  73. rescue Errno::ETIMEDOUT => e
  74. raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
  75. log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
  76. @ip_index -= 1
  77. @io = build_socket
  78. retry
  79. end
  80. 27 def try_connect
  81. 29134 ret = @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
  82. 16399 log(level: 3, color: :cyan) { "TCP CONNECT: #{ret}..." }
  83. 14506 case ret
  84. when :wait_readable
  85. @interests = :r
  86. return
  87. when :wait_writable
  88. 8590 @interests = :w
  89. 8590 return
  90. end
  91. 7689 transition(:connected)
  92. 7689 @interests = :w
  93. rescue Errno::EALREADY
  94. 11942 @interests = :w
  95. end
  96. 27 private :try_connect
  97. 27 def read(size, buffer)
  98. 51164 ret = @io.read_nonblock(size, buffer, exception: false)
  99. 51164 if ret == :wait_readable
  100. 10782 buffer.clear
  101. 9956 return 0
  102. end
  103. 40382 return if ret.nil?
  104. 40447 log { "READ: #{buffer.bytesize} bytes..." }
  105. 40360 buffer.bytesize
  106. end
  107. 27 def write(buffer)
  108. 21342 siz = @io.write_nonblock(buffer, exception: false)
  109. 21319 return 0 if siz == :wait_writable
  110. 21303 return if siz.nil?
  111. 21398 log { "WRITE: #{siz} bytes..." }
  112. 21303 buffer.shift!(siz)
  113. 21303 siz
  114. end
  115. 27 def close
  116. 8471 return if @keep_open || closed?
  117. 859 begin
  118. 7494 @io.close
  119. ensure
  120. 7494 transition(:closed)
  121. end
  122. end
  123. 27 def connected?
  124. 21290 @state == :connected
  125. end
  126. 27 def closed?
  127. 37719 @state == :idle || @state == :closed
  128. end
  129. 27 def expired?
  130. # do not mess with external sockets
  131. return false if @options.io
  132. return true unless @addresses
  133. resolver_addresses = Resolver.nolookup_resolve(@hostname)
  134. (Array(resolver_addresses) & @addresses).empty?
  135. end
  136. skipped # :nocov:
  137. skipped def inspect
  138. skipped "#<#{self.class}:#{object_id} " \
  139. skipped "#{@ip}:#{@port} " \
  140. skipped "@state=#{@state} " \
  141. skipped "@hostname=#{@hostname} " \
  142. skipped "@addresses=#{@addresses} " \
  143. skipped "@state=#{@state}>"
  144. skipped end
  145. skipped # :nocov:
  146. 27 private
  147. 27 def build_socket
  148. 8602 @ip = @addresses[@ip_index]
  149. 8602 Socket.new(@ip.family, :STREAM, 0)
  150. end
  151. 27 def transition(nextstate)
  152. 12068 case nextstate
  153. # when :idle
  154. when :connected
  155. 4585 return unless @state == :idle
  156. when :closed
  157. 4412 return unless @state == :connected
  158. end
  159. 13617 do_transition(nextstate)
  160. end
  161. 27 def do_transition(nextstate)
  162. 26298 log(level: 1) { log_transition_state(nextstate) }
  163. 26126 @state = nextstate
  164. end
  165. 27 def log_transition_state(nextstate)
  166. 172 label = host
  167. 172 label = "#{label}(##{@io.fileno})" if nextstate == :connected
  168. 152 "#{label} #{@state} -> #{nextstate}"
  169. end
  170. end
  171. end

lib/httpx/io/udp.rb

100.0% lines covered

35 relevant lines. 35 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "ipaddr"
  3. 27 module HTTPX
  4. 27 class UDP
  5. 27 include Loggable
  6. 27 def initialize(ip, port, options)
  7. 378 @host = ip
  8. 378 @port = port
  9. 378 @io = UDPSocket.new(IPAddr.new(ip).family)
  10. 378 @options = options
  11. end
  12. 27 def to_io
  13. 1085 @io.to_io
  14. end
  15. 27 def connect; end
  16. 27 def connected?
  17. 378 true
  18. end
  19. 27 def close
  20. 378 @io.close
  21. end
  22. 27 if RUBY_ENGINE == "jruby"
  23. # In JRuby, sendmsg_nonblock is not implemented
  24. 1 def write(buffer)
  25. 63 siz = @io.send(buffer.to_s, 0, @host, @port)
  26. 63 log { "WRITE: #{siz} bytes..." }
  27. 63 buffer.shift!(siz)
  28. 63 siz
  29. end
  30. else
  31. 26 def write(buffer)
  32. 716 siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
  33. 716 return 0 if siz == :wait_writable
  34. 716 return if siz.nil?
  35. 716 log { "WRITE: #{siz} bytes..." }
  36. 716 buffer.shift!(siz)
  37. 716 siz
  38. end
  39. end
  40. 27 def read(size, buffer)
  41. 1308 ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
  42. 1308 return 0 if ret == :wait_readable
  43. 704 return if ret.nil?
  44. 704 log { "READ: #{buffer.bytesize} bytes..." }
  45. 704 buffer.bytesize
  46. rescue IOError
  47. end
  48. end
  49. end

lib/httpx/io/unix.rb

94.29% lines covered

35 relevant lines. 33 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 class UNIX < TCP
  4. 27 using URIExtensions
  5. 27 attr_reader :path
  6. 27 alias_method :host, :path
  7. 27 def initialize(origin, path, options)
  8. 28 @addresses = []
  9. 28 @hostname = origin.host
  10. 28 @state = :idle
  11. 28 @options = options
  12. 28 @fallback_protocol = @options.fallback_protocol
  13. 28 if @options.io
  14. 14 @io = case @options.io
  15. when Hash
  16. 7 @options.io[origin.authority]
  17. else
  18. 7 @options.io
  19. end
  20. 14 raise Error, "Given IO objects do not match the request authority" unless @io
  21. 14 @path = @io.path
  22. 14 @keep_open = true
  23. 14 @state = :connected
  24. 14 elsif path
  25. 14 @path = path
  26. else
  27. raise Error, "No path given where to store the socket"
  28. end
  29. 28 @io ||= build_socket
  30. end
  31. 27 def connect
  32. 21 return unless closed?
  33. begin
  34. 21 if @io.closed?
  35. 7 transition(:idle)
  36. 7 @io = build_socket
  37. end
  38. 21 @io.connect_nonblock(Socket.sockaddr_un(@path))
  39. rescue Errno::EISCONN
  40. end
  41. 14 transition(:connected)
  42. rescue Errno::EINPROGRESS,
  43. Errno::EALREADY,
  44. ::IO::WaitReadable
  45. end
  46. 27 def expired?
  47. false
  48. end
  49. skipped # :nocov:
  50. skipped def inspect
  51. skipped "#<#{self.class}:#{object_id} @path=#{@path}) @state=#{@state})>"
  52. skipped end
  53. skipped # :nocov:
  54. 27 private
  55. 27 def build_socket
  56. 21 Socket.new(Socket::PF_UNIX, :STREAM, 0)
  57. end
  58. end
  59. end

lib/httpx/loggable.rb

100.0% lines covered

20 relevant lines. 20 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Loggable
  4. 27 COLORS = {
  5. black: 30,
  6. red: 31,
  7. green: 32,
  8. yellow: 33,
  9. blue: 34,
  10. magenta: 35,
  11. cyan: 36,
  12. white: 37,
  13. }.freeze
  14. 27 USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
  15. 27 def log(
  16. level: @options.debug_level,
  17. color: nil,
  18. debug_level: @options.debug_level,
  19. debug: @options.debug,
  20. &msg
  21. )
  22. 11167268 return unless debug_level >= level
  23. 226325 debug_stream = debug || ($stderr if USE_DEBUG_LOG)
  24. 226325 return unless debug_stream
  25. 2467 klass = self.class
  26. 4760 until (class_name = klass.name)
  27. 446 klass = klass.superclass
  28. end
  29. 2467 message = +"(pid:#{Process.pid} tid:#{Thread.current.object_id}, self:#{class_name}##{object_id}) "
  30. 2467 message << msg.call << "\n"
  31. 2467 message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
  32. 2467 debug_stream << message
  33. end
  34. 27 def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
  35. 1222 log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
  36. end
  37. 27 def log_redact(text, should_redact = @options.debug_redact)
  38. 797 return text.to_s unless should_redact
  39. 112 "[REDACTED]"
  40. end
  41. end
  42. end

lib/httpx/options.rb

98.79% lines covered

165 relevant lines. 163 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "socket"
  3. 27 module HTTPX
  4. # Contains a set of options which are passed and shared across from session to its requests or
  5. # responses.
  6. 27 class Options
  7. 27 BUFFER_SIZE = 1 << 14
  8. 27 WINDOW_SIZE = 1 << 14 # 16K
  9. 27 MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
  10. 27 KEEP_ALIVE_TIMEOUT = 20
  11. 27 SETTINGS_TIMEOUT = 10
  12. 27 CLOSE_HANDSHAKE_TIMEOUT = 10
  13. 27 CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
  14. 27 REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
  15. # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
  16. 2 ip_address_families = begin
  17. 27 list = Socket.ip_address_list
  18. 110 if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? && !a.ipv6_unique_local? }
  19. [Socket::AF_INET6, Socket::AF_INET]
  20. else
  21. 27 [Socket::AF_INET]
  22. end
  23. rescue NotImplementedError
  24. [Socket::AF_INET]
  25. end.freeze
  26. 27 SET_TEMPORARY_NAME = ->(mod, pl = nil) do
  27. 9566 if mod.respond_to?(:set_temporary_name) # ruby 3.4 only
  28. 3601 name = mod.name || "#{mod.superclass.name}(plugin)"
  29. 3601 name = "#{name}/#{pl}" if pl
  30. 3601 mod.set_temporary_name(name)
  31. end
  32. end
  33. 2 DEFAULT_OPTIONS = {
  34. 25 :max_requests => Float::INFINITY,
  35. :debug => nil,
  36. 27 :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
  37. :debug_redact => ENV.key?("HTTPX_DEBUG_REDACT"),
  38. :ssl => EMPTY_HASH,
  39. :http2_settings => { settings_enable_push: 0 }.freeze,
  40. :fallback_protocol => "http/1.1",
  41. :supported_compression_formats => %w[gzip deflate],
  42. :decompress_response_body => true,
  43. :compress_request_body => true,
  44. :timeout => {
  45. connect_timeout: CONNECT_TIMEOUT,
  46. settings_timeout: SETTINGS_TIMEOUT,
  47. close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
  48. operation_timeout: OPERATION_TIMEOUT,
  49. keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
  50. read_timeout: READ_TIMEOUT,
  51. write_timeout: WRITE_TIMEOUT,
  52. request_timeout: REQUEST_TIMEOUT,
  53. },
  54. :headers_class => Class.new(Headers, &SET_TEMPORARY_NAME),
  55. :headers => {},
  56. :window_size => WINDOW_SIZE,
  57. :buffer_size => BUFFER_SIZE,
  58. :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
  59. :request_class => Class.new(Request, &SET_TEMPORARY_NAME),
  60. :response_class => Class.new(Response, &SET_TEMPORARY_NAME),
  61. :request_body_class => Class.new(Request::Body, &SET_TEMPORARY_NAME),
  62. :response_body_class => Class.new(Response::Body, &SET_TEMPORARY_NAME),
  63. :pool_class => Class.new(Pool, &SET_TEMPORARY_NAME),
  64. :connection_class => Class.new(Connection, &SET_TEMPORARY_NAME),
  65. :options_class => Class.new(self, &SET_TEMPORARY_NAME),
  66. :transport => nil,
  67. :addresses => nil,
  68. :persistent => false,
  69. 27 :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
  70. :resolver_options => { cache: true }.freeze,
  71. :pool_options => EMPTY_HASH,
  72. :ip_families => ip_address_families,
  73. :close_on_fork => false,
  74. }.freeze
  75. 27 class << self
  76. 27 def new(options = {})
  77. # let enhanced options go through
  78. 11902 return options if self == Options && options.class < self
  79. 9159 return options if options.is_a?(self)
  80. 4453 super
  81. end
  82. 27 def method_added(meth)
  83. 21077 super
  84. 21077 return unless meth =~ /^option_(.+)$/
  85. 9868 optname = Regexp.last_match(1).to_sym
  86. 9868 attr_reader(optname)
  87. end
  88. end
  89. # creates a new options instance from a given hash, which optionally define the following:
  90. #
  91. # :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
  92. # :debug_level :: the log level of messages (can be 1, 2, or 3).
  93. # :debug_redact :: whether header/body payload should be redacted (defaults to <tt>false</tt>).
  94. # :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::IO::SSL)
  95. # :http2_settings :: a hash of options to be passed to a HTTP2::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
  96. # :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
  97. # like ALPN (defaults to <tt>"http/1.1"</tt>)
  98. # :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
  99. # :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
  100. # :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
  101. # :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
  102. # <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>, <tt>:read_timeout</tt>, <tt>:write_timeout</tt>
  103. # and <tt>:request_timeout</tt>
  104. # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
  105. # :window_size :: number of bytes to read from a socket
  106. # :buffer_size :: internal read and write buffer size in bytes
  107. # :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
  108. # :request_class :: class used to instantiate a request
  109. # :response_class :: class used to instantiate a response
  110. # :headers_class :: class used to instantiate headers
  111. # :request_body_class :: class used to instantiate a request body
  112. # :response_body_class :: class used to instantiate a response body
  113. # :connection_class :: class used to instantiate connections
  114. # :pool_class :: class used to instantiate the session connection pool
  115. # :options_class :: class used to instantiate options
  116. # :transport :: type of transport to use (set to "unix" for UNIX sockets)
  117. # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
  118. # paths should be used for UNIX sockets instead)
  119. # :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
  120. # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
  121. # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
  122. # using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
  123. # :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
  124. # :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
  125. # :ip_families :: which socket families are supported (system-dependent)
  126. # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
  127. # :base_path :: path to prefix given relative paths with (ex: "/v2")
  128. # :max_concurrent_requests :: max number of requests which can be set concurrently
  129. # :max_requests :: max number of requests which can be made on socket before it reconnects.
  130. # :close_on_fork :: whether the session automatically closes when the process is fork (defaults to <tt>false</tt>).
  131. # it only works if the session is persistent (and ruby 3.1 or higher is used).
  132. #
  133. # This list of options are enhanced with each loaded plugin, see the plugin docs for details.
  134. 27 def initialize(options = {})
  135. 4453 do_initialize(options)
  136. 4437 freeze
  137. end
  138. 27 def freeze
  139. 11891 @origin.freeze
  140. 11891 @base_path.freeze
  141. 11891 @timeout.freeze
  142. 11891 @headers.freeze
  143. 11891 @addresses.freeze
  144. 11891 @supported_compression_formats.freeze
  145. 11891 @ssl.freeze
  146. 11891 @http2_settings.freeze
  147. 11891 @pool_options.freeze
  148. 11891 @resolver_options.freeze
  149. 11891 @ip_families.freeze
  150. 11891 super
  151. end
  152. 27 def option_origin(value)
  153. 608 URI(value)
  154. end
  155. 27 def option_base_path(value)
  156. 32 String(value)
  157. end
  158. 27 def option_headers(value)
  159. 7971 headers_class.new(value)
  160. end
  161. 27 def option_timeout(value)
  162. 8581 Hash[value]
  163. end
  164. 27 def option_supported_compression_formats(value)
  165. 7363 Array(value).map(&:to_s)
  166. end
  167. 27 def option_transport(value)
  168. 49 transport = value.to_s
  169. 49 raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
  170. 49 transport
  171. end
  172. 27 def option_addresses(value)
  173. 43 Array(value)
  174. end
  175. 27 def option_ip_families(value)
  176. 7339 Array(value)
  177. end
  178. # number options
  179. 27 %i[
  180. max_concurrent_requests max_requests window_size buffer_size
  181. body_threshold_size debug_level
  182. ].each do |option|
  183. 162 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  184. 6 # converts +v+ into an Integer before setting the +#{option}+ option.
  185. 6 def option_#{option}(value) # def option_max_requests(v)
  186. value = Integer(value) unless value.infinite?
  187. 6 raise TypeError, ":#{option} must be positive" unless value.positive? # raise TypeError, ":max_requests must be positive" unless value.positive?
  188. value
  189. end
  190. OUT
  191. end
  192. # hashable options
  193. 27 %i[ssl http2_settings resolver_options pool_options].each do |option|
  194. 108 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  195. 4 # converts +v+ into an Hash before setting the +#{option}+ option.
  196. 4 def option_#{option}(value) # def option_ssl(v)
  197. Hash[value]
  198. end
  199. OUT
  200. end
  201. 27 %i[
  202. request_class response_class headers_class request_body_class
  203. response_body_class connection_class options_class
  204. pool_class
  205. io fallback_protocol debug debug_redact resolver_class
  206. compress_request_body decompress_response_body
  207. persistent close_on_fork
  208. ].each do |method_name|
  209. 459 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  210. 17 # sets +v+ as the value of the +#{method_name}+ option
  211. 17 def option_#{method_name}(v); v; end # def option_smth(v); v; end
  212. OUT
  213. end
  214. 27 REQUEST_BODY_IVARS = %i[@headers].freeze
  215. 27 def ==(other)
  216. 1963 super || options_equals?(other)
  217. end
  218. 27 def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
  219. # headers and other request options do not play a role, as they are
  220. # relevant only for the request.
  221. 460 ivars = instance_variables - ignore_ivars
  222. 460 other_ivars = other.instance_variables - ignore_ivars
  223. 460 return false if ivars.size != other_ivars.size
  224. 444 return false if ivars.sort != other_ivars.sort
  225. 444 ivars.all? do |ivar|
  226. 11593 instance_variable_get(ivar) == other.instance_variable_get(ivar)
  227. end
  228. end
  229. 27 def merge(other)
  230. 35997 ivar_map = nil
  231. 35997 other_ivars = case other
  232. when Hash
  233. 43027 ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
  234. 25022 ivar_map.keys
  235. else
  236. 10975 other.instance_variables
  237. end
  238. 35997 return self if other_ivars.empty?
  239. 314313 return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == access_option(other, ivar, ivar_map) }
  240. 13229 opts = dup
  241. 13229 other_ivars.each do |ivar|
  242. 99254 v = access_option(other, ivar, ivar_map)
  243. 99254 unless v
  244. 9238 opts.instance_variable_set(ivar, v)
  245. 9238 next
  246. end
  247. 90016 v = opts.__send__(:"option_#{ivar[1..-1]}", v)
  248. 90000 orig_v = instance_variable_get(ivar)
  249. 90000 v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
  250. 90000 opts.instance_variable_set(ivar, v)
  251. end
  252. 13213 opts
  253. end
  254. 27 def to_hash
  255. 3463 instance_variables.each_with_object({}) do |ivar, hs|
  256. 87085 hs[ivar[1..-1].to_sym] = instance_variable_get(ivar)
  257. end
  258. end
  259. 27 def extend_with_plugin_classes(pl)
  260. 7411 if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
  261. 2146 @request_class = @request_class.dup
  262. 2146 SET_TEMPORARY_NAME[@request_class, pl]
  263. 2146 @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
  264. 2146 @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
  265. end
  266. 7411 if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
  267. 2376 @response_class = @response_class.dup
  268. 2376 SET_TEMPORARY_NAME[@response_class, pl]
  269. 2376 @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
  270. 2376 @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
  271. end
  272. 7411 if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
  273. 152 @headers_class = @headers_class.dup
  274. 152 SET_TEMPORARY_NAME[@headers_class, pl]
  275. 152 @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
  276. 152 @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
  277. end
  278. 7411 if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
  279. 330 @request_body_class = @request_body_class.dup
  280. 330 SET_TEMPORARY_NAME[@request_body_class, pl]
  281. 330 @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
  282. 330 @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
  283. end
  284. 7411 if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
  285. 776 @response_body_class = @response_body_class.dup
  286. 776 SET_TEMPORARY_NAME[@response_body_class, pl]
  287. 776 @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
  288. 776 @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
  289. end
  290. 7411 if defined?(pl::PoolMethods)
  291. 606 @pool_class = @pool_class.dup
  292. 606 SET_TEMPORARY_NAME[@pool_class, pl]
  293. 606 @pool_class.__send__(:include, pl::PoolMethods)
  294. end
  295. 7411 if defined?(pl::ConnectionMethods)
  296. 2964 @connection_class = @connection_class.dup
  297. 2964 SET_TEMPORARY_NAME[@connection_class, pl]
  298. 2964 @connection_class.__send__(:include, pl::ConnectionMethods)
  299. end
  300. 7411 return unless defined?(pl::OptionsMethods)
  301. 2997 @options_class = @options_class.dup
  302. 2997 @options_class.__send__(:include, pl::OptionsMethods)
  303. end
  304. 27 private
  305. 27 def do_initialize(options = {})
  306. 4453 defaults = DEFAULT_OPTIONS.merge(options)
  307. 4453 defaults.each do |k, v|
  308. 138933 next if v.nil?
  309. 125574 option_method_name = :"option_#{k}"
  310. 125574 raise Error, "unknown option: #{k}" unless respond_to?(option_method_name)
  311. 125566 value = __send__(option_method_name, v)
  312. 125558 instance_variable_set(:"@#{k}", value)
  313. end
  314. end
  315. 27 def access_option(obj, k, ivar_map)
  316. 351392 case obj
  317. when Hash
  318. 29120 obj[ivar_map[k]]
  319. else
  320. 362407 obj.instance_variable_get(k)
  321. end
  322. end
  323. end
  324. end

lib/httpx/parser/http1.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Parser
  4. 27 class Error < Error; end
  5. 27 class HTTP1
  6. 27 VERSIONS = %w[1.0 1.1].freeze
  7. 27 attr_reader :status_code, :http_version, :headers
  8. 27 def initialize(observer)
  9. 4606 @observer = observer
  10. 4606 @state = :idle
  11. 4606 @buffer = "".b
  12. 4606 @headers = {}
  13. end
  14. 27 def <<(chunk)
  15. 7299 @buffer << chunk
  16. 7299 parse
  17. end
  18. 27 def reset!
  19. 9408 @state = :idle
  20. 9408 @headers = {}
  21. 9408 @content_length = nil
  22. 9408 @_has_trailers = nil
  23. end
  24. 27 def upgrade?
  25. 4615 @upgrade
  26. end
  27. 27 def upgrade_data
  28. 31 @buffer
  29. end
  30. 27 private
  31. 27 def parse
  32. 7299 loop do
  33. 15566 state = @state
  34. 13933 case @state
  35. when :idle
  36. 4968 parse_headline
  37. when :headers, :trailers
  38. 5059 parse_headers
  39. when :data
  40. 5536 parse_data
  41. end
  42. 11375 return if @buffer.empty? || state == @state
  43. end
  44. end
  45. 27 def parse_headline
  46. 4968 idx = @buffer.index("\n")
  47. 4968 return unless idx
  48. 4968 (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
  49. raise(Error, "wrong head line format")
  50. 4960 version, code, _ = m.captures
  51. 4960 raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
  52. 4952 @http_version = version.split(".").map(&:to_i)
  53. 4952 @status_code = code.to_i
  54. 4952 raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
  55. 4944 @buffer = @buffer.byteslice((idx + 1)..-1)
  56. 4944 nextstate(:headers)
  57. end
  58. 27 def parse_headers
  59. 5062 headers = @headers
  60. 5062 buffer = @buffer
  61. 39140 while (idx = buffer.index("\n"))
  62. # @type var line: String
  63. 38548 line = buffer.byteslice(0..idx)
  64. 38548 raise Error, "wrong header format" if line.start_with?("\s", "\t")
  65. 38540 line.lstrip!
  66. 38540 buffer = @buffer = buffer.byteslice((idx + 1)..-1)
  67. 38540 if line.empty?
  68. 4431 case @state
  69. when :headers
  70. 4928 prepare_data(headers)
  71. 4928 @observer.on_headers(headers)
  72. 4301 return unless @state == :headers
  73. # state might have been reset
  74. # in the :headers callback
  75. 4228 nextstate(:data)
  76. 4228 headers.clear
  77. when :trailers
  78. 16 @observer.on_trailers(headers)
  79. 16 headers.clear
  80. 16 nextstate(:complete)
  81. end
  82. 4236 return
  83. end
  84. 33596 separator_index = line.index(":")
  85. 33596 raise Error, "wrong header format" unless separator_index
  86. # @type var key: String
  87. 33588 key = line.byteslice(0..(separator_index - 1))
  88. 33588 key.rstrip! # was lstripped previously!
  89. # @type var value: String
  90. 33588 value = line.byteslice((separator_index + 1)..-1)
  91. 33588 value.strip!
  92. 33588 raise Error, "wrong header format" if value.nil?
  93. 33588 (headers[key.downcase] ||= []) << value
  94. end
  95. end
  96. 27 def parse_data
  97. 5536 if @buffer.respond_to?(:each)
  98. 159 @buffer.each do |chunk|
  99. 226 @observer.on_data(chunk)
  100. end
  101. 5376 elsif @content_length
  102. # @type var data: String
  103. 5337 data = @buffer.byteslice(0, @content_length)
  104. 5337 @buffer = @buffer.byteslice(@content_length..-1) || "".b
  105. 4772 @content_length -= data.bytesize
  106. 5337 @observer.on_data(data)
  107. 5315 data.clear
  108. else
  109. 40 @observer.on_data(@buffer)
  110. 40 @buffer.clear
  111. end
  112. 5506 return unless no_more_data?
  113. 4078 @buffer = @buffer.to_s
  114. 4078 if @_has_trailers
  115. 16 nextstate(:trailers)
  116. else
  117. 4062 nextstate(:complete)
  118. end
  119. end
  120. 27 def prepare_data(headers)
  121. 4928 @upgrade = headers.key?("upgrade")
  122. 4928 @_has_trailers = headers.key?("trailer")
  123. 4928 if (tr_encodings = headers["transfer-encoding"])
  124. 114 tr_encodings.reverse_each do |tr_encoding|
  125. 114 tr_encoding.split(/ *, */).each do |encoding|
  126. 100 case encoding
  127. when "chunked"
  128. 114 @buffer = Transcoder::Chunker::Decoder.new(@buffer, @_has_trailers)
  129. end
  130. end
  131. end
  132. else
  133. 4814 @content_length = headers["content-length"][0].to_i if headers.key?("content-length")
  134. end
  135. end
  136. 27 def no_more_data?
  137. 5506 if @content_length
  138. 5315 @content_length <= 0
  139. 190 elsif @buffer.respond_to?(:finished?)
  140. 151 @buffer.finished?
  141. else
  142. 40 false
  143. end
  144. end
  145. 27 def nextstate(state)
  146. 13266 @state = state
  147. 11899 case state
  148. when :headers
  149. 4944 @observer.on_start
  150. when :complete
  151. 4078 @observer.on_complete
  152. 584 reset!
  153. 584 nextstate(:idle) unless @buffer.empty?
  154. end
  155. end
  156. end
  157. end
  158. end

lib/httpx/plugins/auth.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds a shim +authorization+ method to the session, which will fill
  6. # the HTTP Authorization header, and another, +bearer_auth+, which fill the "Bearer " prefix
  7. # in its value.
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/Auth#auth
  10. #
  11. 8 module Auth
  12. 8 module InstanceMethods
  13. 8 def authorization(token)
  14. 144 with(headers: { "authorization" => token })
  15. end
  16. 8 def bearer_auth(token)
  17. 16 authorization("Bearer #{token}")
  18. end
  19. end
  20. end
  21. 8 register_plugin :auth, Auth
  22. end
  23. end

lib/httpx/plugins/auth/basic.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 9 require "httpx/base64"
  3. 9 module HTTPX
  4. 9 module Plugins
  5. 9 module Authentication
  6. 9 class Basic
  7. 9 def initialize(user, password, **)
  8. 274 @user = user
  9. 274 @password = password
  10. end
  11. 9 def authenticate(*)
  12. 255 "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
  13. end
  14. end
  15. end
  16. end
  17. end

lib/httpx/plugins/auth/digest.rb

100.0% lines covered

61 relevant lines. 61 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "time"
  3. 8 require "securerandom"
  4. 8 require "digest"
  5. 8 module HTTPX
  6. 8 module Plugins
  7. 8 module Authentication
  8. 8 class Digest
  9. 8 def initialize(user, password, hashed: false, **)
  10. 176 @user = user
  11. 176 @password = password
  12. 176 @nonce = 0
  13. 176 @hashed = hashed
  14. end
  15. 8 def can_authenticate?(authenticate)
  16. 160 authenticate && /Digest .*/.match?(authenticate)
  17. end
  18. 8 def authenticate(request, authenticate)
  19. 160 "Digest #{generate_header(request.verb, request.path, authenticate)}"
  20. end
  21. 8 private
  22. 8 def generate_header(meth, uri, authenticate)
  23. # discard first token, it's Digest
  24. 160 auth_info = authenticate[/^(\w+) (.*)/, 2]
  25. 160 params = auth_info.split(/ *, */)
  26. 832 .to_h { |val| val.split("=", 2) }
  27. 832 .transform_values { |v| v.delete("\"") }
  28. 160 nonce = params["nonce"]
  29. 160 nc = next_nonce
  30. # verify qop
  31. 160 qop = params["qop"]
  32. 160 if params["algorithm"] =~ /(.*?)(-sess)?$/
  33. 144 alg = Regexp.last_match(1)
  34. 144 algorithm = ::Digest.const_get(alg)
  35. 144 raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
  36. 144 sess = Regexp.last_match(2)
  37. else
  38. 16 algorithm = ::Digest::MD5
  39. end
  40. 160 if qop || sess
  41. 160 cnonce = make_cnonce
  42. 160 nc = format("%<nonce>08x", nonce: nc)
  43. end
  44. 160 a1 = if sess
  45. 4 [
  46. 32 (@hashed ? @password : algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}")),
  47. nonce,
  48. cnonce,
  49. 3 ].join ":"
  50. else
  51. 128 @hashed ? @password : "#{@user}:#{params["realm"]}:#{@password}"
  52. end
  53. 160 ha1 = algorithm.hexdigest(a1)
  54. 160 ha2 = algorithm.hexdigest("#{meth}:#{uri}")
  55. 160 request_digest = [ha1, nonce]
  56. 160 request_digest.push(nc, cnonce, qop) if qop
  57. 160 request_digest << ha2
  58. 160 request_digest = request_digest.join(":")
  59. 40 header = [
  60. 140 %(username="#{@user}"),
  61. 20 %(nonce="#{nonce}"),
  62. 20 %(uri="#{uri}"),
  63. 20 %(response="#{algorithm.hexdigest(request_digest)}"),
  64. ]
  65. 160 header << %(realm="#{params["realm"]}") if params.key?("realm")
  66. 160 header << %(algorithm=#{params["algorithm"]}) if params.key?("algorithm")
  67. 160 header << %(cnonce="#{cnonce}") if cnonce
  68. 160 header << %(nc=#{nc})
  69. 160 header << %(qop=#{qop}) if qop
  70. 160 header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
  71. 160 header.join ", "
  72. end
  73. 8 def make_cnonce
  74. 180 ::Digest::MD5.hexdigest [
  75. Time.now.to_i,
  76. Process.pid,
  77. SecureRandom.random_number(2**32),
  78. ].join ":"
  79. end
  80. 8 def next_nonce
  81. 140 @nonce += 1
  82. end
  83. end
  84. end
  85. end
  86. end

lib/httpx/plugins/auth/ntlm.rb

100.0% lines covered

19 relevant lines. 19 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 3 require "httpx/base64"
  3. 3 require "ntlm"
  4. 3 module HTTPX
  5. 3 module Plugins
  6. 3 module Authentication
  7. 3 class Ntlm
  8. 3 def initialize(user, password, domain: nil)
  9. 4 @user = user
  10. 4 @password = password
  11. 4 @domain = domain
  12. end
  13. 3 def can_authenticate?(authenticate)
  14. 2 authenticate && /NTLM .*/.match?(authenticate)
  15. end
  16. 3 def negotiate
  17. 4 "NTLM #{NTLM.negotiate(domain: @domain).to_base64}"
  18. end
  19. 3 def authenticate(_req, www)
  20. 2 challenge = www[/NTLM (.*)/, 1]
  21. 2 challenge = Base64.decode64(challenge)
  22. 2 ntlm_challenge = NTLM.authenticate(challenge, @user, @domain, @password).to_base64
  23. 2 "NTLM #{ntlm_challenge}"
  24. end
  25. end
  26. end
  27. end
  28. end

lib/httpx/plugins/auth/socks5.rb

100.0% lines covered

11 relevant lines. 11 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 module Plugins
  4. 10 module Authentication
  5. 10 class Socks5
  6. 10 def initialize(user, password, **)
  7. 48 @user = user
  8. 48 @password = password
  9. end
  10. 10 def can_authenticate?(*)
  11. 48 @user && @password
  12. end
  13. 10 def authenticate(*)
  14. 48 [0x01, @user.bytesize, @user, @password.bytesize, @password].pack("CCA*CA*")
  15. end
  16. end
  17. end
  18. end
  19. end

lib/httpx/plugins/aws_sdk_authentication.rb

100.0% lines covered

43 relevant lines. 43 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
  6. #
  7. # It requires the "aws-sdk-core" gem.
  8. #
  9. 8 module AwsSdkAuthentication
  10. # Mock configuration, to be used only when resolving credentials
  11. 8 class Configuration
  12. 8 attr_reader :profile
  13. 8 def initialize(profile)
  14. 32 @profile = profile
  15. end
  16. 8 def respond_to_missing?(*)
  17. 16 true
  18. end
  19. 8 def method_missing(*); end
  20. end
  21. #
  22. # encapsulates access to an AWS SDK credentials store.
  23. #
  24. 8 class Credentials
  25. 8 def initialize(aws_credentials)
  26. 16 @aws_credentials = aws_credentials
  27. end
  28. 8 def username
  29. 16 @aws_credentials.access_key_id
  30. end
  31. 8 def password
  32. 16 @aws_credentials.secret_access_key
  33. end
  34. 8 def security_token
  35. 16 @aws_credentials.session_token
  36. end
  37. end
  38. 8 class << self
  39. 8 def load_dependencies(_klass)
  40. 16 require "aws-sdk-core"
  41. end
  42. 8 def configure(klass)
  43. 16 klass.plugin(:aws_sigv4)
  44. end
  45. 8 def extra_options(options)
  46. 16 options.merge(max_concurrent_requests: 1)
  47. end
  48. 8 def credentials(profile)
  49. 16 mock_configuration = Configuration.new(profile)
  50. 16 Credentials.new(Aws::CredentialProviderChain.new(mock_configuration).resolve)
  51. end
  52. 8 def region(profile)
  53. # https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb#L62
  54. 16 keys = %w[AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION]
  55. 16 env_region = ENV.values_at(*keys).compact.first
  56. 16 env_region = nil if env_region == ""
  57. 16 cfg_region = Aws.shared_config.region(profile: profile)
  58. 16 env_region || cfg_region
  59. end
  60. end
  61. # adds support for the following options:
  62. #
  63. # :aws_profile :: AWS account profile to retrieve credentials from.
  64. 8 module OptionsMethods
  65. 8 def option_aws_profile(value)
  66. 80 String(value)
  67. end
  68. end
  69. 8 module InstanceMethods
  70. #
  71. # aws_authentication
  72. # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
  73. # aws_authentication()
  74. #
  75. 8 def aws_sdk_authentication(
  76. credentials: AwsSdkAuthentication.credentials(@options.aws_profile),
  77. region: AwsSdkAuthentication.region(@options.aws_profile),
  78. **options
  79. )
  80. 16 aws_sigv4_authentication(
  81. credentials: credentials,
  82. region: region,
  83. provider_prefix: "aws",
  84. header_provider_field: "amz",
  85. **options
  86. )
  87. end
  88. 8 alias_method :aws_auth, :aws_sdk_authentication
  89. end
  90. end
  91. 8 register_plugin :aws_sdk_authentication, AwsSdkAuthentication
  92. end
  93. end

lib/httpx/plugins/aws_sigv4.rb

100.0% lines covered

121 relevant lines. 121 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds AWS Sigv4 authentication.
  6. #
  7. # https://docs.aws.amazon.com/IAM/latest/UserGuide/signing-elements.html
  8. #
  9. # https://gitlab.com/os85/httpx/wikis/AWS-SigV4
  10. #
  11. 8 module AWSSigV4
  12. 8 Credentials = Struct.new(:username, :password, :security_token)
  13. # Signs requests using the AWS sigv4 signing.
  14. 8 class Signer
  15. 8 def initialize(
  16. service:,
  17. region:,
  18. credentials: nil,
  19. username: nil,
  20. password: nil,
  21. security_token: nil,
  22. provider_prefix: "aws",
  23. header_provider_field: "amz",
  24. unsigned_headers: [],
  25. apply_checksum_header: true,
  26. algorithm: "SHA256"
  27. )
  28. 152 @credentials = credentials || Credentials.new(username, password, security_token)
  29. 152 @service = service
  30. 152 @region = region
  31. 152 @unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
  32. 152 @unsigned_headers << "authorization"
  33. 152 @unsigned_headers << "x-amzn-trace-id"
  34. 152 @unsigned_headers << "expect"
  35. 152 @apply_checksum_header = apply_checksum_header
  36. 152 @provider_prefix = provider_prefix
  37. 152 @header_provider_field = header_provider_field
  38. 152 @algorithm = algorithm
  39. end
  40. 8 def sign!(request)
  41. 152 lower_provider_prefix = "#{@provider_prefix}4"
  42. 152 upper_provider_prefix = lower_provider_prefix.upcase
  43. 152 downcased_algorithm = @algorithm.downcase
  44. 152 datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
  45. 152 date = datetime[0, 8]
  46. 152 content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
  47. 144 request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
  48. 144 request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
  49. 144 signature_headers = request.headers.each.reject do |k, _|
  50. 984 @unsigned_headers.include?(k)
  51. end
  52. # aws sigv4 needs to declare the host, regardless of protocol version
  53. 144 signature_headers << ["host", request.authority] unless request.headers.key?("host")
  54. 144 signature_headers.sort_by!(&:first)
  55. 144 signed_headers = signature_headers.map(&:first).join(";")
  56. 144 canonical_headers = signature_headers.map do |k, v|
  57. # eliminate whitespace between value fields, unless it's a quoted value
  58. 847 "#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
  59. end.join
  60. # canonical request
  61. 144 creq = "#{request.verb}" \
  62. 18 "\n#{request.canonical_path}" \
  63. 18 "\n#{request.canonical_query}" \
  64. 18 "\n#{canonical_headers}" \
  65. 18 "\n#{signed_headers}" \
  66. 18 "\n#{content_hashed}"
  67. 144 credential_scope = "#{date}" \
  68. 18 "/#{@region}" \
  69. 18 "/#{@service}" \
  70. 18 "/#{lower_provider_prefix}_request"
  71. 144 algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
  72. # string to sign
  73. 144 sts = "#{algo_line}" \
  74. 18 "\n#{datetime}" \
  75. 18 "\n#{credential_scope}" \
  76. 18 "\n#{OpenSSL::Digest.new(@algorithm).hexdigest(creq)}"
  77. # signature
  78. 144 k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
  79. 144 k_region = hmac(k_date, @region)
  80. 144 k_service = hmac(k_region, @service)
  81. 144 k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
  82. 144 sig = hexhmac(k_credentials, sts)
  83. 144 credential = "#{@credentials.username}/#{credential_scope}"
  84. # apply signature
  85. 126 request.headers["authorization"] =
  86. 18 "#{algo_line} " \
  87. 18 "Credential=#{credential}, " \
  88. 18 "SignedHeaders=#{signed_headers}, " \
  89. 18 "Signature=#{sig}"
  90. end
  91. 8 private
  92. 8 def hexdigest(value)
  93. 144 digest = OpenSSL::Digest.new(@algorithm)
  94. 144 if value.respond_to?(:read)
  95. 32 if value.respond_to?(:to_path)
  96. # files, pathnames
  97. 8 digest.file(value.to_path).hexdigest
  98. else
  99. # gzipped request bodies
  100. 24 raise Error, "request body must be rewindable" unless value.respond_to?(:rewind)
  101. 24 buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  102. 2 begin
  103. 24 IO.copy_stream(value, buffer)
  104. 24 buffer.flush
  105. 24 digest.file(buffer.to_path).hexdigest
  106. ensure
  107. 24 value.rewind
  108. 24 buffer.close
  109. 24 buffer.unlink
  110. end
  111. end
  112. else
  113. # error on endless generators
  114. 112 raise Error, "hexdigest for endless enumerators is not supported" if value.unbounded_body?
  115. 104 mb_buffer = value.each.with_object("".b) do |chunk, b|
  116. 56 b << chunk
  117. 56 break if b.bytesize >= 1024 * 1024
  118. end
  119. 104 digest.hexdigest(mb_buffer)
  120. end
  121. end
  122. 8 def hmac(key, value)
  123. 576 OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
  124. end
  125. 8 def hexhmac(key, value)
  126. 144 OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
  127. end
  128. end
  129. 8 class << self
  130. 8 def load_dependencies(*)
  131. 152 require "set"
  132. 152 require "digest/sha2"
  133. 152 require "cgi/escape"
  134. end
  135. 8 def configure(klass)
  136. 152 klass.plugin(:expect)
  137. end
  138. end
  139. # adds support for the following options:
  140. #
  141. # :sigv4_signer :: instance of HTTPX::Plugins::AWSSigV4 used to sign requests.
  142. 8 module OptionsMethods
  143. 8 def option_sigv4_signer(value)
  144. 320 value.is_a?(Signer) ? value : Signer.new(value)
  145. end
  146. end
  147. 8 module InstanceMethods
  148. 8 def aws_sigv4_authentication(**options)
  149. 152 with(sigv4_signer: Signer.new(**options))
  150. end
  151. 8 def build_request(*)
  152. 152 request = super
  153. 152 return request if request.headers.key?("authorization")
  154. 152 signer = request.options.sigv4_signer
  155. 152 return request unless signer
  156. 152 signer.sign!(request)
  157. 144 request
  158. end
  159. end
  160. 8 module RequestMethods
  161. 8 def canonical_path
  162. 144 path = uri.path.dup
  163. 144 path << "/" if path.empty?
  164. 176 path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
  165. end
  166. 8 def canonical_query
  167. 176 params = query.split("&")
  168. # params = params.map { |p| p.match(/=/) ? p : p + '=' }
  169. # From: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
  170. # Sort the parameter names by character code point in ascending order.
  171. # Parameters with duplicate names should be sorted by value.
  172. #
  173. # Default sort <=> in JRuby will swap members
  174. # occasionally when <=> is 0 (considered still sorted), but this
  175. # causes our normalized query string to not match the sent querystring.
  176. # When names match, we then sort by their values. When values also
  177. # match then we sort by their original order
  178. 176 params.each.with_index.sort do |a, b|
  179. 64 a, a_offset = a
  180. 64 b, b_offset = b
  181. 64 a_name, a_value = a.split("=", 2)
  182. 64 b_name, b_value = b.split("=", 2)
  183. 64 if a_name == b_name
  184. 32 if a_value == b_value
  185. 16 a_offset <=> b_offset
  186. else
  187. 16 a_value <=> b_value
  188. end
  189. else
  190. 32 a_name <=> b_name
  191. end
  192. end.map(&:first).join("&")
  193. end
  194. end
  195. end
  196. 8 register_plugin :aws_sigv4, AWSSigV4
  197. end
  198. end

lib/httpx/plugins/basic_auth.rb

100.0% lines covered

12 relevant lines. 12 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Basic Auth (https://datatracker.ietf.org/doc/html/rfc7617)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#basic-auth
  8. #
  9. 8 module BasicAuth
  10. 8 class << self
  11. 8 def load_dependencies(_klass)
  12. 112 require_relative "auth/basic"
  13. end
  14. 8 def configure(klass)
  15. 112 klass.plugin(:auth)
  16. end
  17. end
  18. 8 module InstanceMethods
  19. 8 def basic_auth(user, password)
  20. 128 authorization(Authentication::Basic.new(user, password).authenticate)
  21. end
  22. end
  23. end
  24. 8 register_plugin :basic_auth, BasicAuth
  25. end
  26. end

lib/httpx/plugins/brotli.rb

100.0% lines covered

25 relevant lines. 25 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module Brotli
  5. 6 class Deflater < Transcoder::Deflater
  6. 6 def deflate(chunk)
  7. 24 return unless chunk
  8. 12 ::Brotli.deflate(chunk)
  9. end
  10. end
  11. 6 module RequestBodyClassMethods
  12. 6 def initialize_deflater_body(body, encoding)
  13. 24 return Brotli.encode(body) if encoding == "br"
  14. 12 super
  15. end
  16. end
  17. 6 module ResponseBodyClassMethods
  18. 6 def initialize_inflater_by_encoding(encoding, response, **kwargs)
  19. 24 return Brotli.decode(response, **kwargs) if encoding == "br"
  20. 12 super
  21. end
  22. end
  23. 6 module_function
  24. 6 def load_dependencies(*)
  25. 24 require "brotli"
  26. end
  27. 6 def self.extra_options(options)
  28. 24 options.merge(supported_compression_formats: %w[br] + options.supported_compression_formats)
  29. end
  30. 6 def encode(body)
  31. 12 Deflater.new(body)
  32. end
  33. 6 def decode(_response, **)
  34. 12 ::Brotli.method(:inflate)
  35. end
  36. end
  37. 6 register_plugin :brotli, Brotli
  38. end
  39. end

lib/httpx/plugins/callbacks.rb

100.0% lines covered

55 relevant lines. 55 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Plugins
  4. #
  5. # This plugin adds suppoort for callbacks around the request/response lifecycle.
  6. #
  7. # https://gitlab.com/os85/httpx/-/wikis/Events
  8. #
  9. 27 module Callbacks
  10. 27 CALLBACKS = %i[
  11. connection_opened connection_closed
  12. request_error
  13. request_started request_body_chunk request_completed
  14. response_started response_body_chunk response_completed
  15. ].freeze
  16. # connection closed user-space errors happen after errors can be surfaced to requests,
  17. # so they need to pierce through the scheduler, which is only possible by simulating an
  18. # interrupt.
  19. 27 class CallbackError < Exception; end # rubocop:disable Lint/InheritException
  20. 27 module InstanceMethods
  21. 27 include HTTPX::Callbacks
  22. 27 CALLBACKS.each do |meth|
  23. 243 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  24. 9 def on_#{meth}(&blk) # def on_connection_opened(&blk)
  25. 9 on(:#{meth}, &blk) # on(:connection_opened, &blk)
  26. self # self
  27. end # end
  28. MOD
  29. end
  30. 27 private
  31. 27 def branch(options, &blk)
  32. 16 super(options).tap do |sess|
  33. 16 CALLBACKS.each do |cb|
  34. 144 next unless callbacks_for?(cb)
  35. 16 sess.callbacks(cb).concat(callbacks(cb))
  36. end
  37. 16 sess.wrap(&blk) if blk
  38. end
  39. end
  40. 27 def do_init_connection(connection, selector)
  41. 225 super
  42. 225 connection.on(:open) do
  43. 195 next unless connection.current_session == self
  44. 195 emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
  45. end
  46. 225 connection.on(:close) do
  47. 211 next unless connection.current_session == self
  48. 211 emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
  49. end
  50. 225 connection
  51. end
  52. 27 def set_request_callbacks(request)
  53. 227 super
  54. 227 request.on(:headers) do
  55. 179 emit_or_callback_error(:request_started, request)
  56. end
  57. 227 request.on(:body_chunk) do |chunk|
  58. 16 emit_or_callback_error(:request_body_chunk, request, chunk)
  59. end
  60. 227 request.on(:done) do
  61. 163 emit_or_callback_error(:request_completed, request)
  62. end
  63. 227 request.on(:response_started) do |res|
  64. 179 if res.is_a?(Response)
  65. 147 emit_or_callback_error(:response_started, request, res)
  66. 131 res.on(:chunk_received) do |chunk|
  67. 155 emit_or_callback_error(:response_body_chunk, request, res, chunk)
  68. end
  69. else
  70. 32 emit_or_callback_error(:request_error, request, res.error)
  71. end
  72. end
  73. 227 request.on(:response) do |res|
  74. 131 emit_or_callback_error(:response_completed, request, res)
  75. end
  76. end
  77. 27 def emit_or_callback_error(*args)
  78. 1213 emit(*args)
  79. rescue StandardError => e
  80. 128 ex = CallbackError.new(e.message)
  81. 128 ex.set_backtrace(e.backtrace)
  82. 128 raise ex
  83. end
  84. 27 def receive_requests(*)
  85. 227 super
  86. rescue CallbackError => e
  87. 120 raise e.cause
  88. end
  89. 27 def close(*)
  90. 225 super
  91. rescue CallbackError => e
  92. 8 raise e.cause
  93. end
  94. end
  95. end
  96. 27 register_plugin :callbacks, Callbacks
  97. end
  98. end

lib/httpx/plugins/circuit_breaker.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin implements a circuit breaker around connection errors.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Circuit-Breaker
  8. #
  9. 8 module CircuitBreaker
  10. 8 using URIExtensions
  11. 8 def self.load_dependencies(*)
  12. 56 require_relative "circuit_breaker/circuit"
  13. 56 require_relative "circuit_breaker/circuit_store"
  14. end
  15. 8 def self.extra_options(options)
  16. 56 options.merge(
  17. circuit_breaker_max_attempts: 3,
  18. circuit_breaker_reset_attempts_in: 60,
  19. circuit_breaker_break_in: 60,
  20. circuit_breaker_half_open_drip_rate: 1
  21. )
  22. end
  23. 8 module InstanceMethods
  24. 8 include HTTPX::Callbacks
  25. 8 def initialize(*)
  26. 56 super
  27. 56 @circuit_store = CircuitStore.new(@options)
  28. end
  29. 8 %i[circuit_open].each do |meth|
  30. 8 class_eval(<<-MOD, __FILE__, __LINE__ + 1)
  31. 1 def on_#{meth}(&blk) # def on_circuit_open(&blk)
  32. 1 on(:#{meth}, &blk) # on(:circuit_open, &blk)
  33. self # self
  34. end # end
  35. MOD
  36. end
  37. 8 private
  38. 8 def send_requests(*requests)
  39. # @type var short_circuit_responses: Array[response]
  40. 224 short_circuit_responses = []
  41. # run all requests through the circuit breaker, see if the circuit is
  42. # open for any of them.
  43. 224 real_requests = requests.each_with_index.with_object([]) do |(req, idx), real_reqs|
  44. 224 short_circuit_response = @circuit_store.try_respond(req)
  45. 224 if short_circuit_response.nil?
  46. 176 real_reqs << req
  47. 176 next
  48. end
  49. 42 short_circuit_responses[idx] = short_circuit_response
  50. end
  51. # run requests for the remainder
  52. 224 unless real_requests.empty?
  53. 176 responses = super(*real_requests)
  54. 176 real_requests.each_with_index do |request, idx|
  55. 154 short_circuit_responses[requests.index(request)] = responses[idx]
  56. end
  57. end
  58. 224 short_circuit_responses
  59. end
  60. 8 def set_request_callbacks(request)
  61. 224 super
  62. 224 request.on(:response) do |response|
  63. 176 emit(:circuit_open, request) if try_circuit_open(request, response)
  64. end
  65. end
  66. 8 def try_circuit_open(request, response)
  67. 176 if response.is_a?(ErrorResponse)
  68. 112 case response.error
  69. when RequestTimeoutError
  70. 80 @circuit_store.try_open(request.uri, response)
  71. else
  72. 48 @circuit_store.try_open(request.origin, response)
  73. end
  74. 48 elsif (break_on = request.options.circuit_breaker_break_on) && break_on.call(response)
  75. 16 @circuit_store.try_open(request.uri, response)
  76. else
  77. 32 @circuit_store.try_close(request.uri)
  78. 12 nil
  79. end
  80. end
  81. end
  82. # adds support for the following options:
  83. #
  84. # :circuit_breaker_max_attempts :: the number of attempts the circuit allows, before it is opened (defaults to <tt>3</tt>).
  85. # :circuit_breaker_reset_attempts_in :: the time a circuit stays open at most, before it resets (defaults to <tt>60</tt>).
  86. # :circuit_breaker_break_on :: callable defining an alternative rule for a response to break
  87. # (i.e. <tt>->(res) { res.status == 429 } </tt>)
  88. # :circuit_breaker_break_in :: the time that must elapse before an open circuit can transit to the half-open state
  89. # (defaults to <tt><60</tt>).
  90. # :circuit_breaker_half_open_drip_rate :: the rate of requests a circuit allows to be performed when in an half-open state
  91. # (defaults to <tt>1</tt>).
  92. 8 module OptionsMethods
  93. 8 def option_circuit_breaker_max_attempts(value)
  94. 112 attempts = Integer(value)
  95. 112 raise TypeError, ":circuit_breaker_max_attempts must be positive" unless attempts.positive?
  96. 112 attempts
  97. end
  98. 8 def option_circuit_breaker_reset_attempts_in(value)
  99. 64 timeout = Float(value)
  100. 64 raise TypeError, ":circuit_breaker_reset_attempts_in must be positive" unless timeout.positive?
  101. 64 timeout
  102. end
  103. 8 def option_circuit_breaker_break_in(value)
  104. 88 timeout = Float(value)
  105. 88 raise TypeError, ":circuit_breaker_break_in must be positive" unless timeout.positive?
  106. 88 timeout
  107. end
  108. 8 def option_circuit_breaker_half_open_drip_rate(value)
  109. 88 ratio = Float(value)
  110. 88 raise TypeError, ":circuit_breaker_half_open_drip_rate must be a number between 0 and 1" unless (0..1).cover?(ratio)
  111. 88 ratio
  112. end
  113. 8 def option_circuit_breaker_break_on(value)
  114. 16 raise TypeError, ":circuit_breaker_break_on must be called with the response" unless value.respond_to?(:call)
  115. 16 value
  116. end
  117. end
  118. end
  119. 8 register_plugin :circuit_breaker, CircuitBreaker
  120. end
  121. end

lib/httpx/plugins/circuit_breaker/circuit.rb

100.0% lines covered

47 relevant lines. 47 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::CircuitBreaker
  4. #
  5. # A circuit is assigned to a given absoolute url or origin.
  6. #
  7. # It sets +max_attempts+, the number of attempts the circuit allows, before it is opened.
  8. # It sets +reset_attempts_in+, the time a circuit stays open at most, before it resets.
  9. # It sets +break_in+, the time that must elapse before an open circuit can transit to the half-open state.
  10. # It sets +circuit_breaker_half_open_drip_rate+, the rate of requests a circuit allows to be performed when in an half-open state.
  11. #
  12. 8 class Circuit
  13. 8 def initialize(max_attempts, reset_attempts_in, break_in, circuit_breaker_half_open_drip_rate)
  14. 56 @max_attempts = max_attempts
  15. 56 @reset_attempts_in = reset_attempts_in
  16. 56 @break_in = break_in
  17. 56 @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
  18. 56 @attempts = 0
  19. 56 total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
  20. 56 @drip_factor = (@max_attempts / total_real_attempts).round
  21. 56 @state = :closed
  22. end
  23. 8 def respond
  24. 224 try_close
  25. 196 case @state
  26. when :closed
  27. 51 nil
  28. when :half_open
  29. 49 @attempts += 1
  30. # do real requests while drip rate valid
  31. 56 if (@real_attempts % @drip_factor).zero?
  32. 35 @real_attempts += 1
  33. 35 return
  34. end
  35. 16 @response
  36. when :open
  37. 32 @response
  38. end
  39. end
  40. 8 def try_open(response)
  41. 126 case @state
  42. when :closed
  43. 120 now = Utils.now
  44. 120 if @attempts.positive?
  45. # reset if error happened long ago
  46. 48 @attempts = 0 if now - @attempted_at > @reset_attempts_in
  47. else
  48. 72 @attempted_at = now
  49. end
  50. 105 @attempts += 1
  51. 120 return unless @attempts >= @max_attempts
  52. 64 @state = :open
  53. 64 @opened_at = now
  54. 64 @response = response
  55. when :half_open
  56. # open immediately
  57. 24 @state = :open
  58. 24 @attempted_at = @opened_at = Utils.now
  59. 24 @response = response
  60. end
  61. end
  62. 8 def try_close
  63. 224 case @state
  64. when :closed
  65. 51 nil
  66. when :half_open
  67. # do not close circuit unless attempts exhausted
  68. 48 return unless @attempts >= @max_attempts
  69. # reset!
  70. 16 @attempts = 0
  71. 16 @opened_at = @attempted_at = @response = nil
  72. 16 @state = :closed
  73. when :open
  74. 72 if Utils.elapsed_time(@opened_at) > @break_in
  75. 40 @state = :half_open
  76. 40 @attempts = 0
  77. 40 @real_attempts = 0
  78. end
  79. end
  80. end
  81. end
  82. end
  83. end

lib/httpx/plugins/circuit_breaker/circuit_store.rb

100.0% lines covered

23 relevant lines. 23 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX::Plugins::CircuitBreaker
  3. 8 using HTTPX::URIExtensions
  4. 8 class CircuitStore
  5. 8 def initialize(options)
  6. 56 @circuits = Hash.new do |h, k|
  7. 49 h[k] = Circuit.new(
  8. options.circuit_breaker_max_attempts,
  9. options.circuit_breaker_reset_attempts_in,
  10. options.circuit_breaker_break_in,
  11. options.circuit_breaker_half_open_drip_rate
  12. )
  13. end
  14. 56 @circuits_mutex = Thread::Mutex.new
  15. end
  16. 8 def try_open(uri, response)
  17. 288 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(uri) }
  18. 144 circuit.try_open(response)
  19. end
  20. 8 def try_close(uri)
  21. 32 circuit = @circuits_mutex.synchronize do
  22. 32 return unless @circuits.key?(uri.origin) || @circuits.key?(uri.to_s)
  23. 32 get_circuit_for_uri(uri)
  24. end
  25. 32 circuit.try_close
  26. end
  27. # if circuit is open, it'll respond with the stored response.
  28. # if not, nil.
  29. 8 def try_respond(request)
  30. 448 circuit = @circuits_mutex.synchronize { get_circuit_for_uri(request.uri) }
  31. 224 circuit.respond
  32. end
  33. 8 private
  34. 8 def get_circuit_for_uri(uri)
  35. 400 if uri.respond_to?(:origin) && @circuits.key?(uri.origin)
  36. 288 @circuits[uri.origin]
  37. else
  38. 112 @circuits[uri.to_s]
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/content_digest.rb

100.0% lines covered

102 relevant lines. 102 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds `Content-Digest` headers to requests
  6. # and can validate these headers on responses
  7. #
  8. # https://datatracker.ietf.org/doc/html/rfc9530
  9. #
  10. 8 module ContentDigest
  11. 8 class Error < HTTPX::Error; end
  12. # Error raised on response "content-digest" header validation.
  13. 8 class ValidationError < Error
  14. 8 attr_reader :response
  15. 8 def initialize(message, response)
  16. 48 super(message)
  17. 48 @response = response
  18. end
  19. end
  20. 8 class MissingContentDigestError < ValidationError; end
  21. 8 class InvalidContentDigestError < ValidationError; end
  22. 2 SUPPORTED_ALGORITHMS = {
  23. 6 "sha-256" => OpenSSL::Digest::SHA256,
  24. "sha-512" => OpenSSL::Digest::SHA512,
  25. }.freeze
  26. 8 class << self
  27. 8 def extra_options(options)
  28. 208 options.merge(encode_content_digest: true, validate_content_digest: false, content_digest_algorithm: "sha-256")
  29. end
  30. end
  31. # add support for the following options:
  32. #
  33. # :content_digest_algorithm :: the digest algorithm to use. Currently supports `sha-256` and `sha-512`. (defaults to `sha-256`)
  34. # :encode_content_digest :: whether a <tt>Content-Digest</tt> header should be computed for the request;
  35. # can also be a callable object (i.e. <tt>->(req) { ... }</tt>, defaults to <tt>true</tt>)
  36. # :validate_content_digest :: whether a <tt>Content-Digest</tt> header in the response should be validated;
  37. # can also be a callable object (i.e. <tt>->(res) { ... }</tt>, defaults to <tt>false</tt>)
  38. 8 module OptionsMethods
  39. 8 def option_content_digest_algorithm(value)
  40. 216 raise TypeError, ":content_digest_algorithm must be one of 'sha-256', 'sha-512'" unless SUPPORTED_ALGORITHMS.key?(value)
  41. 216 value
  42. end
  43. 8 def option_encode_content_digest(value)
  44. 208 value
  45. end
  46. 8 def option_validate_content_digest(value)
  47. 144 value
  48. end
  49. end
  50. 8 module ResponseBodyMethods
  51. 8 attr_reader :content_digest_buffer
  52. 8 def initialize(response, options)
  53. 176 super
  54. 176 return unless response.headers.key?("content-digest")
  55. 128 should_validate = options.validate_content_digest
  56. 128 should_validate = should_validate.call(response) if should_validate.respond_to?(:call)
  57. 128 return unless should_validate
  58. 112 @content_digest_buffer = Response::Buffer.new(
  59. threshold_size: @options.body_threshold_size,
  60. bytesize: @length,
  61. encoding: @encoding
  62. )
  63. end
  64. 8 def write(chunk)
  65. 290 @content_digest_buffer.write(chunk) if @content_digest_buffer
  66. 290 super
  67. end
  68. 8 def close
  69. 112 if @content_digest_buffer
  70. 112 @content_digest_buffer.close
  71. 112 @content_digest_buffer = nil
  72. end
  73. 112 super
  74. end
  75. end
  76. 8 module InstanceMethods
  77. 8 def build_request(*)
  78. 224 request = super
  79. 224 return request if request.empty?
  80. 48 return request if request.headers.key?("content-digest")
  81. 48 perform_encoding = @options.encode_content_digest
  82. 48 perform_encoding = perform_encoding.call(request) if perform_encoding.respond_to?(:call)
  83. 48 return request unless perform_encoding
  84. 40 digest = base64digest(request.body)
  85. 40 request.headers.add("content-digest", "#{@options.content_digest_algorithm}=:#{digest}:")
  86. 40 request
  87. end
  88. 8 private
  89. 8 def fetch_response(request, _, _)
  90. 588 response = super
  91. 588 return response unless response.is_a?(Response)
  92. 176 perform_validation = @options.validate_content_digest
  93. 176 perform_validation = perform_validation.call(response) if perform_validation.respond_to?(:call)
  94. 176 validate_content_digest(response) if perform_validation
  95. 128 response
  96. rescue ValidationError => e
  97. 48 ErrorResponse.new(request, e)
  98. end
  99. 8 def validate_content_digest(response)
  100. 128 content_digest_header = response.headers["content-digest"]
  101. 128 raise MissingContentDigestError.new("response is missing a `content-digest` header", response) unless content_digest_header
  102. 112 digests = extract_content_digests(content_digest_header)
  103. 112 included_algorithms = SUPPORTED_ALGORITHMS.keys & digests.keys
  104. 112 raise MissingContentDigestError.new("unsupported algorithms: #{digests.keys.join(", ")}", response) if included_algorithms.empty?
  105. 112 content_buffer = response.body.content_digest_buffer
  106. 112 included_algorithms.each do |algorithm|
  107. 112 digest = SUPPORTED_ALGORITHMS.fetch(algorithm).new
  108. 112 digest_received = digests[algorithm]
  109. 14 digest_computed =
  110. 111 if content_buffer.respond_to?(:to_path)
  111. 16 content_buffer.flush
  112. 16 digest.file(content_buffer.to_path).base64digest
  113. else
  114. 96 digest.base64digest(content_buffer.to_s)
  115. end
  116. 4 raise InvalidContentDigestError.new("#{algorithm} digest does not match content",
  117. 111 response) unless digest_received == digest_computed
  118. end
  119. end
  120. 8 def extract_content_digests(header)
  121. 112 header.split(",").to_h do |entry|
  122. 128 algorithm, digest = entry.split("=", 2)
  123. 128 raise Error, "#{entry} is an invalid digest format" unless algorithm && digest
  124. 128 [algorithm, digest.byteslice(1..-2)]
  125. end
  126. end
  127. 8 def base64digest(body)
  128. 40 digest = SUPPORTED_ALGORITHMS.fetch(@options.content_digest_algorithm).new
  129. 40 if body.respond_to?(:read)
  130. 32 if body.respond_to?(:to_path)
  131. 8 digest.file(body.to_path).base64digest
  132. else
  133. 24 raise ContentDigestError, "request body must be rewindable" unless body.respond_to?(:rewind)
  134. 24 buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  135. 2 begin
  136. 24 IO.copy_stream(body, buffer)
  137. 24 buffer.flush
  138. 24 digest.file(buffer.to_path).base64digest
  139. ensure
  140. 24 body.rewind
  141. 24 buffer.close
  142. 24 buffer.unlink
  143. end
  144. end
  145. else
  146. 8 raise ContentDigestError, "base64digest for endless enumerators is not supported" if body.unbounded_body?
  147. 8 buffer = "".b
  148. 16 body.each { |chunk| buffer << chunk }
  149. 8 digest.base64digest(buffer)
  150. end
  151. end
  152. end
  153. end
  154. 8 register_plugin :content_digest, ContentDigest
  155. end
  156. end

lib/httpx/plugins/cookies.rb

100.0% lines covered

52 relevant lines. 52 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "forwardable"
  3. 8 module HTTPX
  4. 8 module Plugins
  5. #
  6. # This plugin implements a persistent cookie jar for the duration of a session.
  7. #
  8. # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Cookies
  11. #
  12. 8 module Cookies
  13. 8 def self.load_dependencies(*)
  14. 144 require "httpx/plugins/cookies/jar"
  15. 144 require "httpx/plugins/cookies/cookie"
  16. 144 require "httpx/plugins/cookies/set_cookie_parser"
  17. end
  18. 8 module InstanceMethods
  19. 8 extend Forwardable
  20. 8 def_delegator :@options, :cookies
  21. 8 def initialize(options = {}, &blk)
  22. 288 super({ cookies: Jar.new }.merge(options), &blk)
  23. end
  24. 8 def wrap
  25. 16 return super unless block_given?
  26. 16 super do |session|
  27. 16 old_cookies_jar = @options.cookies.dup
  28. 1 begin
  29. 16 yield session
  30. ensure
  31. 16 @options = @options.merge(cookies: old_cookies_jar)
  32. end
  33. end
  34. end
  35. 8 def build_request(*)
  36. 320 request = super
  37. 320 request.headers.set_cookie(request.options.cookies[request.uri])
  38. 320 request
  39. end
  40. 8 private
  41. 8 def set_request_callbacks(request)
  42. 320 super
  43. 320 request.on(:response) do |response|
  44. 320 next unless response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
  45. 64 log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
  46. 64 @options.cookies.parse(set_cookie)
  47. end
  48. end
  49. end
  50. 8 module HeadersMethods
  51. 8 def set_cookie(cookies)
  52. 320 return if cookies.empty?
  53. 272 header_value = cookies.sort.join("; ")
  54. 272 add("cookie", header_value)
  55. end
  56. end
  57. # adds support for the following options:
  58. #
  59. # :cookies :: cookie jar for the session (can be a Hash, an Array, an instance of HTTPX::Plugins::Cookies::CookieJar)
  60. 8 module OptionsMethods
  61. 8 def option_headers(*)
  62. 320 value = super
  63. 320 merge_cookie_in_jar(value.delete("cookie"), @cookies) if defined?(@cookies) && value.key?("cookie")
  64. 320 value
  65. end
  66. 8 def option_cookies(value)
  67. 480 jar = value.is_a?(Jar) ? value : Jar.new(value)
  68. 480 merge_cookie_in_jar(@headers.delete("cookie"), jar) if defined?(@headers) && @headers.key?("cookie")
  69. 480 jar
  70. end
  71. 8 private
  72. 8 def merge_cookie_in_jar(cookies, jar)
  73. 16 cookies.each do |ck|
  74. 16 ck.split(/ *; */).each do |cookie|
  75. 32 name, value = cookie.split("=", 2)
  76. 32 jar.add(Cookie.new(name, value))
  77. end
  78. end
  79. end
  80. end
  81. end
  82. 8 register_plugin :cookies, Cookies
  83. end
  84. end

lib/httpx/plugins/cookies/cookie.rb

100.0% lines covered

77 relevant lines. 77 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::Cookies
  4. # The HTTP Cookie.
  5. #
  6. # Contains the single cookie info: name, value and attributes.
  7. 8 class Cookie
  8. 8 include Comparable
  9. # Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
  10. # least)
  11. 8 MAX_LENGTH = 4096
  12. 8 attr_reader :domain, :path, :name, :value, :created_at
  13. 8 def path=(path)
  14. 184 path = String(path)
  15. 184 @path = path.start_with?("/") ? path : "/"
  16. end
  17. # See #domain.
  18. 8 def domain=(domain)
  19. 40 domain = String(domain)
  20. 40 if domain.start_with?(".")
  21. 16 @for_domain = true
  22. 16 domain = domain[1..-1]
  23. end
  24. 40 return if domain.empty?
  25. 40 @domain_name = DomainName.new(domain)
  26. # RFC 6265 5.3 5.
  27. 40 @for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
  28. 40 @domain = @domain_name.hostname
  29. end
  30. # Compares the cookie with another. When there are many cookies with
  31. # the same name for a URL, the value of the smallest must be used.
  32. 8 def <=>(other)
  33. # RFC 6265 5.4
  34. # Precedence: 1. longer path 2. older creation
  35. 703 (@name <=> other.name).nonzero? ||
  36. 60 (other.path.length <=> @path.length).nonzero? ||
  37. 35 (@created_at <=> other.created_at).nonzero? || 0
  38. end
  39. 8 class << self
  40. 8 def new(cookie, *args)
  41. 504 return cookie if cookie.is_a?(self)
  42. 504 super
  43. end
  44. # Tests if +target_path+ is under +base_path+ as described in RFC
  45. # 6265 5.1.4. +base_path+ must be an absolute path.
  46. # +target_path+ may be empty, in which case it is treated as the
  47. # root path.
  48. #
  49. # e.g.
  50. #
  51. # path_match?('/admin/', '/admin/index') == true
  52. # path_match?('/admin/', '/Admin/index') == false
  53. # path_match?('/admin/', '/admin/') == true
  54. # path_match?('/admin/', '/admin') == false
  55. #
  56. # path_match?('/admin', '/admin') == true
  57. # path_match?('/admin', '/Admin') == false
  58. # path_match?('/admin', '/admins') == false
  59. # path_match?('/admin', '/admin/') == true
  60. # path_match?('/admin', '/admin/index') == true
  61. 8 def path_match?(base_path, target_path)
  62. 1352 base_path.start_with?("/") || (return false)
  63. # RFC 6265 5.1.4
  64. 1352 bsize = base_path.size
  65. 1352 tsize = target_path.size
  66. 1352 return bsize == 1 if tsize.zero? # treat empty target_path as "/"
  67. 1352 return false unless target_path.start_with?(base_path)
  68. 1344 return true if bsize == tsize || base_path.end_with?("/")
  69. 16 target_path[bsize] == "/"
  70. end
  71. end
  72. 8 def initialize(arg, *attrs)
  73. 504 @created_at = Time.now
  74. 504 if attrs.empty?
  75. 24 attr_hash = Hash.try_convert(arg)
  76. else
  77. 480 @name = arg
  78. 480 @value, attr_hash = attrs
  79. 480 attr_hash = Hash.try_convert(attr_hash)
  80. end
  81. 33 attr_hash.each do |key, val|
  82. 312 key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
  83. 273 case key
  84. when :domain, :path
  85. 201 __send__(:"#{key}=", val)
  86. else
  87. 88 instance_variable_set(:"@#{key}", val)
  88. end
  89. 503 end if attr_hash
  90. 504 @path ||= "/"
  91. 504 raise ArgumentError, "name must be specified" if @name.nil?
  92. 504 @name = @name.to_s
  93. end
  94. 8 def expires
  95. 760 @expires || (@created_at && @max_age ? @created_at + @max_age : nil)
  96. end
  97. 8 def expired?(time = Time.now)
  98. 728 return false unless expires
  99. 32 expires <= time
  100. end
  101. # Returns a string for use in the Cookie header, i.e. `name=value`
  102. # or `name="value"`.
  103. 8 def cookie_value
  104. 483 "#{@name}=#{Scanner.quote(@value.to_s)}"
  105. end
  106. 8 alias_method :to_s, :cookie_value
  107. # Tests if it is OK to send this cookie to a given `uri`. A
  108. # RuntimeError is raised if the cookie's domain is unknown.
  109. 8 def valid_for_uri?(uri)
  110. 712 uri = URI(uri)
  111. # RFC 6265 5.4
  112. 712 return false if @secure && uri.scheme != "https"
  113. 704 acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
  114. end
  115. 8 private
  116. # Tests if it is OK to accept this cookie if it is sent from a given
  117. # URI/URL, `uri`.
  118. 8 def acceptable_from_uri?(uri)
  119. 736 uri = URI(uri)
  120. 736 host = DomainName.new(uri.host)
  121. # RFC 6265 5.3
  122. 736 if host.hostname == @domain
  123. 16 true
  124. 719 elsif @for_domain # !host-only-flag
  125. 32 host.cookie_domain?(@domain_name)
  126. else
  127. 688 @domain.nil?
  128. end
  129. end
  130. 8 module Scanner
  131. 8 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  132. 8 module_function
  133. 8 def quote(s)
  134. 552 return s unless s.match(RE_BAD_CHAR)
  135. 8 "\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
  136. end
  137. end
  138. end
  139. end
  140. end

lib/httpx/plugins/cookies/jar.rb

100.0% lines covered

46 relevant lines. 46 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins::Cookies
  4. # The Cookie Jar
  5. #
  6. # It holds a bunch of cookies.
  7. 8 class Jar
  8. 8 using URIExtensions
  9. 8 include Enumerable
  10. 8 def initialize_dup(orig)
  11. 216 super
  12. 216 @cookies = orig.instance_variable_get(:@cookies).dup
  13. end
  14. 8 def initialize(cookies = nil)
  15. 536 @cookies = []
  16. 120 cookies.each do |elem|
  17. 176 cookie = case elem
  18. when Cookie
  19. 16 elem
  20. when Array
  21. 144 Cookie.new(*elem)
  22. else
  23. 16 Cookie.new(elem)
  24. end
  25. 176 @cookies << cookie
  26. 535 end if cookies
  27. end
  28. 8 def parse(set_cookie)
  29. 144 SetCookieParser.call(set_cookie) do |name, value, attrs|
  30. 208 add(Cookie.new(name, value, attrs))
  31. end
  32. end
  33. 8 def add(cookie, path = nil)
  34. 456 c = cookie.dup
  35. 456 c.path = path if path && c.path == "/"
  36. # If the user agent receives a new cookie with the same cookie-name, domain-value, and path-value
  37. # as a cookie that it has already stored, the existing cookie is evicted and replaced with the new cookie.
  38. 864 @cookies.delete_if { |ck| ck.name == c.name && ck.domain == c.domain && ck.path == c.path }
  39. 456 @cookies << c
  40. end
  41. 8 def [](uri)
  42. 472 each(uri).sort
  43. end
  44. 8 def each(uri = nil, &blk)
  45. 1184 return enum_for(__method__, uri) unless blk
  46. 680 return @cookies.each(&blk) unless uri
  47. 472 now = Time.now
  48. 472 tpath = uri.path
  49. 472 @cookies.delete_if do |cookie|
  50. 728 if cookie.expired?(now)
  51. 16 true
  52. else
  53. 712 yield cookie if cookie.valid_for_uri?(uri) && Cookie.path_match?(cookie.path, tpath)
  54. 712 false
  55. end
  56. end
  57. end
  58. 8 def merge(other)
  59. 200 cookies_dup = dup
  60. 200 other.each do |elem|
  61. 216 cookie = case elem
  62. when Cookie
  63. 200 elem
  64. when Array
  65. 8 Cookie.new(*elem)
  66. else
  67. 8 Cookie.new(elem)
  68. end
  69. 216 cookies_dup.add(cookie)
  70. end
  71. 200 cookies_dup
  72. end
  73. end
  74. end
  75. end

lib/httpx/plugins/cookies/set_cookie_parser.rb

100.0% lines covered

72 relevant lines. 72 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "strscan"
  3. 8 require "time"
  4. 8 module HTTPX
  5. 8 module Plugins::Cookies
  6. 8 module SetCookieParser
  7. # Whitespace.
  8. 8 RE_WSP = /[ \t]+/.freeze
  9. # A pattern that matches a cookie name or attribute name which may
  10. # be empty, capturing trailing whitespace.
  11. 8 RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
  12. 8 RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
  13. # A pattern that matches the comma in a (typically date) value.
  14. 8 RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
  15. 8 module_function
  16. 8 def scan_dquoted(scanner)
  17. 16 s = +""
  18. 22 until scanner.eos?
  19. 64 break if scanner.skip(/"/)
  20. 48 if scanner.skip(/\\/)
  21. 16 s << scanner.getch
  22. 31 elsif scanner.scan(/[^"\\]+/)
  23. 32 s << scanner.matched
  24. end
  25. end
  26. 16 s
  27. end
  28. 8 def scan_value(scanner, comma_as_separator = false)
  29. 440 value = +""
  30. 498 until scanner.eos?
  31. 760 if scanner.scan(/[^,;"]+/)
  32. 432 value << scanner.matched
  33. 327 elsif scanner.skip(/"/)
  34. # RFC 6265 2.2
  35. # A cookie-value may be DQUOTE'd.
  36. 16 value << scan_dquoted(scanner)
  37. 311 elsif scanner.check(/;/)
  38. 232 break
  39. 79 elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
  40. 64 break
  41. else
  42. 16 value << scanner.getch
  43. end
  44. end
  45. 440 value.rstrip!
  46. 440 value
  47. end
  48. 8 def scan_name_value(scanner, comma_as_separator = false)
  49. 440 name = scanner.scan(RE_NAME)
  50. 440 name.rstrip! if name
  51. 440 if scanner.skip(/=/)
  52. 432 value = scan_value(scanner, comma_as_separator)
  53. else
  54. 8 scan_value(scanner, comma_as_separator)
  55. 8 value = nil
  56. end
  57. 440 [name, value]
  58. end
  59. 8 def call(set_cookie)
  60. 144 scanner = StringScanner.new(set_cookie)
  61. # RFC 6265 4.1.1 & 5.2
  62. 170 until scanner.eos?
  63. 208 start = scanner.pos
  64. 208 len = nil
  65. 208 scanner.skip(RE_WSP)
  66. 208 name, value = scan_name_value(scanner, true)
  67. 208 value = nil if name && name.empty?
  68. 208 attrs = {}
  69. 237 until scanner.eos?
  70. 296 if scanner.skip(/,/)
  71. # The comma is used as separator for concatenating multiple
  72. # values of a header.
  73. 64 len = (scanner.pos - 1) - start
  74. 64 break
  75. 231 elsif scanner.skip(/;/)
  76. 232 scanner.skip(RE_WSP)
  77. 232 aname, avalue = scan_name_value(scanner, true)
  78. 232 next if (aname.nil? || aname.empty?) || value.nil?
  79. 232 aname.downcase!
  80. 203 case aname
  81. when "expires"
  82. 16 next unless avalue
  83. # RFC 6265 5.2.1
  84. 16 (avalue = Time.parse(avalue)) || next
  85. when "max-age"
  86. 8 next unless avalue
  87. # RFC 6265 5.2.2
  88. 8 next unless /\A-?\d+\z/.match?(avalue)
  89. 8 avalue = Integer(avalue)
  90. when "domain"
  91. # RFC 6265 5.2.3
  92. # An empty value SHOULD be ignored.
  93. 24 next if avalue.nil? || avalue.empty?
  94. when "path"
  95. # RFC 6265 5.2.4
  96. # A relative path must be ignored rather than normalizing it
  97. # to "/".
  98. 176 next unless avalue && avalue.start_with?("/")
  99. when "secure", "httponly"
  100. # RFC 6265 5.2.5, 5.2.6
  101. 7 avalue = true
  102. end
  103. 203 attrs[aname] = avalue
  104. end
  105. end
  106. 208 len ||= scanner.pos - start
  107. 208 next if len > Cookie::MAX_LENGTH
  108. 208 yield(name, value, attrs) if name && !name.empty? && value
  109. end
  110. end
  111. end
  112. end
  113. end

lib/httpx/plugins/digest_auth.rb

100.0% lines covered

29 relevant lines. 29 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds helper methods to implement HTTP Digest Auth (https://datatracker.ietf.org/doc/html/rfc7616)
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Auth#digest-auth
  8. #
  9. 8 module DigestAuth
  10. 8 DigestError = Class.new(Error)
  11. 8 class << self
  12. 8 def extra_options(options)
  13. 160 options.merge(max_concurrent_requests: 1)
  14. end
  15. 8 def load_dependencies(*)
  16. 160 require_relative "auth/digest"
  17. end
  18. end
  19. # adds support for the following options:
  20. #
  21. # :digest :: instance of HTTPX::Plugins::Authentication::Digest, used to authenticate requests in the session.
  22. 8 module OptionsMethods
  23. 8 def option_digest(value)
  24. 320 raise TypeError, ":digest must be a #{Authentication::Digest}" unless value.is_a?(Authentication::Digest)
  25. 320 value
  26. end
  27. end
  28. 8 module InstanceMethods
  29. 8 def digest_auth(user, password, hashed: false)
  30. 160 with(digest: Authentication::Digest.new(user, password, hashed: hashed))
  31. end
  32. 8 private
  33. 8 def send_requests(*requests)
  34. 192 requests.flat_map do |request|
  35. 192 digest = request.options.digest
  36. 192 next super(request) unless digest
  37. 320 probe_response = wrap { super(request).first }
  38. 160 return probe_response unless probe_response.is_a?(Response)
  39. 160 if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
  40. 144 request.transition(:idle)
  41. 126 request.headers["authorization"] = digest.authenticate(request, probe_response.headers["www-authenticate"])
  42. 144 super(request)
  43. else
  44. 16 probe_response
  45. end
  46. end
  47. end
  48. end
  49. end
  50. 8 register_plugin :digest_auth, DigestAuth
  51. end
  52. end

lib/httpx/plugins/expect.rb

100.0% lines covered

56 relevant lines. 56 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin makes all HTTP/1.1 requests with a body send the "Expect: 100-continue".
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Expect#expect
  8. #
  9. 8 module Expect
  10. 8 EXPECT_TIMEOUT = 2
  11. 8 class << self
  12. 8 def no_expect_store
  13. 178 @no_expect_store ||= []
  14. end
  15. 8 def extra_options(options)
  16. 216 options.merge(expect_timeout: EXPECT_TIMEOUT)
  17. end
  18. end
  19. # adds support for the following options:
  20. #
  21. # :expect_timeout :: time (in seconds) to wait for a 100-expect response,
  22. # before retrying without the Expect header (defaults to <tt>2</tt>).
  23. # :expect_threshold_size :: min threshold (in bytes) of the request payload to enable the 100-continue negotiation on.
  24. 8 module OptionsMethods
  25. 8 def option_expect_timeout(value)
  26. 384 seconds = Float(value)
  27. 384 raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
  28. 384 seconds
  29. end
  30. 8 def option_expect_threshold_size(value)
  31. 16 bytes = Integer(value)
  32. 16 raise TypeError, ":expect_threshold_size must be positive" unless bytes.positive?
  33. 16 bytes
  34. end
  35. end
  36. 8 module RequestMethods
  37. 8 def initialize(*)
  38. 248 super
  39. 248 return if @body.empty?
  40. 168 threshold = @options.expect_threshold_size
  41. 168 return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
  42. 152 return if Expect.no_expect_store.include?(origin)
  43. 126 @headers["expect"] = "100-continue"
  44. end
  45. 8 def response=(response)
  46. 184 if response.is_a?(Response) &&
  47. response.status == 100 &&
  48. !@headers.key?("expect") &&
  49. 3 (@state == :body || @state == :done)
  50. # if we're past this point, this means that we just received a 100-Continue response,
  51. # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
  52. #
  53. # this means that expect was deactivated for this request too soon, i.e. response took longer.
  54. #
  55. # so we have to reactivate it again.
  56. 8 @headers["expect"] = "100-continue"
  57. 9 @informational_status = 100
  58. 9 Expect.no_expect_store.delete(origin)
  59. end
  60. 184 super
  61. end
  62. end
  63. 8 module ConnectionMethods
  64. 8 def send_request_to_parser(request)
  65. 112 super
  66. 112 return unless request.headers["expect"] == "100-continue"
  67. 80 expect_timeout = request.options.expect_timeout
  68. 80 return if expect_timeout.nil? || expect_timeout.infinite?
  69. 80 set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
  70. # expect timeout expired
  71. 17 if request.state == :expect && !request.expects?
  72. 17 Expect.no_expect_store << request.origin
  73. 17 request.headers.delete("expect")
  74. 17 consume
  75. end
  76. end
  77. end
  78. end
  79. 8 module InstanceMethods
  80. 8 def fetch_response(request, selector, options)
  81. 430 response = super
  82. 430 return unless response
  83. 112 if response.is_a?(Response) && response.status == 417 && request.headers.key?("expect")
  84. 16 response.close
  85. 16 request.headers.delete("expect")
  86. 16 request.transition(:idle)
  87. 16 send_request(request, selector, options)
  88. 14 return
  89. end
  90. 96 response
  91. end
  92. end
  93. end
  94. 8 register_plugin :expect, Expect
  95. end
  96. end

lib/httpx/plugins/follow_redirects.rb

100.0% lines covered

108 relevant lines. 108 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 15 module HTTPX
  3. 15 InsecureRedirectError = Class.new(Error)
  4. 15 module Plugins
  5. #
  6. # This plugin adds support for automatically following redirect (status 30X) responses.
  7. #
  8. # It has a default upper bound of followed redirects (see *MAX_REDIRECTS* and the *max_redirects* option),
  9. # after which it will return the last redirect response. It will **not** raise an exception.
  10. #
  11. # It doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
  12. #
  13. # It doesn't propagate authorization related headers to requests redirecting to different origins
  14. # (see *allow_auth_to_other_origins*) to override.
  15. #
  16. # It allows customization of when to redirect via the *redirect_on* callback option).
  17. #
  18. # https://gitlab.com/os85/httpx/wikis/Follow-Redirects
  19. #
  20. 15 module FollowRedirects
  21. 15 MAX_REDIRECTS = 3
  22. 15 REDIRECT_STATUS = (300..399).freeze
  23. 15 REQUEST_BODY_HEADERS = %w[transfer-encoding content-encoding content-type content-length content-language content-md5 trailer].freeze
  24. 15 using URIExtensions
  25. # adds support for the following options:
  26. #
  27. # :max_redirects :: max number of times a request will be redirected (defaults to <tt>3</tt>).
  28. # :follow_insecure_redirects :: whether redirects to an "http://" URI, when coming from an "https//", are allowed
  29. # (defaults to <tt>false</tt>).
  30. # :allow_auth_to_other_origins :: whether auth-related headers, such as "Authorization", are propagated on redirection
  31. # (defaults to <tt>false</tt>).
  32. # :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
  33. 15 module OptionsMethods
  34. 15 def option_max_redirects(value)
  35. 462 num = Integer(value)
  36. 462 raise TypeError, ":max_redirects must be positive" if num.negative?
  37. 462 num
  38. end
  39. 15 def option_follow_insecure_redirects(value)
  40. 24 value
  41. end
  42. 15 def option_allow_auth_to_other_origins(value)
  43. 24 value
  44. end
  45. 15 def option_redirect_on(value)
  46. 48 raise TypeError, ":redirect_on must be callable" unless value.respond_to?(:call)
  47. 48 value
  48. end
  49. end
  50. 15 module InstanceMethods
  51. # returns a session with the *max_redirects* option set to +n+
  52. 15 def max_redirects(n)
  53. 48 with(max_redirects: n.to_i)
  54. end
  55. 15 private
  56. 15 def fetch_response(request, selector, options)
  57. 4242768 redirect_request = request.redirect_request
  58. 4242768 response = super(redirect_request, selector, options)
  59. 4242768 return unless response
  60. 568 max_redirects = redirect_request.max_redirects
  61. 568 return response unless response.is_a?(Response)
  62. 552 return response unless REDIRECT_STATUS.include?(response.status) && response.headers.key?("location")
  63. 361 return response unless max_redirects.positive?
  64. 329 redirect_uri = __get_location_from_response(response)
  65. 329 if options.redirect_on
  66. 32 redirect_allowed = options.redirect_on.call(redirect_uri)
  67. 32 return response unless redirect_allowed
  68. end
  69. # build redirect request
  70. 313 request_body = redirect_request.body
  71. 313 redirect_method = "GET"
  72. 313 redirect_params = {}
  73. 313 if response.status == 305 && options.respond_to?(:proxy)
  74. 8 request_body.rewind
  75. # The requested resource MUST be accessed through the proxy given by
  76. # the Location field. The Location field gives the URI of the proxy.
  77. 8 redirect_options = options.merge(headers: redirect_request.headers,
  78. proxy: { uri: redirect_uri },
  79. max_redirects: max_redirects - 1)
  80. 7 redirect_params[:body] = request_body
  81. 8 redirect_uri = redirect_request.uri
  82. 8 options = redirect_options
  83. else
  84. 305 redirect_headers = redirect_request_headers(redirect_request.uri, redirect_uri, request.headers, options)
  85. 305 redirect_opts = Hash[options]
  86. 268 redirect_params[:max_redirects] = max_redirects - 1
  87. 305 unless request_body.empty?
  88. 24 if response.status == 307
  89. # The method and the body of the original request are reused to perform the redirected request.
  90. 8 redirect_method = redirect_request.verb
  91. 8 request_body.rewind
  92. 7 redirect_params[:body] = request_body
  93. else
  94. # redirects are **ALWAYS** GET, so remove body-related headers
  95. 16 REQUEST_BODY_HEADERS.each do |h|
  96. 112 redirect_headers.delete(h)
  97. end
  98. 14 redirect_params[:body] = nil
  99. end
  100. end
  101. 305 options = options.class.new(redirect_opts.merge(headers: redirect_headers.to_h))
  102. end
  103. 313 redirect_uri = Utils.to_uri(redirect_uri)
  104. 313 if !options.follow_insecure_redirects &&
  105. response.uri.scheme == "https" &&
  106. redirect_uri.scheme == "http"
  107. 8 error = InsecureRedirectError.new(redirect_uri.to_s)
  108. 8 error.set_backtrace(caller)
  109. 7 return ErrorResponse.new(request, error)
  110. end
  111. 305 retry_request = build_request(redirect_method, redirect_uri, redirect_params, options)
  112. 305 request.redirect_request = retry_request
  113. 305 redirect_after = response.headers["retry-after"]
  114. 305 if redirect_after
  115. # Servers send the "Retry-After" header field to indicate how long the
  116. # user agent ought to wait before making a follow-up request.
  117. # When sent with any 3xx (Redirection) response, Retry-After indicates
  118. # the minimum time that the user agent is asked to wait before issuing
  119. # the redirected request.
  120. #
  121. 31 redirect_after = Utils.parse_retry_after(redirect_after)
  122. 31 retry_start = Utils.now
  123. 31 log { "redirecting after #{redirect_after} secs..." }
  124. 31 selector.after(redirect_after) do
  125. 31 if (response = request.response)
  126. 15 response.finish!
  127. 15 retry_request.response = response
  128. # request has terminated abruptly meanwhile
  129. 15 retry_request.emit(:response, response)
  130. else
  131. 16 log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
  132. 16 send_request(retry_request, selector, options)
  133. end
  134. end
  135. else
  136. 274 send_request(retry_request, selector, options)
  137. end
  138. 116 nil
  139. end
  140. # :nodoc:
  141. 15 def redirect_request_headers(original_uri, redirect_uri, headers, options)
  142. 305 headers = headers.dup
  143. 305 return headers if options.allow_auth_to_other_origins
  144. 297 return headers unless headers.key?("authorization")
  145. 8 return headers if original_uri.origin == redirect_uri.origin
  146. 8 headers.delete("authorization")
  147. 8 headers
  148. end
  149. # :nodoc:
  150. 15 def __get_location_from_response(response)
  151. # @type var location_uri: http_uri
  152. 329 location_uri = URI(response.headers["location"])
  153. 329 location_uri = response.uri.merge(location_uri) if location_uri.relative?
  154. 329 location_uri
  155. end
  156. end
  157. 15 module RequestMethods
  158. # returns the top-most original HTTPX::Request from the redirect chain
  159. 15 attr_accessor :root_request
  160. # returns the follow-up redirect request, or itself
  161. 15 def redirect_request
  162. 4242768 @redirect_request || self
  163. end
  164. # sets the follow-up redirect request
  165. 15 def redirect_request=(req)
  166. 305 @redirect_request = req
  167. 305 req.root_request = @root_request || self
  168. 305 @response = nil
  169. end
  170. 15 def response
  171. 4244658 return super unless @redirect_request && @response.nil?
  172. 73 @redirect_request.response
  173. end
  174. 15 def max_redirects
  175. 568 @options.max_redirects || MAX_REDIRECTS
  176. end
  177. end
  178. 15 module ConnectionMethods
  179. 15 private
  180. 15 def set_request_request_timeout(request)
  181. 541 return unless request.root_request.nil?
  182. 257 super
  183. end
  184. end
  185. end
  186. 15 register_plugin :follow_redirects, FollowRedirects
  187. end
  188. end

lib/httpx/plugins/grpc.rb

100.0% lines covered

133 relevant lines. 133 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 GRPCError = Class.new(Error) do
  4. 6 attr_reader :status, :details, :metadata
  5. 6 def initialize(status, details, metadata)
  6. 24 @status = status
  7. 24 @details = details
  8. 24 @metadata = metadata
  9. 24 super("GRPC error, code=#{status}, details=#{details}, metadata=#{metadata}")
  10. end
  11. end
  12. 6 module Plugins
  13. #
  14. # This plugin adds DSL to build GRPC interfaces.
  15. #
  16. # https://gitlab.com/os85/httpx/wikis/GRPC
  17. #
  18. 6 module GRPC
  19. 6 unless String.method_defined?(:underscore)
  20. 6 module StringExtensions
  21. 6 refine String do
  22. 6 def underscore
  23. 312 s = dup # Avoid mutating the argument, as it might be frozen.
  24. 312 s.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
  25. 312 s.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
  26. 312 s.tr!("-", "_")
  27. 312 s.downcase!
  28. 312 s
  29. end
  30. end
  31. end
  32. 6 using StringExtensions
  33. end
  34. 6 DEADLINE = 60
  35. 6 MARSHAL_METHOD = :encode
  36. 6 UNMARSHAL_METHOD = :decode
  37. 6 HEADERS = {
  38. "content-type" => "application/grpc",
  39. "te" => "trailers",
  40. "accept" => "application/grpc",
  41. # metadata fits here
  42. # ex "foo-bin" => base64("bar")
  43. }.freeze
  44. 6 class << self
  45. 6 def load_dependencies(*)
  46. 138 require "stringio"
  47. 138 require "httpx/plugins/grpc/grpc_encoding"
  48. 138 require "httpx/plugins/grpc/message"
  49. 138 require "httpx/plugins/grpc/call"
  50. end
  51. 6 def configure(klass)
  52. 138 klass.plugin(:persistent)
  53. 138 klass.plugin(:stream)
  54. end
  55. 6 def extra_options(options)
  56. 138 options.merge(
  57. fallback_protocol: "h2",
  58. grpc_rpcs: {}.freeze,
  59. grpc_compression: false,
  60. grpc_deadline: DEADLINE
  61. )
  62. end
  63. end
  64. 6 module OptionsMethods
  65. 6 def option_grpc_service(value)
  66. 120 String(value)
  67. end
  68. 6 def option_grpc_compression(value)
  69. 162 case value
  70. when true, false
  71. 138 value
  72. else
  73. 24 value.to_s
  74. end
  75. end
  76. 6 def option_grpc_rpcs(value)
  77. 1116 Hash[value]
  78. end
  79. 6 def option_grpc_deadline(value)
  80. 804 raise TypeError, ":grpc_deadline must be positive" unless value.positive?
  81. 804 value
  82. end
  83. 6 def option_call_credentials(value)
  84. 18 raise TypeError, ":call_credentials must respond to #call" unless value.respond_to?(:call)
  85. 18 value
  86. end
  87. end
  88. 6 module ResponseMethods
  89. 6 attr_reader :trailing_metadata
  90. 6 def merge_headers(trailers)
  91. 114 @trailing_metadata = Hash[trailers]
  92. 114 super
  93. end
  94. end
  95. 6 module RequestBodyMethods
  96. 6 def initialize(*, **)
  97. 126 super
  98. 126 if (compression = @headers["grpc-encoding"])
  99. 12 deflater_body = self.class.initialize_deflater_body(@body, compression)
  100. 12 @body = Transcoder::GRPCEncoding.encode(deflater_body || @body, compressed: !deflater_body.nil?)
  101. else
  102. 114 @body = Transcoder::GRPCEncoding.encode(@body, compressed: false)
  103. end
  104. end
  105. end
  106. 6 module InstanceMethods
  107. 6 def with_channel_credentials(ca_path, key = nil, cert = nil, **ssl_opts)
  108. # @type var ssl_params: ::Hash[::Symbol, untyped]
  109. 72 ssl_params = {
  110. **ssl_opts,
  111. ca_file: ca_path,
  112. }
  113. 72 if key
  114. 72 key = File.read(key) if File.file?(key)
  115. 72 ssl_params[:key] = OpenSSL::PKey.read(key)
  116. end
  117. 72 if cert
  118. 72 cert = File.read(cert) if File.file?(cert)
  119. 72 ssl_params[:cert] = OpenSSL::X509::Certificate.new(cert)
  120. end
  121. 72 with(ssl: ssl_params)
  122. end
  123. 6 def rpc(rpc_name, input, output, **opts)
  124. 312 rpc_name = rpc_name.to_s
  125. 312 raise Error, "rpc #{rpc_name} already defined" if @options.grpc_rpcs.key?(rpc_name)
  126. rpc_opts = {
  127. 312 deadline: @options.grpc_deadline,
  128. }.merge(opts)
  129. 312 local_rpc_name = rpc_name.underscore
  130. 312 session_class = Class.new(self.class) do
  131. # define rpc method with ruby style name
  132. 312 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  133. def #{local_rpc_name}(input, **opts) # def grpc_action(input, **opts)
  134. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  135. end # end
  136. OUT
  137. # define rpc method with original name
  138. 312 unless local_rpc_name == rpc_name
  139. 12 class_eval(<<-OUT, __FILE__, __LINE__ + 1)
  140. def #{rpc_name}(input, **opts) # def grpcAction(input, **opts)
  141. rpc_execute("#{local_rpc_name}", input, **opts) # rpc_execute("grpc_action", input, **opts)
  142. end # end
  143. OUT
  144. end
  145. end
  146. 312 session_class.new(@options.merge(
  147. grpc_rpcs: @options.grpc_rpcs.merge(
  148. local_rpc_name => [rpc_name, input, output, rpc_opts]
  149. ).freeze
  150. ))
  151. end
  152. 6 def build_stub(origin, service: nil, compression: false)
  153. 138 scheme = @options.ssl.empty? ? "http" : "https"
  154. 138 origin = URI.parse("#{scheme}://#{origin}")
  155. 138 session = self
  156. 138 if service && service.respond_to?(:rpc_descs)
  157. # it's a grpc generic service
  158. 60 service.rpc_descs.each do |rpc_name, rpc_desc|
  159. rpc_opts = {
  160. 300 marshal_method: rpc_desc.marshal_method,
  161. unmarshal_method: rpc_desc.unmarshal_method,
  162. }
  163. 300 input = rpc_desc.input
  164. 300 input = input.type if input.respond_to?(:type)
  165. 300 output = rpc_desc.output
  166. 300 if output.respond_to?(:type)
  167. 120 rpc_opts[:stream] = true
  168. 120 output = output.type
  169. end
  170. 300 session = session.rpc(rpc_name, input, output, **rpc_opts)
  171. end
  172. 60 service = service.service_name
  173. end
  174. 138 session.with(origin: origin, grpc_service: service, grpc_compression: compression)
  175. end
  176. 6 def execute(rpc_method, input,
  177. deadline: DEADLINE,
  178. metadata: nil,
  179. **opts)
  180. 126 grpc_request = build_grpc_request(rpc_method, input, deadline: deadline, metadata: metadata, **opts)
  181. 126 response = request(grpc_request, **opts)
  182. 126 response.raise_for_status unless opts[:stream]
  183. 114 GRPC::Call.new(response)
  184. end
  185. 6 private
  186. 6 def rpc_execute(rpc_name, input, **opts)
  187. 60 rpc_name, input_enc, output_enc, rpc_opts = @options.grpc_rpcs[rpc_name]
  188. 60 exec_opts = rpc_opts.merge(opts)
  189. 60 marshal_method ||= exec_opts.delete(:marshal_method) || MARSHAL_METHOD
  190. 60 unmarshal_method ||= exec_opts.delete(:unmarshal_method) || UNMARSHAL_METHOD
  191. 60 messages = if input.respond_to?(:each)
  192. 24 Enumerator.new do |y|
  193. 24 input.each do |message|
  194. 48 y << input_enc.__send__(marshal_method, message)
  195. end
  196. end
  197. else
  198. 36 input_enc.__send__(marshal_method, input)
  199. end
  200. 60 call = execute(rpc_name, messages, **exec_opts)
  201. 60 call.decoder = output_enc.method(unmarshal_method)
  202. 60 call
  203. end
  204. 6 def build_grpc_request(rpc_method, input, deadline:, metadata: nil, **)
  205. 126 uri = @options.origin.dup
  206. 126 rpc_method = "/#{rpc_method}" unless rpc_method.start_with?("/")
  207. 126 rpc_method = "/#{@options.grpc_service}#{rpc_method}" if @options.grpc_service
  208. 126 uri.path = rpc_method
  209. 126 headers = HEADERS.merge(
  210. "grpc-accept-encoding" => ["identity", *@options.supported_compression_formats]
  211. )
  212. 126 unless deadline == Float::INFINITY
  213. # convert to milliseconds
  214. 126 deadline = (deadline * 1000.0).to_i
  215. 126 headers["grpc-timeout"] = "#{deadline}m"
  216. end
  217. 126 headers = headers.merge(metadata.transform_keys(&:to_s)) if metadata
  218. # prepare compressor
  219. 126 compression = @options.grpc_compression == true ? "gzip" : @options.grpc_compression
  220. 126 headers["grpc-encoding"] = compression if compression
  221. 126 headers.merge!(@options.call_credentials.call.transform_keys(&:to_s)) if @options.call_credentials
  222. 126 build_request("POST", uri, headers: headers, body: input)
  223. end
  224. end
  225. end
  226. 6 register_plugin :grpc, GRPC
  227. end
  228. end

lib/httpx/plugins/grpc/call.rb

90.91% lines covered

33 relevant lines. 30 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module GRPC
  5. # Encapsulates call information
  6. 6 class Call
  7. 6 attr_writer :decoder
  8. 6 def initialize(response)
  9. 114 @response = response
  10. 156 @decoder = ->(z) { z }
  11. 114 @consumed = false
  12. 114 @grpc_response = nil
  13. end
  14. 6 def inspect
  15. "#{self.class}(#{grpc_response})"
  16. end
  17. 6 def to_s
  18. 66 grpc_response.to_s
  19. end
  20. 6 def metadata
  21. response.headers
  22. end
  23. 6 def trailing_metadata
  24. 72 return unless @consumed
  25. 48 @response.trailing_metadata
  26. end
  27. 6 private
  28. 6 def grpc_response
  29. 186 @grpc_response ||= if @response.respond_to?(:each)
  30. 24 Enumerator.new do |y|
  31. 24 Message.stream(@response).each do |message|
  32. 48 y << @decoder.call(message)
  33. end
  34. 24 @consumed = true
  35. end
  36. else
  37. 90 @consumed = true
  38. 90 @decoder.call(Message.unary(@response))
  39. end
  40. end
  41. 6 def respond_to_missing?(meth, *args, &blk)
  42. 24 grpc_response.respond_to?(meth, *args) || super
  43. end
  44. 6 def method_missing(meth, *args, &blk)
  45. 48 return grpc_response.__send__(meth, *args, &blk) if grpc_response.respond_to?(meth)
  46. super
  47. end
  48. end
  49. end
  50. end
  51. end

lib/httpx/plugins/grpc/grpc_encoding.rb

97.87% lines covered

47 relevant lines. 46 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Transcoder
  4. 6 module GRPCEncoding
  5. 6 class Deflater
  6. 6 extend Forwardable
  7. 6 attr_reader :content_type
  8. 6 def initialize(body, compressed:)
  9. 126 @content_type = body.content_type
  10. 126 @body = BodyReader.new(body)
  11. 126 @compressed = compressed
  12. end
  13. 6 def bytesize
  14. 450 return @body.bytesize if @body.respond_to?(:bytesize)
  15. Float::INFINITY
  16. end
  17. 6 def read(length = nil, outbuf = nil)
  18. 276 buf = @body.read(length, outbuf)
  19. 252 return unless buf
  20. 138 compressed_flag = @compressed ? 1 : 0
  21. 138 buf = outbuf if outbuf
  22. 138 buf = buf.b if buf.frozen?
  23. 138 buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
  24. 138 buf
  25. end
  26. end
  27. 6 class Inflater
  28. 6 def initialize(response)
  29. 90 @response = response
  30. 90 @grpc_encodings = nil
  31. end
  32. 6 def call(message, &blk)
  33. 114 data = "".b
  34. 114 until message.empty?
  35. 114 compressed, size = message.unpack("CL>")
  36. 114 encoded_data = message.byteslice(5..size + 5 - 1)
  37. 114 if compressed == 1
  38. 12 grpc_encodings.reverse_each do |encoding|
  39. 12 decoder = @response.body.class.initialize_inflater_by_encoding(encoding, @response, bytesize: encoded_data.bytesize)
  40. 12 encoded_data = decoder.call(encoded_data)
  41. 12 blk.call(encoded_data) if blk
  42. 12 data << encoded_data
  43. end
  44. else
  45. 102 blk.call(encoded_data) if blk
  46. 102 data << encoded_data
  47. end
  48. 114 message = message.byteslice((size + 5)..-1)
  49. end
  50. 114 data
  51. end
  52. 6 private
  53. 6 def grpc_encodings
  54. 12 @grpc_encodings ||= @response.headers.get("grpc-encoding")
  55. end
  56. end
  57. 6 def self.encode(*args, **kwargs)
  58. 126 Deflater.new(*args, **kwargs)
  59. end
  60. 6 def self.decode(response)
  61. 90 Inflater.new(response)
  62. end
  63. end
  64. end
  65. end

lib/httpx/plugins/grpc/message.rb

95.83% lines covered

24 relevant lines. 23 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 module HTTPX
  3. 6 module Plugins
  4. 6 module GRPC
  5. # Encoding module for GRPC responses
  6. #
  7. # Can encode and decode grpc messages.
  8. 6 module Message
  9. 6 module_function
  10. # decodes a unary grpc response
  11. 6 def unary(response)
  12. 90 verify_status(response)
  13. 66 decoder = Transcoder::GRPCEncoding.decode(response)
  14. 66 decoder.call(response.to_s)
  15. end
  16. # lazy decodes a grpc stream response
  17. 6 def stream(response, &block)
  18. 48 return enum_for(__method__, response) unless block
  19. 24 decoder = Transcoder::GRPCEncoding.decode(response)
  20. 24 response.each do |frame|
  21. 48 decoder.call(frame, &block)
  22. end
  23. 24 verify_status(response)
  24. end
  25. 6 def cancel(request)
  26. request.emit(:refuse, :client_cancellation)
  27. end
  28. # interprets the grpc call trailing metadata, and raises an
  29. # exception in case of error code
  30. 6 def verify_status(response)
  31. # return standard errors if need be
  32. 114 response.raise_for_status
  33. 114 status = Integer(response.headers["grpc-status"])
  34. 114 message = response.headers["grpc-message"]
  35. 114 return if status.zero?
  36. 24 response.close
  37. 24 raise GRPCError.new(status, message, response.trailing_metadata)
  38. end
  39. end
  40. end
  41. end
  42. end

lib/httpx/plugins/h2c.rb

94.92% lines covered

59 relevant lines. 56 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
  6. # (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
  9. #
  10. 8 module H2C
  11. 8 VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
  12. 8 class << self
  13. 8 def load_dependencies(klass)
  14. 16 klass.plugin(:upgrade)
  15. end
  16. 8 def call(connection, request, response)
  17. 16 connection.upgrade_to_h2c(request, response)
  18. end
  19. 8 def extra_options(options)
  20. 16 options.merge(max_concurrent_requests: 1, upgrade_handlers: options.upgrade_handlers.merge("h2c" => self))
  21. end
  22. end
  23. 8 class H2CParser < Connection::HTTP2
  24. 8 def upgrade(request, response)
  25. # skip checks, it is assumed that this is the first
  26. # request in the connection
  27. 16 stream = @connection.upgrade
  28. # on_settings
  29. 16 handle_stream(stream, request)
  30. 14 @streams[request] = stream
  31. # clean up data left behind in the buffer, if the server started
  32. # sending frames
  33. 16 data = response.read
  34. 16 @connection << data
  35. end
  36. end
  37. 8 module RequestMethods
  38. 8 def valid_h2c_verb?
  39. 16 VALID_H2C_VERBS.include?(@verb)
  40. end
  41. end
  42. 8 module ConnectionMethods
  43. 8 using URIExtensions
  44. 8 def initialize(*)
  45. 16 super
  46. 16 @h2c_handshake = false
  47. end
  48. 8 def send(request)
  49. 56 return super if @h2c_handshake
  50. 16 return super unless request.valid_h2c_verb? && request.scheme == "http"
  51. 16 return super if @upgrade_protocol == "h2c"
  52. 16 @h2c_handshake = true
  53. # build upgrade request
  54. 16 request.headers.add("connection", "upgrade")
  55. 16 request.headers.add("connection", "http2-settings")
  56. 14 request.headers["upgrade"] = "h2c"
  57. 14 request.headers["http2-settings"] = ::HTTP2::Client.settings_header(request.options.http2_settings)
  58. 16 super
  59. end
  60. 8 def upgrade_to_h2c(request, response)
  61. 16 prev_parser = @parser
  62. 16 if prev_parser
  63. 16 prev_parser.reset
  64. 14 @inflight -= prev_parser.requests.size
  65. end
  66. 16 @parser = H2CParser.new(@write_buffer, @options)
  67. 16 set_parser_callbacks(@parser)
  68. 14 @inflight += 1
  69. 16 @parser.upgrade(request, response)
  70. 16 @upgrade_protocol = "h2c"
  71. 16 prev_parser.requests.each do |req|
  72. 16 req.transition(:idle)
  73. 16 send(req)
  74. end
  75. end
  76. 8 private
  77. 8 def send_request_to_parser(request)
  78. 56 super
  79. 56 return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
  80. 16 max_concurrent_requests = parser.max_concurrent_requests
  81. 16 return if max_concurrent_requests == 1
  82. parser.max_concurrent_requests = 1
  83. request.once(:response) do
  84. parser.max_concurrent_requests = max_concurrent_requests
  85. end
  86. end
  87. end
  88. end
  89. 8 register_plugin(:h2c, H2C)
  90. end
  91. end

lib/httpx/plugins/ntlm_auth.rb

100.0% lines covered

30 relevant lines. 30 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 3 module HTTPX
  3. 3 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/Auth#ntlm-auth
  6. #
  7. 3 module NTLMAuth
  8. 3 class << self
  9. 3 def load_dependencies(_klass)
  10. 2 require_relative "auth/ntlm"
  11. end
  12. 3 def extra_options(options)
  13. 2 options.merge(max_concurrent_requests: 1)
  14. end
  15. end
  16. 3 module OptionsMethods
  17. 3 def option_ntlm(value)
  18. 8 raise TypeError, ":ntlm must be a #{Authentication::Ntlm}" unless value.is_a?(Authentication::Ntlm)
  19. 8 value
  20. end
  21. end
  22. 3 module InstanceMethods
  23. 3 def ntlm_auth(user, password, domain = nil)
  24. 4 with(ntlm: Authentication::Ntlm.new(user, password, domain: domain))
  25. end
  26. 3 private
  27. 3 def send_requests(*requests)
  28. 8 requests.flat_map do |request|
  29. 8 ntlm = request.options.ntlm
  30. 8 if ntlm
  31. 4 request.headers["authorization"] = ntlm.negotiate
  32. 8 probe_response = wrap { super(request).first }
  33. 4 return probe_response unless probe_response.is_a?(Response)
  34. 4 if probe_response.status == 401 && ntlm.can_authenticate?(probe_response.headers["www-authenticate"])
  35. 2 request.transition(:idle)
  36. 2 request.headers["authorization"] = ntlm.authenticate(request, probe_response.headers["www-authenticate"])
  37. 2 super(request)
  38. else
  39. 2 probe_response
  40. end
  41. else
  42. 4 super(request)
  43. end
  44. end
  45. end
  46. end
  47. end
  48. 3 register_plugin :ntlm_auth, NTLMAuth
  49. end
  50. end

lib/httpx/plugins/oauth.rb

100.0% lines covered

87 relevant lines. 87 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # https://gitlab.com/os85/httpx/wikis/OAuth
  6. #
  7. 8 module OAuth
  8. 8 class << self
  9. 8 def load_dependencies(_klass)
  10. 144 require_relative "auth/basic"
  11. end
  12. end
  13. 8 SUPPORTED_GRANT_TYPES = %w[client_credentials refresh_token].freeze
  14. 8 SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
  15. 8 class OAuthSession
  16. 8 attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope
  17. 8 def initialize(
  18. issuer:,
  19. client_id:,
  20. client_secret:,
  21. access_token: nil,
  22. refresh_token: nil,
  23. scope: nil,
  24. token_endpoint: nil,
  25. response_type: nil,
  26. grant_type: nil,
  27. token_endpoint_auth_method: nil
  28. )
  29. 128 @issuer = URI(issuer)
  30. 128 @client_id = client_id
  31. 128 @client_secret = client_secret
  32. 128 @token_endpoint = URI(token_endpoint) if token_endpoint
  33. 128 @response_type = response_type
  34. 128 @scope = case scope
  35. when String
  36. 48 scope.split
  37. when Array
  38. 32 scope
  39. end
  40. 128 @access_token = access_token
  41. 128 @refresh_token = refresh_token
  42. 128 @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
  43. 128 @grant_type = grant_type || (@refresh_token ? "refresh_token" : "client_credentials")
  44. 128 unless @token_endpoint_auth_method.nil? || SUPPORTED_AUTH_METHODS.include?(@token_endpoint_auth_method)
  45. 16 raise Error, "#{@token_endpoint_auth_method} is not a supported auth method"
  46. end
  47. 112 return if SUPPORTED_GRANT_TYPES.include?(@grant_type)
  48. 16 raise Error, "#{@grant_type} is not a supported grant type"
  49. end
  50. 8 def token_endpoint
  51. 112 @token_endpoint || "#{@issuer}/token"
  52. end
  53. 8 def token_endpoint_auth_method
  54. 160 @token_endpoint_auth_method || "client_secret_basic"
  55. end
  56. 8 def load(http)
  57. 48 return if @grant_type && @scope
  58. 16 metadata = http.get("#{@issuer}/.well-known/oauth-authorization-server").raise_for_status.json
  59. 16 @token_endpoint = metadata["token_endpoint"]
  60. 16 @scope = metadata["scopes_supported"]
  61. 64 @grant_type = Array(metadata["grant_types_supported"]).find { |gr| SUPPORTED_GRANT_TYPES.include?(gr) }
  62. 16 @token_endpoint_auth_method = Array(metadata["token_endpoint_auth_methods_supported"]).find do |am|
  63. 16 SUPPORTED_AUTH_METHODS.include?(am)
  64. end
  65. 6 nil
  66. end
  67. 8 def merge(other)
  68. 96 obj = dup
  69. 84 case other
  70. when OAuthSession
  71. 48 other.instance_variables.each do |ivar|
  72. 432 val = other.instance_variable_get(ivar)
  73. 432 next unless val
  74. 336 obj.instance_variable_set(ivar, val)
  75. end
  76. when Hash
  77. 48 other.each do |k, v|
  78. 96 obj.instance_variable_set(:"@#{k}", v) if obj.instance_variable_defined?(:"@#{k}")
  79. end
  80. end
  81. 96 obj
  82. end
  83. end
  84. 8 module OptionsMethods
  85. 8 def option_oauth_session(value)
  86. 266 case value
  87. when Hash
  88. 16 OAuthSession.new(**value)
  89. when OAuthSession
  90. 272 value
  91. else
  92. 16 raise TypeError, ":oauth_session must be a #{OAuthSession}"
  93. end
  94. end
  95. end
  96. 8 module InstanceMethods
  97. 8 def oauth_auth(**args)
  98. 112 with(oauth_session: OAuthSession.new(**args))
  99. end
  100. 8 def with_access_token
  101. 48 oauth_session = @options.oauth_session
  102. 48 oauth_session.load(self)
  103. 48 grant_type = oauth_session.grant_type
  104. 48 headers = {}
  105. 48 form_post = { "grant_type" => grant_type, "scope" => Array(oauth_session.scope).join(" ") }.compact
  106. # auth
  107. 42 case oauth_session.token_endpoint_auth_method
  108. when "client_secret_post"
  109. 14 form_post["client_id"] = oauth_session.client_id
  110. 14 form_post["client_secret"] = oauth_session.client_secret
  111. when "client_secret_basic"
  112. 28 headers["authorization"] = Authentication::Basic.new(oauth_session.client_id, oauth_session.client_secret).authenticate
  113. end
  114. 42 case grant_type
  115. when "client_credentials"
  116. # do nothing
  117. when "refresh_token"
  118. 14 form_post["refresh_token"] = oauth_session.refresh_token
  119. end
  120. 48 token_request = build_request("POST", oauth_session.token_endpoint, headers: headers, form: form_post)
  121. 48 token_request.headers.delete("authorization") unless oauth_session.token_endpoint_auth_method == "client_secret_basic"
  122. 48 token_response = request(token_request)
  123. 48 token_response.raise_for_status
  124. 48 payload = token_response.json
  125. 48 access_token = payload["access_token"]
  126. 48 refresh_token = payload["refresh_token"]
  127. 48 with(oauth_session: oauth_session.merge(access_token: access_token, refresh_token: refresh_token))
  128. end
  129. 8 def build_request(*)
  130. 128 request = super
  131. 128 return request if request.headers.key?("authorization")
  132. 96 oauth_session = @options.oauth_session
  133. 96 return request unless oauth_session && oauth_session.access_token
  134. 56 request.headers["authorization"] = "Bearer #{oauth_session.access_token}"
  135. 64 request
  136. end
  137. end
  138. end
  139. 8 register_plugin :oauth, OAuth
  140. end
  141. end

lib/httpx/plugins/persistent.rb

100.0% lines covered

28 relevant lines. 28 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 17 module HTTPX
  3. 17 module Plugins
  4. # This plugin implements a session that persists connections over the duration of the process.
  5. #
  6. # This will improve connection reuse in a long-running process.
  7. #
  8. # One important caveat to note is, although this session might not close connections,
  9. # other sessions from the same process that don't have this plugin turned on might.
  10. #
  11. # This session will still be able to work with it, as if, when expecting a connection
  12. # terminated by a different session, it will just retry on a new one and keep it open.
  13. #
  14. # This plugin is also not recommendable when connecting to >9000 (like, a lot) different origins.
  15. # So when you use this, make sure that you don't fall into this trap.
  16. #
  17. # https://gitlab.com/os85/httpx/wikis/Persistent
  18. #
  19. 17 module Persistent
  20. 17 def self.load_dependencies(klass)
  21. 500 max_retries = if klass.default_options.respond_to?(:max_retries)
  22. 8 [klass.default_options.max_retries, 1].max
  23. else
  24. 492 1
  25. end
  26. 500 klass.plugin(:retries, max_retries: max_retries)
  27. end
  28. 17 def self.extra_options(options)
  29. 500 options.merge(persistent: true)
  30. end
  31. 17 module InstanceMethods
  32. 17 private
  33. 17 def repeatable_request?(request, _)
  34. 531 super || begin
  35. 196 response = request.response
  36. 196 return false unless response && response.is_a?(ErrorResponse)
  37. 28 error = response.error
  38. 308 Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
  39. end
  40. end
  41. 17 def retryable_error?(ex)
  42. 79 super &&
  43. # under the persistent plugin rules, requests are only retried for connection related errors,
  44. # which do not include request timeout related errors. This only gets overriden if the end user
  45. # manually changed +:max_retries+ to something else, which means it is aware of the
  46. # consequences.
  47. 65 (!ex.is_a?(RequestTimeoutError) || @options.max_retries != 1)
  48. end
  49. 17 def get_current_selector
  50. 494 super(&nil) || begin
  51. 478 return unless block_given?
  52. 478 default = yield
  53. 478 set_current_selector(default)
  54. 478 default
  55. end
  56. end
  57. end
  58. end
  59. 17 register_plugin :persistent, Persistent
  60. end
  61. end

lib/httpx/plugins/proxy.rb

98.01% lines covered

151 relevant lines. 148 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 class HTTPProxyError < ConnectionError; end
  4. 10 module Plugins
  5. #
  6. # This plugin adds support for proxies. It ships with support for:
  7. #
  8. # * HTTP proxies
  9. # * HTTPS proxies
  10. # * Socks4/4a proxies
  11. # * Socks5 proxies
  12. #
  13. # https://gitlab.com/os85/httpx/wikis/Proxy
  14. #
  15. 10 module Proxy
  16. 10 Error = HTTPProxyError
  17. 10 PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
  18. 10 class << self
  19. 10 def configure(klass)
  20. 337 klass.plugin(:"proxy/http")
  21. 337 klass.plugin(:"proxy/socks4")
  22. 337 klass.plugin(:"proxy/socks5")
  23. end
  24. 10 def extra_options(options)
  25. 337 options.merge(supported_proxy_protocols: [])
  26. end
  27. end
  28. 10 class Parameters
  29. 10 attr_reader :uri, :username, :password, :scheme, :no_proxy
  30. 10 def initialize(uri: nil, scheme: nil, username: nil, password: nil, no_proxy: nil, **extra)
  31. 371 @no_proxy = Array(no_proxy) if no_proxy
  32. 371 @uris = Array(uri)
  33. 371 uri = @uris.first
  34. 371 @username = username
  35. 371 @password = password
  36. 371 @ns = 0
  37. 371 if uri
  38. 331 @uri = uri.is_a?(URI::Generic) ? uri : URI(uri)
  39. 331 @username ||= @uri.user
  40. 331 @password ||= @uri.password
  41. end
  42. 371 @scheme = scheme
  43. 371 return unless @uri && @username && @password
  44. 210 @authenticator = nil
  45. 210 @scheme ||= infer_default_auth_scheme(@uri)
  46. 210 return unless @scheme
  47. 162 @authenticator = load_authenticator(@scheme, @username, @password, **extra)
  48. end
  49. 10 def shift
  50. # TODO: this operation must be synchronized
  51. 105 @ns += 1
  52. 120 @uri = @uris[@ns]
  53. 120 return unless @uri
  54. 16 @uri = URI(@uri) unless @uri.is_a?(URI::Generic)
  55. 16 scheme = infer_default_auth_scheme(@uri)
  56. 16 return unless scheme != @scheme
  57. 16 @scheme = scheme
  58. 16 @username = username || @uri.user
  59. 16 @password = password || @uri.password
  60. 16 @authenticator = load_authenticator(scheme, @username, @password)
  61. end
  62. 10 def can_authenticate?(*args)
  63. 184 return false unless @authenticator
  64. 64 @authenticator.can_authenticate?(*args)
  65. end
  66. 10 def authenticate(*args)
  67. 159 return unless @authenticator
  68. 159 @authenticator.authenticate(*args)
  69. end
  70. 10 def ==(other)
  71. 384 case other
  72. when Parameters
  73. 396 @uri == other.uri &&
  74. @username == other.username &&
  75. @password == other.password &&
  76. @scheme == other.scheme
  77. when URI::Generic, String
  78. 24 proxy_uri = @uri.dup
  79. 24 proxy_uri.user = @username
  80. 24 proxy_uri.password = @password
  81. 24 other_uri = other.is_a?(URI::Generic) ? other : URI.parse(other)
  82. 24 proxy_uri == other_uri
  83. else
  84. 16 super
  85. end
  86. end
  87. 10 private
  88. 10 def infer_default_auth_scheme(uri)
  89. 185 case uri.scheme
  90. when "socks5"
  91. 48 uri.scheme
  92. when "http", "https"
  93. 101 "basic"
  94. end
  95. end
  96. 10 def load_authenticator(scheme, username, password, **extra)
  97. 178 auth_scheme = scheme.to_s.capitalize
  98. 178 require_relative "auth/#{scheme}" unless defined?(Authentication) && Authentication.const_defined?(auth_scheme, false)
  99. 178 Authentication.const_get(auth_scheme).new(username, password, **extra)
  100. end
  101. end
  102. # adds support for the following options:
  103. #
  104. # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
  105. # *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
  106. 10 module OptionsMethods
  107. 10 def option_proxy(value)
  108. 672 value.is_a?(Parameters) ? value : Parameters.new(**Hash[value])
  109. end
  110. 10 def option_supported_proxy_protocols(value)
  111. 1699 raise TypeError, ":supported_proxy_protocols must be an Array" unless value.is_a?(Array)
  112. 1699 value.map(&:to_s)
  113. end
  114. end
  115. 10 module InstanceMethods
  116. 10 def find_connection(request_uri, selector, options)
  117. 417 return super unless options.respond_to?(:proxy)
  118. 417 if (next_proxy = request_uri.find_proxy)
  119. 4 return super(request_uri, selector, options.merge(proxy: Parameters.new(uri: next_proxy)))
  120. end
  121. 413 proxy = options.proxy
  122. 413 return super unless proxy
  123. 403 next_proxy = proxy.uri
  124. 403 raise Error, "Failed to connect to proxy" unless next_proxy
  125. 1 raise Error,
  126. 387 "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
  127. 379 if (no_proxy = proxy.no_proxy)
  128. 16 no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
  129. # TODO: setting proxy to nil leaks the connection object in the pool
  130. 16 return super(request_uri, selector, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(request_uri.host, next_proxy.host,
  131. next_proxy.port, no_proxy)
  132. end
  133. 371 super(request_uri, selector, options.merge(proxy: proxy))
  134. end
  135. 10 private
  136. 10 def fetch_response(request, selector, options)
  137. 1563 response = super
  138. 1563 if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
  139. 120 options.proxy.shift
  140. # return last error response if no more proxies to try
  141. 120 return response if options.proxy.uri.nil?
  142. 16 log { "failed connecting to proxy, trying next..." }
  143. 16 request.transition(:idle)
  144. 16 send_request(request, selector, options)
  145. 14 return
  146. end
  147. 1443 response
  148. end
  149. 10 def proxy_error?(_request, response, options)
  150. 169 return false unless options.proxy
  151. 168 error = response.error
  152. 147 case error
  153. when NativeResolveError
  154. 16 proxy_uri = URI(options.proxy.uri)
  155. 16 peer = error.connection.peer
  156. # failed resolving proxy domain
  157. 16 peer.host == proxy_uri.host && peer.port == proxy_uri.port
  158. when ResolveError
  159. proxy_uri = URI(options.proxy.uri)
  160. error.message.end_with?(proxy_uri.to_s)
  161. when *PROXY_ERRORS
  162. # timeout errors connecting to proxy
  163. 152 true
  164. else
  165. false
  166. end
  167. end
  168. end
  169. 10 module ConnectionMethods
  170. 10 using URIExtensions
  171. 10 def initialize(*)
  172. 384 super
  173. 384 return unless @options.proxy
  174. # redefining the connection origin as the proxy's URI,
  175. # as this will be used as the tcp peer ip.
  176. 366 @proxy_uri = URI(@options.proxy.uri)
  177. end
  178. 10 def peer
  179. 950 @proxy_uri || super
  180. end
  181. 10 def connecting?
  182. 4593 return super unless @options.proxy
  183. 4429 super || @state == :connecting || @state == :connected
  184. end
  185. 10 def call
  186. 1125 super
  187. 1125 return unless @options.proxy
  188. 991 case @state
  189. when :connecting
  190. 297 consume
  191. end
  192. end
  193. 10 def reset
  194. 384 return super unless @options.proxy
  195. 367 @state = :open
  196. 367 super
  197. # emit(:close)
  198. end
  199. 10 private
  200. 10 def initialize_type(uri, options)
  201. 384 return super unless options.proxy
  202. 366 "tcp"
  203. end
  204. 10 def connect
  205. 1042 return super unless @options.proxy
  206. 891 case @state
  207. when :idle
  208. 712 transition(:connecting)
  209. when :connected
  210. 296 transition(:open)
  211. end
  212. end
  213. 10 def handle_transition(nextstate)
  214. 2117 return super unless @options.proxy
  215. 1791 case nextstate
  216. when :closing
  217. # this is a hack so that we can use the super method
  218. # and it'll think that the current state is open
  219. 375 @state = :open if @state == :connecting
  220. end
  221. 2030 super
  222. end
  223. end
  224. end
  225. 10 register_plugin :proxy, Proxy
  226. end
  227. 10 class ProxySSL < SSL
  228. 10 def initialize(tcp, request_uri, options)
  229. 89 @io = tcp.to_io
  230. 89 super(request_uri, tcp.addresses, options)
  231. 89 @hostname = request_uri.host
  232. 89 @state = :connected
  233. end
  234. end
  235. end

lib/httpx/plugins/proxy/http.rb

100.0% lines covered

104 relevant lines. 104 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 module Plugins
  4. 10 module Proxy
  5. 10 module HTTP
  6. 10 class << self
  7. 10 def extra_options(options)
  8. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[http])
  9. end
  10. end
  11. 10 module InstanceMethods
  12. 10 def with_proxy_basic_auth(opts)
  13. 8 with(proxy: opts.merge(scheme: "basic"))
  14. end
  15. 10 def with_proxy_digest_auth(opts)
  16. 24 with(proxy: opts.merge(scheme: "digest"))
  17. end
  18. 10 def with_proxy_ntlm_auth(opts)
  19. 8 with(proxy: opts.merge(scheme: "ntlm"))
  20. end
  21. 10 def fetch_response(request, selector, options)
  22. 1563 response = super
  23. 1563 if response &&
  24. response.is_a?(Response) &&
  25. response.status == 407 &&
  26. !request.headers.key?("proxy-authorization") &&
  27. response.headers.key?("proxy-authenticate") && options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  28. 8 request.transition(:idle)
  29. 7 request.headers["proxy-authorization"] =
  30. options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  31. 8 send_request(request, selector, options)
  32. 7 return
  33. end
  34. 1555 response
  35. end
  36. end
  37. 10 module ConnectionMethods
  38. 10 def connecting?
  39. 4593 super || @state == :connecting || @state == :connected
  40. end
  41. 10 private
  42. 10 def handle_transition(nextstate)
  43. 2356 return super unless @options.proxy && @options.proxy.uri.scheme == "http"
  44. 1017 case nextstate
  45. when :connecting
  46. 304 return unless @state == :idle
  47. 304 @io.connect
  48. 304 return unless @io.connected?
  49. 152 @parser || begin
  50. 144 @parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
  51. 144 parser = @parser
  52. 144 parser.extend(ProxyParser)
  53. 144 parser.on(:response, &method(:__http_on_connect))
  54. 144 parser.on(:close) do |force|
  55. 57 next unless @parser
  56. 8 if force
  57. 8 reset
  58. 8 emit(:terminate)
  59. end
  60. end
  61. 144 parser.on(:reset) do
  62. 16 if parser.empty?
  63. 8 reset
  64. else
  65. 8 transition(:closing)
  66. 8 transition(:closed)
  67. 8 parser.reset if @parser
  68. 8 transition(:idle)
  69. 8 transition(:connecting)
  70. end
  71. end
  72. 144 __http_proxy_connect(parser)
  73. end
  74. 152 return if @state == :connected
  75. when :connected
  76. 136 return unless @state == :idle || @state == :connecting
  77. 121 case @state
  78. when :connecting
  79. 49 parser = @parser
  80. 49 @parser = nil
  81. 49 parser.close
  82. when :idle
  83. 87 @parser.callbacks.clear
  84. 87 set_parser_callbacks(@parser)
  85. end
  86. end
  87. 904 super
  88. end
  89. 10 def __http_proxy_connect(parser)
  90. 144 req = @pending.first
  91. 144 if req && req.uri.scheme == "https"
  92. # if the first request after CONNECT is to an https address, it is assumed that
  93. # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
  94. # and therefore, will share the connection.
  95. #
  96. 57 connect_request = ConnectRequest.new(req.uri, @options)
  97. 51 @inflight += 1
  98. 57 parser.send(connect_request)
  99. else
  100. 87 handle_transition(:connected)
  101. end
  102. end
  103. 10 def __http_on_connect(request, response)
  104. 58 @inflight -= 1
  105. 65 if response.is_a?(Response) && response.status == 200
  106. 49 req = @pending.first
  107. 49 request_uri = req.uri
  108. 49 @io = ProxySSL.new(@io, request_uri, @options)
  109. 49 transition(:connected)
  110. 49 throw(:called)
  111. 15 elsif response.is_a?(Response) &&
  112. response.status == 407 &&
  113. !request.headers.key?("proxy-authorization") &&
  114. @options.proxy.can_authenticate?(response.headers["proxy-authenticate"])
  115. 8 request.transition(:idle)
  116. 7 request.headers["proxy-authorization"] = @options.proxy.authenticate(request, response.headers["proxy-authenticate"])
  117. 8 @parser.send(request)
  118. 7 @inflight += 1
  119. else
  120. 8 pending = @pending + @parser.pending
  121. 21 while (req = pending.shift)
  122. 8 response.finish!
  123. 8 req.response = response
  124. 8 req.emit(:response, response)
  125. end
  126. 8 reset
  127. end
  128. end
  129. end
  130. 10 module ProxyParser
  131. 10 def join_headline(request)
  132. 144 return super if request.verb == "CONNECT"
  133. 70 "#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
  134. end
  135. 10 def set_protocol_headers(request)
  136. 152 extra_headers = super
  137. 152 proxy_params = @options.proxy
  138. 152 if proxy_params.scheme == "basic"
  139. # opt for basic auth
  140. 85 extra_headers["proxy-authorization"] = proxy_params.authenticate(extra_headers)
  141. end
  142. 152 extra_headers["proxy-connection"] = extra_headers.delete("connection") if extra_headers.key?("connection")
  143. 152 extra_headers
  144. end
  145. end
  146. 10 class ConnectRequest < Request
  147. 10 def initialize(uri, options)
  148. 57 super("CONNECT", uri, options)
  149. 57 @headers.delete("accept")
  150. end
  151. 10 def path
  152. 65 "#{@uri.hostname}:#{@uri.port}"
  153. end
  154. end
  155. end
  156. end
  157. 10 register_plugin :"proxy/http", Proxy::HTTP
  158. end
  159. end

lib/httpx/plugins/proxy/socks4.rb

97.47% lines covered

79 relevant lines. 77 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 require "resolv"
  3. 10 require "ipaddr"
  4. 10 module HTTPX
  5. 10 class Socks4Error < HTTPProxyError; end
  6. 10 module Plugins
  7. 10 module Proxy
  8. 10 module Socks4
  9. 10 VERSION = 4
  10. 10 CONNECT = 1
  11. 10 GRANTED = 0x5A
  12. 10 PROTOCOLS = %w[socks4 socks4a].freeze
  13. 10 Error = Socks4Error
  14. 10 class << self
  15. 10 def extra_options(options)
  16. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + PROTOCOLS)
  17. end
  18. end
  19. 10 module ConnectionMethods
  20. 10 def interests
  21. 3904 if @state == :connecting
  22. return @write_buffer.empty? ? :r : :w
  23. end
  24. 3904 super
  25. end
  26. 10 private
  27. 10 def handle_transition(nextstate)
  28. 2420 return super unless @options.proxy && PROTOCOLS.include?(@options.proxy.uri.scheme)
  29. 385 case nextstate
  30. when :connecting
  31. 128 return unless @state == :idle
  32. 128 @io.connect
  33. 128 return unless @io.connected?
  34. 64 req = @pending.first
  35. 64 return unless req
  36. 64 request_uri = req.uri
  37. 64 @write_buffer << Packet.connect(@options.proxy, request_uri)
  38. 64 __socks4_proxy_connect
  39. when :connected
  40. 48 return unless @state == :connecting
  41. 48 @parser = nil
  42. end
  43. 375 log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  44. 375 super
  45. end
  46. 10 def __socks4_proxy_connect
  47. 64 @parser = SocksParser.new(@write_buffer, @options)
  48. 64 @parser.once(:packet, &method(:__socks4_on_packet))
  49. end
  50. 10 def __socks4_on_packet(packet)
  51. 64 _version, status, _port, _ip = packet.unpack("CCnN")
  52. 64 if status == GRANTED
  53. 48 req = @pending.first
  54. 48 request_uri = req.uri
  55. 48 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  56. 48 transition(:connected)
  57. 48 throw(:called)
  58. else
  59. 16 on_socks4_error("socks error: #{status}")
  60. end
  61. end
  62. 10 def on_socks4_error(message)
  63. 16 ex = Error.new(message)
  64. 16 ex.set_backtrace(caller)
  65. 16 on_error(ex)
  66. 16 throw(:called)
  67. end
  68. end
  69. 10 class SocksParser
  70. 10 include HTTPX::Callbacks
  71. 10 def initialize(buffer, options)
  72. 64 @buffer = buffer
  73. 64 @options = options
  74. end
  75. 10 def close; end
  76. 10 def consume(*); end
  77. 10 def empty?
  78. true
  79. end
  80. 10 def <<(packet)
  81. 64 emit(:packet, packet)
  82. end
  83. end
  84. 10 module Packet
  85. 10 module_function
  86. 10 def connect(parameters, uri)
  87. 64 packet = [VERSION, CONNECT, uri.port].pack("CCn")
  88. 56 case parameters.uri.scheme
  89. when "socks4"
  90. 48 socks_host = uri.host
  91. 5 begin
  92. 96 ip = IPAddr.new(socks_host)
  93. 48 packet << ip.hton
  94. rescue IPAddr::InvalidAddressError
  95. 48 socks_host = Resolv.getaddress(socks_host)
  96. 48 retry
  97. end
  98. 48 packet << [parameters.username].pack("Z*")
  99. when "socks4a"
  100. 16 packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
  101. end
  102. 64 packet
  103. end
  104. end
  105. end
  106. end
  107. 10 register_plugin :"proxy/socks4", Proxy::Socks4
  108. end
  109. end

lib/httpx/plugins/proxy/socks5.rb

99.12% lines covered

113 relevant lines. 112 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 10 module HTTPX
  3. 10 class Socks5Error < HTTPProxyError; end
  4. 10 module Plugins
  5. 10 module Proxy
  6. 10 module Socks5
  7. 10 VERSION = 5
  8. 10 NOAUTH = 0
  9. 10 PASSWD = 2
  10. 10 NONE = 0xff
  11. 10 CONNECT = 1
  12. 10 IPV4 = 1
  13. 10 DOMAIN = 3
  14. 10 IPV6 = 4
  15. 10 SUCCESS = 0
  16. 10 Error = Socks5Error
  17. 10 class << self
  18. 10 def load_dependencies(*)
  19. 337 require_relative "../auth/socks5"
  20. end
  21. 10 def extra_options(options)
  22. 337 options.merge(supported_proxy_protocols: options.supported_proxy_protocols + %w[socks5])
  23. end
  24. end
  25. 10 module ConnectionMethods
  26. 10 def call
  27. 1125 super
  28. 1125 return unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  29. 304 case @state
  30. when :connecting,
  31. :negotiating,
  32. :authenticating
  33. 144 consume
  34. end
  35. end
  36. 10 def connecting?
  37. 4593 super || @state == :authenticating || @state == :negotiating
  38. end
  39. 10 def interests
  40. 6434 if @state == :connecting || @state == :authenticating || @state == :negotiating
  41. 2322 return @write_buffer.empty? ? :r : :w
  42. end
  43. 3904 super
  44. end
  45. 10 private
  46. 10 def handle_transition(nextstate)
  47. 2708 return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
  48. 909 case nextstate
  49. when :connecting
  50. 288 return unless @state == :idle
  51. 288 @io.connect
  52. 288 return unless @io.connected?
  53. 144 @write_buffer << Packet.negotiate(@options.proxy)
  54. 144 __socks5_proxy_connect
  55. when :authenticating
  56. 48 return unless @state == :connecting
  57. 48 @write_buffer << Packet.authenticate(@options.proxy)
  58. when :negotiating
  59. 192 return unless @state == :connecting || @state == :authenticating
  60. 48 req = @pending.first
  61. 48 request_uri = req.uri
  62. 48 @write_buffer << Packet.connect(request_uri)
  63. when :connected
  64. 32 return unless @state == :negotiating
  65. 32 @parser = nil
  66. end
  67. 751 log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
  68. 751 super
  69. end
  70. 10 def __socks5_proxy_connect
  71. 144 @parser = SocksParser.new(@write_buffer, @options)
  72. 144 @parser.on(:packet, &method(:__socks5_on_packet))
  73. 144 transition(:negotiating)
  74. end
  75. 10 def __socks5_on_packet(packet)
  76. 210 case @state
  77. when :connecting
  78. 144 version, method = packet.unpack("CC")
  79. 144 __socks5_check_version(version)
  80. 126 case method
  81. when PASSWD
  82. 48 transition(:authenticating)
  83. 18 nil
  84. when NONE
  85. 80 __on_socks5_error("no supported authorization methods")
  86. else
  87. 16 transition(:negotiating)
  88. end
  89. when :authenticating
  90. 48 _, status = packet.unpack("CC")
  91. 48 return transition(:negotiating) if status == SUCCESS
  92. 16 __on_socks5_error("socks authentication error: #{status}")
  93. when :negotiating
  94. 48 version, reply, = packet.unpack("CC")
  95. 48 __socks5_check_version(version)
  96. 48 __on_socks5_error("socks5 negotiation error: #{reply}") unless reply == SUCCESS
  97. 32 req = @pending.first
  98. 32 request_uri = req.uri
  99. 32 @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
  100. 32 transition(:connected)
  101. 32 throw(:called)
  102. end
  103. end
  104. 10 def __socks5_check_version(version)
  105. 192 __on_socks5_error("invalid SOCKS version (#{version})") if version != 5
  106. end
  107. 10 def __on_socks5_error(message)
  108. 112 ex = Error.new(message)
  109. 112 ex.set_backtrace(caller)
  110. 112 on_error(ex)
  111. 112 throw(:called)
  112. end
  113. end
  114. 10 class SocksParser
  115. 10 include HTTPX::Callbacks
  116. 10 def initialize(buffer, options)
  117. 144 @buffer = buffer
  118. 144 @options = options
  119. end
  120. 10 def close; end
  121. 10 def consume(*); end
  122. 10 def empty?
  123. true
  124. end
  125. 10 def <<(packet)
  126. 240 emit(:packet, packet)
  127. end
  128. end
  129. 10 module Packet
  130. 10 module_function
  131. 10 def negotiate(parameters)
  132. 144 methods = [NOAUTH]
  133. 144 methods << PASSWD if parameters.can_authenticate?
  134. 144 methods.unshift(methods.size)
  135. 144 methods.unshift(VERSION)
  136. 144 methods.pack("C*")
  137. end
  138. 10 def authenticate(parameters)
  139. 48 parameters.authenticate
  140. end
  141. 10 def connect(uri)
  142. 48 packet = [VERSION, CONNECT, 0].pack("C*")
  143. 5 begin
  144. 48 ip = IPAddr.new(uri.host)
  145. 16 ipcode = ip.ipv6? ? IPV6 : IPV4
  146. 16 packet << [ipcode].pack("C") << ip.hton
  147. rescue IPAddr::InvalidAddressError
  148. 32 packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
  149. end
  150. 48 packet << [uri.port].pack("n")
  151. 48 packet
  152. end
  153. end
  154. end
  155. end
  156. 10 register_plugin :"proxy/socks5", Proxy::Socks5
  157. end
  158. end

lib/httpx/plugins/proxy/ssh.rb

92.31% lines covered

52 relevant lines. 48 lines covered and 4 lines missed.
    
  1. # frozen_string_literal: true
  2. 6 require "httpx/plugins/proxy"
  3. 6 module HTTPX
  4. 6 module Plugins
  5. 6 module Proxy
  6. 6 module SSH
  7. 6 class << self
  8. 6 def load_dependencies(*)
  9. 12 require "net/ssh/gateway"
  10. end
  11. end
  12. 6 module OptionsMethods
  13. 6 def option_proxy(value)
  14. 24 Hash[value]
  15. end
  16. end
  17. 6 module InstanceMethods
  18. 6 def request(*args, **options)
  19. 12 raise ArgumentError, "must perform at least one request" if args.empty?
  20. 12 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  21. 12 request = requests.first or return super
  22. 12 request_options = request.options
  23. 12 return super unless request_options.proxy
  24. 12 ssh_options = request_options.proxy
  25. 12 ssh_uris = ssh_options.delete(:uri)
  26. 12 ssh_uri = URI.parse(ssh_uris.shift)
  27. 12 return super unless ssh_uri.scheme == "ssh"
  28. 12 ssh_username = ssh_options.delete(:username)
  29. 12 ssh_options[:port] ||= ssh_uri.port || 22
  30. 12 if request_options.debug
  31. ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
  32. end
  33. 12 request_uri = URI(requests.first.uri)
  34. 12 @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
  35. begin
  36. 12 @_gateway.open(request_uri.host, request_uri.port) do |local_port|
  37. 12 io = build_gateway_socket(local_port, request_uri, request_options)
  38. 12 super(*args, **options.merge(io: io))
  39. end
  40. ensure
  41. 12 @_gateway.shutdown!
  42. end
  43. end
  44. 6 private
  45. 6 def build_gateway_socket(port, request_uri, options)
  46. 12 case request_uri.scheme
  47. when "https"
  48. 6 ctx = OpenSSL::SSL::SSLContext.new
  49. 6 ctx_options = SSL::TLS_OPTIONS.merge(options.ssl)
  50. 6 ctx.set_params(ctx_options) unless ctx_options.empty?
  51. 6 sock = TCPSocket.open("localhost", port)
  52. 6 io = OpenSSL::SSL::SSLSocket.new(sock, ctx)
  53. 6 io.hostname = request_uri.host
  54. 6 io.sync_close = true
  55. 6 io.connect
  56. 6 io.post_connection_check(request_uri.host) if ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
  57. 6 io
  58. when "http"
  59. 6 TCPSocket.open("localhost", port)
  60. else
  61. raise TypeError, "unexpected scheme: #{request_uri.scheme}"
  62. end
  63. end
  64. end
  65. 6 module ConnectionMethods
  66. # should not coalesce connections here, as the IP is the IP of the proxy
  67. 6 def coalescable?(*)
  68. return super unless @options.proxy
  69. false
  70. end
  71. end
  72. end
  73. end
  74. 6 register_plugin :"proxy/ssh", Proxy::SSH
  75. end
  76. end

lib/httpx/plugins/push_promise.rb

100.0% lines covered

41 relevant lines. 41 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for HTTP/2 Push responses.
  6. #
  7. # In order to benefit from this, requests are sent one at a time, so that
  8. # no push responses are received after corresponding request has been sent.
  9. #
  10. # https://gitlab.com/os85/httpx/wikis/Server-Push
  11. #
  12. 8 module PushPromise
  13. 8 def self.extra_options(options)
  14. 16 options.merge(http2_settings: { settings_enable_push: 1 },
  15. max_concurrent_requests: 1)
  16. end
  17. 8 module ResponseMethods
  18. 8 def pushed?
  19. 16 @__pushed
  20. end
  21. 8 def mark_as_pushed!
  22. 8 @__pushed = true
  23. end
  24. end
  25. 8 module InstanceMethods
  26. 8 private
  27. 8 def promise_headers
  28. 16 @promise_headers ||= {}
  29. end
  30. 8 def on_promise(parser, stream)
  31. 16 stream.on(:promise_headers) do |h|
  32. 16 __on_promise_request(parser, stream, h)
  33. end
  34. 16 stream.on(:headers) do |h|
  35. 8 __on_promise_response(parser, stream, h)
  36. end
  37. end
  38. 8 def __on_promise_request(parser, stream, h)
  39. 16 log(level: 1, color: :yellow) do
  40. skipped # :nocov:
  41. skipped h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
  42. skipped # :nocov:
  43. end
  44. 16 headers = @options.headers_class.new(h)
  45. 16 path = headers[":path"]
  46. 16 authority = headers[":authority"]
  47. 24 request = parser.pending.find { |r| r.authority == authority && r.path == path }
  48. 16 if request
  49. 8 request.merge_headers(headers)
  50. 7 promise_headers[stream] = request
  51. 8 parser.pending.delete(request)
  52. 7 parser.streams[request] = stream
  53. 8 request.transition(:done)
  54. else
  55. 8 stream.refuse
  56. end
  57. end
  58. 8 def __on_promise_response(parser, stream, h)
  59. 8 request = promise_headers.delete(stream)
  60. 8 return unless request
  61. 8 parser.__send__(:on_stream_headers, stream, request, h)
  62. 8 response = request.response
  63. 8 response.mark_as_pushed!
  64. 8 stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
  65. 8 stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
  66. end
  67. end
  68. end
  69. 8 register_plugin(:push_promise, PushPromise)
  70. end
  71. end

lib/httpx/plugins/query.rb

100.0% lines covered

15 relevant lines. 15 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for using the experimental QUERY HTTP method
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Query
  8. 8 module Query
  9. 8 def self.subplugins
  10. 3 {
  11. 20 retries: QueryRetries,
  12. }
  13. end
  14. 8 module InstanceMethods
  15. 8 def query(*uri, **options)
  16. 16 request("QUERY", uri, **options)
  17. end
  18. end
  19. 8 module QueryRetries
  20. 8 module InstanceMethods
  21. 8 private
  22. 8 def repeatable_request?(request, options)
  23. 24 super || request.verb == "QUERY"
  24. end
  25. end
  26. end
  27. end
  28. 8 register_plugin :query, Query
  29. end
  30. end

lib/httpx/plugins/rate_limiter.rb

100.0% lines covered

18 relevant lines. 18 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when the request:
  6. #
  7. # * is rate limited;
  8. # * when the server is unavailable (503);
  9. # * when a 3xx request comes with a "retry-after" value
  10. #
  11. # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
  12. #
  13. 8 module RateLimiter
  14. 8 class << self
  15. 8 RATE_LIMIT_CODES = [429, 503].freeze
  16. 8 def configure(klass)
  17. 64 klass.plugin(:retries,
  18. retry_change_requests: true,
  19. 7 retry_on: method(:retry_on_rate_limited_response),
  20. retry_after: method(:retry_after_rate_limit))
  21. end
  22. 8 def retry_on_rate_limited_response(response)
  23. 128 return false unless response.is_a?(Response)
  24. 128 status = response.status
  25. 128 RATE_LIMIT_CODES.include?(status)
  26. end
  27. # Servers send the "Retry-After" header field to indicate how long the
  28. # user agent ought to wait before making a follow-up request. When
  29. # sent with a 503 (Service Unavailable) response, Retry-After indicates
  30. # how long the service is expected to be unavailable to the client.
  31. # When sent with any 3xx (Redirection) response, Retry-After indicates
  32. # the minimum time that the user agent is asked to wait before issuing
  33. # the redirected request.
  34. #
  35. 8 def retry_after_rate_limit(_, response)
  36. 64 return unless response.is_a?(Response)
  37. 64 retry_after = response.headers["retry-after"]
  38. 64 return unless retry_after
  39. 32 Utils.parse_retry_after(retry_after)
  40. end
  41. end
  42. end
  43. 8 register_plugin :rate_limiter, RateLimiter
  44. end
  45. end

lib/httpx/plugins/response_cache.rb

100.0% lines covered

143 relevant lines. 143 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when certain errors happen.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/Response-Cache
  8. #
  9. 8 module ResponseCache
  10. 8 CACHEABLE_VERBS = %w[GET HEAD].freeze
  11. 8 CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
  12. 8 SUPPORTED_VARY_HEADERS = %w[accept accept-encoding accept-language cookie origin].sort.freeze
  13. 8 private_constant :CACHEABLE_VERBS
  14. 8 private_constant :CACHEABLE_STATUS_CODES
  15. 8 class << self
  16. 8 def load_dependencies(*)
  17. 224 require_relative "response_cache/store"
  18. 224 require_relative "response_cache/file_store"
  19. end
  20. # whether the +response+ can be stored in the response cache.
  21. # (i.e. has a cacheable body, does not contain directives prohibiting storage, etc...)
  22. 8 def cacheable_response?(response)
  23. 136 response.is_a?(Response) &&
  24. (
  25. 136 response.cache_control.nil? ||
  26. # TODO: !response.cache_control.include?("private") && is shared cache
  27. !response.cache_control.include?("no-store")
  28. ) &&
  29. CACHEABLE_STATUS_CODES.include?(response.status) &&
  30. # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
  31. # 410 MAY be stored by a cache and used in reply to a subsequent
  32. # request, subject to the expiration mechanism, unless a cache-control
  33. # directive prohibits caching. However, a cache that does not support
  34. # the Range and Content-Range headers MUST NOT cache 206 (Partial
  35. # Content) responses.
  36. response.status != 206
  37. end
  38. # whether the +response+
  39. 8 def not_modified?(response)
  40. 168 response.is_a?(Response) && response.status == 304
  41. end
  42. 8 def extra_options(options)
  43. 224 options.merge(
  44. supported_vary_headers: SUPPORTED_VARY_HEADERS,
  45. response_cache_store: :store,
  46. )
  47. end
  48. end
  49. # adds support for the following options:
  50. #
  51. # :supported_vary_headers :: array of header values that will be considered for a "vary" header based cache validation
  52. # (defaults to {SUPPORTED_VARY_HEADERS}).
  53. # :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
  54. # cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
  55. # abides by the Cache Store Interface
  56. #
  57. # The Cache Store Interface requires implementation of the following methods:
  58. #
  59. # * +#get(request) -> response or nil+
  60. # * +#set(request, response) -> void+
  61. # * +#clear() -> void+)
  62. #
  63. 8 module OptionsMethods
  64. 8 def option_response_cache_store(value)
  65. 329 case value
  66. when :store
  67. 240 Store.new
  68. when :file_store
  69. 16 FileStore.new
  70. else
  71. 120 value
  72. end
  73. end
  74. 8 def option_supported_vary_headers(value)
  75. 224 Array(value).sort
  76. end
  77. end
  78. 8 module InstanceMethods
  79. # wipes out all cached responses from the cache store.
  80. 8 def clear_response_cache
  81. 136 @options.response_cache_store.clear
  82. end
  83. 8 def build_request(*)
  84. 464 request = super
  85. 464 return request unless cacheable_request?(request)
  86. 448 prepare_cache(request)
  87. 448 request
  88. end
  89. 8 private
  90. 8 def send_request(request, *)
  91. 168 return request if request.response
  92. 152 super
  93. end
  94. 8 def fetch_response(request, *)
  95. 532 response = super
  96. 532 return unless response
  97. 168 if ResponseCache.not_modified?(response)
  98. 32 log { "returning cached response for #{request.uri}" }
  99. 32 response.copy_from_cached!
  100. 135 elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
  101. 112 request.options.response_cache_store.set(request, response) unless response.cached?
  102. end
  103. 168 response
  104. end
  105. # will either assign a still-fresh cached response to +request+, or set up its HTTP
  106. # cache invalidation headers in case it's not fresh anymore.
  107. 8 def prepare_cache(request)
  108. 672 cached_response = request.options.response_cache_store.get(request)
  109. 672 return unless cached_response && match_by_vary?(request, cached_response)
  110. 304 cached_response.body.rewind
  111. 304 if cached_response.fresh?
  112. 64 cached_response = cached_response.dup
  113. 64 cached_response.mark_as_cached!
  114. 64 request.response = cached_response
  115. 64 request.emit(:response, cached_response)
  116. 56 return
  117. end
  118. 240 request.cached_response = cached_response
  119. 240 if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
  120. 32 request.headers.add("if-modified-since", last_modified)
  121. end
  122. 240 if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
  123. 176 request.headers.add("if-none-match", etag)
  124. end
  125. end
  126. 8 def cacheable_request?(request)
  127. 464 request.cacheable_verb? &&
  128. (
  129. 448 !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
  130. )
  131. end
  132. # whether the +response+ complies with the directives set by the +request+ "vary" header
  133. # (true when none is available).
  134. 8 def match_by_vary?(request, response)
  135. 304 vary = response.vary
  136. 304 return true unless vary
  137. 96 original_request = response.original_request
  138. 96 if vary == %w[*]
  139. 32 request.options.supported_vary_headers.each do |field|
  140. 160 return false unless request.headers[field] == original_request.headers[field]
  141. end
  142. 28 return true
  143. end
  144. 64 vary.all? do |field|
  145. 64 !original_request.headers.key?(field) || request.headers[field] == original_request.headers[field]
  146. end
  147. end
  148. end
  149. 8 module RequestMethods
  150. # points to a previously cached Response corresponding to this request.
  151. 8 attr_accessor :cached_response
  152. 8 def initialize(*)
  153. 624 super
  154. 624 @cached_response = nil
  155. end
  156. 8 def merge_headers(*)
  157. 296 super
  158. 296 @response_cache_key = nil
  159. end
  160. # returns whether this request is cacheable as per HTTP caching rules.
  161. 8 def cacheable_verb?
  162. 600 CACHEABLE_VERBS.include?(@verb)
  163. end
  164. # returns a unique cache key as a String identifying this request
  165. 8 def response_cache_key
  166. 1296 @response_cache_key ||= begin
  167. 472 keys = [@verb, @uri]
  168. 472 @options.supported_vary_headers.each do |field|
  169. 2360 value = @headers[field]
  170. 2360 keys << value if value
  171. end
  172. 472 Digest::SHA1.hexdigest("httpx-response-cache-#{keys.join("-")}")
  173. end
  174. end
  175. end
  176. 8 module ResponseMethods
  177. 8 attr_writer :original_request
  178. 8 def initialize(*)
  179. 504 super
  180. 504 @cached = false
  181. end
  182. # a copy of the request this response was originally cached from
  183. 8 def original_request
  184. 96 @original_request || @request
  185. end
  186. # whether this Response was duplicated from a previously {RequestMethods#cached_response}.
  187. 8 def cached?
  188. 112 @cached
  189. end
  190. # sets this Response as being duplicated from a previously cached response.
  191. 8 def mark_as_cached!
  192. 64 @cached = true
  193. end
  194. # eager-copies the response headers and body from {RequestMethods#cached_response}.
  195. 8 def copy_from_cached!
  196. 32 cached_response = @request.cached_response
  197. 32 return unless cached_response
  198. # 304 responses do not have content-type, which are needed for decoding.
  199. 32 @headers = @headers.class.new(cached_response.headers.merge(@headers))
  200. 32 @body = cached_response.body.dup
  201. 32 @body.rewind
  202. end
  203. # A response is fresh if its age has not yet exceeded its freshness lifetime.
  204. # other (#cache_control} directives may influence the outcome, as per the rules
  205. # from the {rfc}[https://www.rfc-editor.org/rfc/rfc7234]
  206. 8 def fresh?
  207. 304 if cache_control
  208. 112 return false if cache_control.include?("no-cache")
  209. 80 return true if cache_control.include?("immutable")
  210. # check age: max-age
  211. 192 max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
  212. 192 max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
  213. 80 max_age = max_age[/age=(\d+)/, 1] if max_age
  214. 80 max_age = max_age.to_i if max_age
  215. 80 return max_age > age if max_age
  216. end
  217. # check age: expires
  218. 192 if @headers.key?("expires")
  219. 5 begin
  220. 48 expires = Time.httpdate(@headers["expires"])
  221. rescue ArgumentError
  222. 16 return false
  223. end
  224. 28 return (expires - Time.now).to_i.positive?
  225. end
  226. 144 false
  227. end
  228. # returns the "cache-control" directives as an Array of String(s).
  229. 8 def cache_control
  230. 848 return @cache_control if defined?(@cache_control)
  231. 48 @cache_control = begin
  232. 384 return unless @headers.key?("cache-control")
  233. 112 @headers["cache-control"].split(/ *, */)
  234. end
  235. end
  236. # returns the "vary" header value as an Array of (String) headers.
  237. 8 def vary
  238. 304 return @vary if defined?(@vary)
  239. 34 @vary = begin
  240. 272 return unless @headers.key?("vary")
  241. 64 @headers["vary"].split(/ *, */).map(&:downcase)
  242. end
  243. end
  244. 8 private
  245. # returns the value of the "age" header as an Integer (time since epoch).
  246. # if no "age" of header exists, it returns the number of seconds since {#date}.
  247. 8 def age
  248. 80 return @headers["age"].to_i if @headers.key?("age")
  249. 80 (Time.now - date).to_i
  250. end
  251. # returns the value of the "date" header as a Time object
  252. 8 def date
  253. 80 @date ||= Time.httpdate(@headers["date"])
  254. rescue NoMethodError, ArgumentError
  255. 16 Time.now
  256. end
  257. end
  258. end
  259. 8 register_plugin :response_cache, ResponseCache
  260. end
  261. end

lib/httpx/plugins/response_cache/file_store.rb

100.0% lines covered

72 relevant lines. 72 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 require "pathname"
  3. 8 module HTTPX::Plugins
  4. 8 module ResponseCache
  5. # Implementation of a file system based cache store.
  6. #
  7. # It stores cached responses in a file under a directory pointed by the +dir+
  8. # variable (defaults to the default temp directory from the OS), in a custom
  9. # format (similar but different from HTTP/1.1 request/response framing).
  10. 8 class FileStore
  11. 8 CRLF = HTTPX::Connection::HTTP1::CRLF
  12. 8 attr_reader :dir
  13. 8 def initialize(dir = Dir.tmpdir)
  14. 80 @dir = Pathname.new(dir).join("httpx-response-cache")
  15. 80 FileUtils.mkdir_p(@dir)
  16. end
  17. 8 def clear
  18. 64 FileUtils.rm_rf(@dir)
  19. end
  20. 8 def get(request)
  21. 296 path = file_path(request)
  22. 296 return unless File.exist?(path)
  23. 152 File.open(path, mode: File::RDONLY | File::BINARY) do |f|
  24. 152 f.flock(File::Constants::LOCK_SH)
  25. 152 read_from_file(request, f)
  26. end
  27. end
  28. 8 def set(request, response)
  29. 96 path = file_path(request)
  30. 96 file_exists = File.exist?(path)
  31. 96 mode = file_exists ? File::RDWR : File::CREAT | File::Constants::WRONLY
  32. 96 File.open(path, mode: mode | File::BINARY) do |f|
  33. 96 f.flock(File::Constants::LOCK_EX)
  34. 96 if file_exists
  35. 8 cached_response = read_from_file(request, f)
  36. 8 if cached_response
  37. 8 next if cached_response == request.cached_response
  38. 8 cached_response.close
  39. 8 f.truncate(0)
  40. 8 f.rewind
  41. end
  42. end
  43. # cache the request headers
  44. 96 f << request.verb << CRLF
  45. 96 f << request.uri << CRLF
  46. 96 request.headers.each do |field, value|
  47. 288 f << field << ":" << value << CRLF
  48. end
  49. 96 f << CRLF
  50. # cache the response
  51. 96 f << response.status << CRLF
  52. 96 f << response.version << CRLF
  53. 96 response.headers.each do |field, value|
  54. 264 f << field << ":" << value << CRLF
  55. end
  56. 96 f << CRLF
  57. 96 response.body.rewind
  58. 96 ::IO.copy_stream(response.body, f)
  59. end
  60. end
  61. 8 private
  62. 8 def file_path(request)
  63. 392 @dir.join(request.response_cache_key)
  64. end
  65. 8 def read_from_file(request, f)
  66. # if it's an empty file
  67. 160 return if f.eof?
  68. # read request data
  69. 160 verb = f.readline.delete_suffix!(CRLF)
  70. 160 uri = f.readline.delete_suffix!(CRLF)
  71. 160 request_headers = {}
  72. 700 while (line = f.readline) != CRLF
  73. 480 line.delete_suffix!(CRLF)
  74. 480 sep_index = line.index(":")
  75. 480 field = line.byteslice(0..(sep_index - 1))
  76. 480 value = line.byteslice((sep_index + 1)..-1)
  77. 420 request_headers[field] = value
  78. end
  79. 160 status = f.readline.delete_suffix!(CRLF)
  80. 160 version = f.readline.delete_suffix!(CRLF)
  81. 160 response_headers = {}
  82. 665 while (line = f.readline) != CRLF
  83. 440 line.delete_suffix!(CRLF)
  84. 440 sep_index = line.index(":")
  85. 440 field = line.byteslice(0..(sep_index - 1))
  86. 440 value = line.byteslice((sep_index + 1)..-1)
  87. 385 response_headers[field] = value
  88. end
  89. 160 original_request = request.options.request_class.new(verb, uri, request.options)
  90. 160 original_request.merge_headers(request_headers)
  91. 160 response = request.options.response_class.new(request, status, version, response_headers)
  92. 160 response.original_request = original_request
  93. 160 response.finish!
  94. 160 ::IO.copy_stream(f, response.body)
  95. 160 response
  96. end
  97. end
  98. end
  99. end

lib/httpx/plugins/response_cache/store.rb

100.0% lines covered

16 relevant lines. 16 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX::Plugins
  3. 8 module ResponseCache
  4. # Implementation of a thread-safe in-memory cache store.
  5. 8 class Store
  6. 8 def initialize
  7. 296 @store = {}
  8. 296 @store_mutex = Thread::Mutex.new
  9. end
  10. 8 def clear
  11. 144 @store_mutex.synchronize { @store.clear }
  12. end
  13. 8 def get(request)
  14. 448 @store_mutex.synchronize do
  15. 448 @store[request.response_cache_key]
  16. end
  17. end
  18. 8 def set(request, response)
  19. 200 @store_mutex.synchronize do
  20. 200 cached_response = @store[request.response_cache_key]
  21. 200 cached_response.close if cached_response
  22. 175 @store[request.response_cache_key] = response
  23. end
  24. end
  25. end
  26. end
  27. end

lib/httpx/plugins/retries.rb

96.91% lines covered

97 relevant lines. 94 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 17 module HTTPX
  3. 17 module Plugins
  4. #
  5. # This plugin adds support for retrying requests when errors happen.
  6. #
  7. # It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
  8. # after which it will return the last response, error or not. It will **not** raise an exception.
  9. #
  10. # It does not retry which are not considered idempotent (see *retry_change_requests* to override).
  11. #
  12. # https://gitlab.com/os85/httpx/wikis/Retries
  13. #
  14. 17 module Retries
  15. 17 MAX_RETRIES = 3
  16. # TODO: pass max_retries in a configure/load block
  17. 17 IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
  18. # subset of retryable errors which are safe to retry when reconnecting
  19. 2 RECONNECTABLE_ERRORS = [
  20. 15 IOError,
  21. EOFError,
  22. Errno::ECONNRESET,
  23. Errno::ECONNABORTED,
  24. Errno::EPIPE,
  25. Errno::EINVAL,
  26. Errno::ETIMEDOUT,
  27. ConnectionError,
  28. TLSError,
  29. Connection::HTTP2::Error,
  30. ].freeze
  31. 17 RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
  32. Parser::Error,
  33. TimeoutError,
  34. ]).freeze
  35. 17 DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
  36. 17 if ENV.key?("HTTPX_NO_JITTER")
  37. 16 def self.extra_options(options)
  38. 773 options.merge(max_retries: MAX_RETRIES)
  39. end
  40. else
  41. 1 def self.extra_options(options)
  42. 3 options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
  43. end
  44. end
  45. # adds support for the following options:
  46. #
  47. # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
  48. # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
  49. # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
  50. # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
  51. # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
  52. # (i.e. <tt>->(res) { ... }</tt>).
  53. 17 module OptionsMethods
  54. 17 def option_retry_after(value)
  55. # return early if callable
  56. 208 unless value.respond_to?(:call)
  57. 96 value = Float(value)
  58. 96 raise TypeError, ":retry_after must be positive" unless value.positive?
  59. end
  60. 208 value
  61. end
  62. 17 def option_retry_jitter(value)
  63. # return early if callable
  64. 56 raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
  65. 56 value
  66. end
  67. 17 def option_max_retries(value)
  68. 2334 num = Integer(value)
  69. 2334 raise TypeError, ":max_retries must be positive" unless num >= 0
  70. 2334 num
  71. end
  72. 17 def option_retry_change_requests(v)
  73. 128 v
  74. end
  75. 17 def option_retry_on(value)
  76. 286 raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
  77. 286 value
  78. end
  79. end
  80. 17 module InstanceMethods
  81. # returns a `:retries` plugin enabled session with +n+ maximum retries per request setting.
  82. 17 def max_retries(n)
  83. 96 with(max_retries: n)
  84. end
  85. 17 private
  86. 17 def fetch_response(request, selector, options)
  87. 6416027 response = super
  88. 6416027 if response &&
  89. request.retries.positive? &&
  90. repeatable_request?(request, options) &&
  91. (
  92. 97 (
  93. 339 response.is_a?(ErrorResponse) && retryable_error?(response.error)
  94. ) ||
  95. (
  96. 240 options.retry_on && options.retry_on.call(response)
  97. )
  98. )
  99. 489 try_partial_retry(request, response)
  100. 489 log { "failed to get response, #{request.retries} tries to go..." }
  101. 489 request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
  102. 489 request.transition(:idle)
  103. 489 retry_after = options.retry_after
  104. 489 retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
  105. 489 if retry_after
  106. # apply jitter
  107. 96 if (jitter = request.options.retry_jitter)
  108. 16 retry_after = jitter.call(retry_after)
  109. end
  110. 96 retry_start = Utils.now
  111. 96 log { "retrying after #{retry_after} secs..." }
  112. 96 selector.after(retry_after) do
  113. 96 if (response = request.response)
  114. response.finish!
  115. # request has terminated abruptly meanwhile
  116. request.emit(:response, response)
  117. else
  118. 96 log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
  119. 96 send_request(request, selector, options)
  120. end
  121. end
  122. else
  123. 393 send_request(request, selector, options)
  124. end
  125. 433 return
  126. end
  127. 6415538 response
  128. end
  129. # returns whether +request+ can be retried.
  130. 17 def repeatable_request?(request, options)
  131. 1081 IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
  132. end
  133. # returns whether the +ex+ exception happend for a retriable request.
  134. 17 def retryable_error?(ex)
  135. 3610 RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
  136. end
  137. 17 def proxy_error?(request, response, _)
  138. 64 super && !request.retries.positive?
  139. end
  140. #
  141. # Attempt to set the request to perform a partial range request.
  142. # This happens if the peer server accepts byte-range requests, and
  143. # the last response contains some body payload.
  144. #
  145. 17 def try_partial_retry(request, response)
  146. 489 response = response.response if response.is_a?(ErrorResponse)
  147. 489 return unless response
  148. 233 unless response.headers.key?("accept-ranges") &&
  149. response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
  150. 16 (original_body = response.body)
  151. 217 response.body.close
  152. 191 return
  153. end
  154. 16 request.partial_response = response
  155. 16 size = original_body.bytesize
  156. 14 request.headers["range"] = "bytes=#{size}-"
  157. end
  158. end
  159. 17 module RequestMethods
  160. # number of retries left.
  161. 17 attr_accessor :retries
  162. # a response partially received before.
  163. 17 attr_writer :partial_response
  164. # initializes the request instance, sets the number of retries for the request.
  165. 17 def initialize(*args)
  166. 810 super
  167. 810 @retries = @options.max_retries
  168. end
  169. 17 def response=(response)
  170. 1315 if @partial_response
  171. 16 if response.is_a?(Response) && response.status == 206
  172. 16 response.from_partial_response(@partial_response)
  173. else
  174. @partial_response.close
  175. end
  176. 16 @partial_response = nil
  177. end
  178. 1315 super
  179. end
  180. end
  181. 17 module ResponseMethods
  182. 17 def from_partial_response(response)
  183. 16 @status = response.status
  184. 16 @headers = response.headers
  185. 16 @body = response.body
  186. end
  187. end
  188. end
  189. 17 register_plugin :retries, Retries
  190. end
  191. end

lib/httpx/plugins/ssrf_filter.rb

100.0% lines covered

61 relevant lines. 61 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 class ServerSideRequestForgeryError < Error; end
  4. 8 module Plugins
  5. #
  6. # This plugin adds support for preventing Server-Side Request Forgery attacks.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
  9. #
  10. 8 module SsrfFilter
  11. 8 module IPAddrExtensions
  12. 8 refine IPAddr do
  13. 8 def prefixlen
  14. 128 mask_addr = @mask_addr
  15. 128 raise "Invalid mask" if mask_addr.zero?
  16. 387 mask_addr >>= 1 while (mask_addr & 0x1).zero?
  17. 128 length = 0
  18. 381 while mask_addr & 0x1 == 0x1
  19. 1771 length += 1
  20. 1771 mask_addr >>= 1
  21. end
  22. 128 length
  23. end
  24. end
  25. end
  26. 8 using IPAddrExtensions
  27. # https://en.wikipedia.org/wiki/Reserved_IP_addresses
  28. 2 IPV4_BLACKLIST = [
  29. 6 IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
  30. IPAddr.new("10.0.0.0/8"), # Private network
  31. IPAddr.new("100.64.0.0/10"), # Shared Address Space
  32. IPAddr.new("127.0.0.0/8"), # Loopback
  33. IPAddr.new("169.254.0.0/16"), # Link-local
  34. IPAddr.new("172.16.0.0/12"), # Private network
  35. IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
  36. IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
  37. IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
  38. IPAddr.new("192.168.0.0/16"), # Private network
  39. IPAddr.new("198.18.0.0/15"), # Network benchmark tests
  40. IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
  41. IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
  42. IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
  43. IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
  44. IPAddr.new("255.255.255.255"), # Broadcast
  45. ].freeze
  46. 3 IPV6_BLACKLIST = ([
  47. 6 IPAddr.new("::1/128"), # Loopback
  48. IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
  49. IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
  50. IPAddr.new("2001::/32"), # Teredo tunneling
  51. IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
  52. IPAddr.new("2001:20::/28"), # ORCHIDv2
  53. IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
  54. IPAddr.new("2002::/16"), # 6to4
  55. IPAddr.new("fc00::/7"), # Unique local address
  56. IPAddr.new("fe80::/10"), # Link-local address
  57. IPAddr.new("ff00::/8"), # Multicast
  58. ] + IPV4_BLACKLIST.flat_map do |ipaddr|
  59. 128 prefixlen = ipaddr.prefixlen
  60. 128 ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
  61. 128 ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
  62. 128 [ipv4_compatible, ipv4_mapped]
  63. end).freeze
  64. 8 class << self
  65. 8 def extra_options(options)
  66. 70 options.merge(allowed_schemes: %w[https http])
  67. end
  68. 8 def unsafe_ip_address?(ipaddr)
  69. 154 range = ipaddr.to_range
  70. 154 return true if range.first != range.last
  71. 170 return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
  72. 1964 IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
  73. end
  74. end
  75. # adds support for the following options:
  76. #
  77. # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
  78. 8 module OptionsMethods
  79. 8 def option_allowed_schemes(value)
  80. 78 Array(value)
  81. end
  82. end
  83. 8 module InstanceMethods
  84. 8 def send_requests(*requests)
  85. 86 responses = requests.map do |request|
  86. 86 next if @options.allowed_schemes.include?(request.uri.scheme)
  87. 8 error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
  88. 8 error.set_backtrace(caller)
  89. 8 response = ErrorResponse.new(request, error)
  90. 8 request.emit(:response, response)
  91. 8 response
  92. end
  93. 172 allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
  94. 86 allowed_responses = super(*allowed_requests)
  95. 86 allowed_responses.each_with_index do |res, idx|
  96. 78 req = allowed_requests[idx]
  97. 68 responses[requests.index(req)] = res
  98. end
  99. 86 responses
  100. end
  101. end
  102. 8 module ConnectionMethods
  103. 8 def initialize(*)
  104. begin
  105. 78 super
  106. 8 rescue ServerSideRequestForgeryError => e
  107. # may raise when IPs are passed as options via :addresses
  108. 16 throw(:resolve_error, e)
  109. end
  110. end
  111. 8 def addresses=(addrs)
  112. 232 addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
  113. 78 addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
  114. 78 raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
  115. 16 super
  116. end
  117. end
  118. end
  119. 8 register_plugin :ssrf_filter, SsrfFilter
  120. end
  121. end

lib/httpx/plugins/stream.rb

97.78% lines covered

90 relevant lines. 88 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 14 module HTTPX
  3. 14 class StreamResponse
  4. 14 attr_reader :request
  5. 14 def initialize(request, session)
  6. 186 @request = request
  7. 186 @options = @request.options
  8. 186 @session = session
  9. 186 @response_enum = nil
  10. 186 @buffered_chunks = []
  11. end
  12. 14 def each(&block)
  13. 314 return enum_for(__method__) unless block
  14. 202 if (response_enum = @response_enum)
  15. 16 @response_enum = nil
  16. # streaming already started, let's finish it
  17. 42 while (chunk = @buffered_chunks.shift)
  18. 16 block.call(chunk)
  19. end
  20. # consume enum til the end
  21. 1 begin
  22. 49 while (chunk = response_enum.next)
  23. 25 block.call(chunk)
  24. end
  25. rescue StopIteration
  26. 16 return
  27. end
  28. end
  29. 186 @request.stream = self
  30. 17 begin
  31. 186 @on_chunk = block
  32. 186 response = @session.request(@request)
  33. 170 response.raise_for_status
  34. ensure
  35. 170 @on_chunk = nil
  36. end
  37. end
  38. 14 def each_line
  39. 108 return enum_for(__method__) unless block_given?
  40. 54 line = "".b
  41. 54 each do |chunk|
  42. 50 line << chunk
  43. 140 while (idx = line.index("\n"))
  44. 54 yield line.byteslice(0..idx - 1)
  45. 54 line = line.byteslice(idx + 1..-1)
  46. end
  47. end
  48. 22 yield line unless line.empty?
  49. end
  50. # This is a ghost method. It's to be used ONLY internally, when processing streams
  51. 14 def on_chunk(chunk)
  52. 348 raise NoMethodError unless @on_chunk
  53. 348 @on_chunk.call(chunk)
  54. end
  55. skipped # :nocov:
  56. skipped def inspect
  57. skipped "#<#{self.class}:#{object_id}>"
  58. skipped end
  59. skipped # :nocov:
  60. 14 def to_s
  61. 16 if @request.response
  62. @request.response.to_s
  63. else
  64. 16 @buffered_chunks.join
  65. end
  66. end
  67. 14 private
  68. 14 def response
  69. 356 @request.response || begin
  70. 32 response_enum = each
  71. 56 while (chunk = response_enum.next)
  72. 32 @buffered_chunks << chunk
  73. 32 break if @request.response
  74. end
  75. 32 @response_enum = response_enum
  76. 32 @request.response
  77. end
  78. end
  79. 14 def respond_to_missing?(meth, include_private)
  80. 24 if (response = @request.response)
  81. response.respond_to_missing?(meth, include_private)
  82. else
  83. 24 @options.response_class.method_defined?(meth) || (include_private && @options.response_class.private_method_defined?(meth))
  84. end || super
  85. end
  86. 14 def method_missing(meth, *args, **kwargs, &block)
  87. 178 return super unless response.respond_to?(meth)
  88. 178 response.__send__(meth, *args, **kwargs, &block)
  89. end
  90. end
  91. 14 module Plugins
  92. #
  93. # This plugin adds support for streaming a response (useful for i.e. "text/event-stream" payloads).
  94. #
  95. # https://gitlab.com/os85/httpx/wikis/Stream
  96. #
  97. 14 module Stream
  98. 14 def self.extra_options(options)
  99. 316 options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
  100. end
  101. 14 module InstanceMethods
  102. 14 def request(*args, stream: false, **options)
  103. 506 return super(*args, **options) unless stream
  104. 202 requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
  105. 202 raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
  106. 186 request = requests.first
  107. 186 StreamResponse.new(request, self)
  108. end
  109. end
  110. 14 module RequestMethods
  111. 14 attr_accessor :stream
  112. end
  113. 14 module ResponseMethods
  114. 14 def stream
  115. 298 request = @request.root_request if @request.respond_to?(:root_request)
  116. 298 request ||= @request
  117. 298 request.stream
  118. end
  119. end
  120. 14 module ResponseBodyMethods
  121. 14 def initialize(*)
  122. 298 super
  123. 298 @stream = @response.stream
  124. end
  125. 14 def write(chunk)
  126. 479 return super unless @stream
  127. 393 return 0 if chunk.empty?
  128. 348 chunk = decode_chunk(chunk)
  129. 348 @stream.on_chunk(chunk.dup)
  130. 332 chunk.size
  131. end
  132. 14 private
  133. 14 def transition(*)
  134. 110 return if @stream
  135. 110 super
  136. end
  137. end
  138. end
  139. 14 register_plugin :stream, Stream
  140. end
  141. end

lib/httpx/plugins/stream_bidi.rb

99.3% lines covered

142 relevant lines. 141 lines covered and 1 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for bidirectional HTTP/2 streams.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/StreamBidi
  8. #
  9. # It is required that the request body allows chunk to be buffered, (i.e., responds to +#<<(chunk)+).
  10. 8 module StreamBidi
  11. # Extension of the Connection::HTTP2 class, which adds functionality to
  12. # deal with a request that can't be drained and must be interleaved with
  13. # the response streams.
  14. #
  15. # The streams keeps send DATA frames while there's data; when they're ain't,
  16. # the stream is kept open; it must be explicitly closed by the end user.
  17. #
  18. 8 class HTTP2Bidi < Connection::HTTP2
  19. 8 def initialize(*)
  20. 16 super
  21. 16 @lock = Thread::Mutex.new
  22. end
  23. 8 %i[close empty? exhausted? send <<].each do |lock_meth|
  24. 40 class_eval(<<-METH, __FILE__, __LINE__ + 1)
  25. 5 # lock.aware version of +#{lock_meth}+
  26. 5 def #{lock_meth}(*) # def close(*)
  27. return super if @lock.owned?
  28. # small race condition between
  29. # checking for ownership and
  30. # acquiring lock.
  31. # TODO: fix this at the parser.
  32. @lock.synchronize { super }
  33. end
  34. METH
  35. end
  36. 8 private
  37. 8 %i[join_headers join_trailers join_body].each do |lock_meth|
  38. 24 class_eval(<<-METH, __FILE__, __LINE__ + 1)
  39. 3 # lock.aware version of +#{lock_meth}+
  40. 3 def #{lock_meth}(*) # def join_headers(*)
  41. return super if @lock.owned?
  42. # small race condition between
  43. # checking for ownership and
  44. # acquiring lock.
  45. # TODO: fix this at the parser.
  46. @lock.synchronize { super }
  47. end
  48. METH
  49. end
  50. 8 def handle_stream(stream, request)
  51. 16 request.on(:body) do
  52. 96 next unless request.headers_sent
  53. 80 handle(request, stream)
  54. 80 emit(:flush_buffer)
  55. end
  56. 16 super
  57. end
  58. # when there ain't more chunks, it makes the buffer as full.
  59. 8 def send_chunk(request, stream, chunk, next_chunk)
  60. 96 super
  61. 96 return if next_chunk
  62. 96 request.transition(:waiting_for_chunk)
  63. 96 throw(:buffer_full)
  64. end
  65. # sets end-stream flag when the request is closed.
  66. 8 def end_stream?(request, next_chunk)
  67. 96 request.closed? && next_chunk.nil?
  68. end
  69. end
  70. # BidiBuffer is a Buffer which can be receive data from threads othr
  71. # than the thread of the corresponding Connection/Session.
  72. #
  73. # It synchronizes access to a secondary internal +@oob_buffer+, which periodically
  74. # is reconciled to the main internal +@buffer+.
  75. 8 class BidiBuffer < Buffer
  76. 8 def initialize(*)
  77. 16 super
  78. 16 @parent_thread = Thread.current
  79. 16 @oob_mutex = Thread::Mutex.new
  80. 16 @oob_buffer = "".b
  81. end
  82. # buffers the +chunk+ to be sent
  83. 8 def <<(chunk)
  84. 176 return super if Thread.current == @parent_thread
  85. 80 @oob_mutex.synchronize { @oob_buffer << chunk }
  86. end
  87. # reconciles the main and secondary buffer (which receives data from other threads).
  88. 8 def rebuffer
  89. 1040 raise Error, "can only rebuffer while waiting on a response" unless Thread.current == @parent_thread
  90. 1040 @oob_mutex.synchronize do
  91. 1040 @buffer << @oob_buffer
  92. 1040 @oob_buffer.clear
  93. end
  94. end
  95. end
  96. # Proxy to wake up the session main loop when one
  97. # of the connections has buffered data to write. It abides by the HTTPX::_Selectable API,
  98. # which allows it to be registered in the selector alongside actual HTTP-based
  99. # HTTPX::Connection objects.
  100. 8 class Signal
  101. 8 def initialize
  102. 16 @closed = false
  103. 16 @pipe_read, @pipe_write = ::IO.pipe
  104. end
  105. 8 def state
  106. 247 @closed ? :closed : :open
  107. end
  108. # noop
  109. 8 def log(**); end
  110. 8 def to_io
  111. 463 @pipe_read.to_io
  112. end
  113. 8 def wakeup
  114. 80 return if @closed
  115. 80 @pipe_write.write("\0")
  116. end
  117. 8 def call
  118. 77 return if @closed
  119. 77 @pipe_read.readpartial(1)
  120. end
  121. 8 def interests
  122. 247 return if @closed
  123. 247 :r
  124. end
  125. 8 def timeout; end
  126. 8 def terminate
  127. 16 @pipe_write.close
  128. 16 @pipe_read.close
  129. 16 @closed = true
  130. end
  131. # noop (the owner connection will take of it)
  132. 8 def handle_socket_timeout(interval); end
  133. end
  134. 8 class << self
  135. 8 def load_dependencies(klass)
  136. 16 klass.plugin(:stream)
  137. end
  138. 8 def extra_options(options)
  139. 16 options.merge(fallback_protocol: "h2")
  140. end
  141. end
  142. 8 module InstanceMethods
  143. 8 def initialize(*)
  144. 16 @signal = Signal.new
  145. 16 super
  146. end
  147. 8 def close(selector = Selector.new)
  148. 16 @signal.terminate
  149. 16 selector.deregister(@signal)
  150. 16 super(selector)
  151. end
  152. 8 def select_connection(connection, selector)
  153. 32 super
  154. 32 selector.register(@signal)
  155. 32 connection.signal = @signal
  156. end
  157. 8 def deselect_connection(connection, *)
  158. 16 super
  159. 16 connection.signal = nil
  160. end
  161. end
  162. # Adds synchronization to request operations which may buffer payloads from different
  163. # threads.
  164. 8 module RequestMethods
  165. 8 attr_accessor :headers_sent
  166. 8 def initialize(*)
  167. 16 super
  168. 16 @headers_sent = false
  169. 16 @closed = false
  170. 16 @mutex = Thread::Mutex.new
  171. end
  172. 8 def closed?
  173. 96 @closed
  174. end
  175. 8 def can_buffer?
  176. 221 super && @state != :waiting_for_chunk
  177. end
  178. # overrides state management transitions to introduce an intermediate
  179. # +:waiting_for_chunk+ state, which the request transitions to once payload
  180. # is buffered.
  181. 8 def transition(nextstate)
  182. 376 headers_sent = @headers_sent
  183. 326 case nextstate
  184. when :waiting_for_chunk
  185. 96 return unless @state == :body
  186. when :body
  187. 155 case @state
  188. when :headers
  189. 16 headers_sent = true
  190. when :waiting_for_chunk
  191. # HACK: to allow super to pass through
  192. 80 @state = :headers
  193. end
  194. end
  195. 376 super.tap do
  196. # delay setting this up until after the first transition to :body
  197. 376 @headers_sent = headers_sent
  198. end
  199. end
  200. 8 def <<(chunk)
  201. 80 @mutex.synchronize do
  202. 80 if @drainer
  203. 80 @body.clear if @body.respond_to?(:clear)
  204. 80 @drainer = nil
  205. end
  206. 80 @body << chunk
  207. 80 transition(:body)
  208. end
  209. end
  210. 8 def close
  211. 16 @mutex.synchronize do
  212. 16 return if @closed
  213. 16 @closed = true
  214. end
  215. # last chunk to send which ends the stream
  216. 16 self << ""
  217. end
  218. end
  219. 8 module RequestBodyMethods
  220. 8 def initialize(*, **)
  221. 16 super
  222. 16 @headers.delete("content-length")
  223. end
  224. 8 def empty?
  225. 114 false
  226. end
  227. end
  228. # overrides the declaration of +@write_buffer+, which is now a thread-safe buffer
  229. # responding to the same API.
  230. 8 module ConnectionMethods
  231. 8 attr_writer :signal
  232. 8 def initialize(*)
  233. 16 super
  234. 16 @write_buffer = BidiBuffer.new(@options.buffer_size)
  235. end
  236. # rebuffers the +@write_buffer+ before calculating interests.
  237. 8 def interests
  238. 1040 @write_buffer.rebuffer
  239. 1040 super
  240. end
  241. 8 private
  242. 8 def parser_type(protocol)
  243. 16 return HTTP2Bidi if protocol == "h2"
  244. super
  245. end
  246. 8 def set_parser_callbacks(parser)
  247. 16 super
  248. 16 parser.on(:flush_buffer) do
  249. 80 @signal.wakeup if @signal
  250. end
  251. end
  252. end
  253. end
  254. 8 register_plugin :stream_bidi, StreamBidi
  255. end
  256. end

lib/httpx/plugins/upgrade.rb

100.0% lines covered

34 relevant lines. 34 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
  6. # Upgrade header.
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Upgrade
  9. #
  10. 8 module Upgrade
  11. 8 class << self
  12. 8 def configure(klass)
  13. 31 klass.plugin(:"upgrade/h2")
  14. end
  15. 8 def extra_options(options)
  16. 31 options.merge(upgrade_handlers: {})
  17. end
  18. end
  19. 8 module OptionsMethods
  20. 8 def option_upgrade_handlers(value)
  21. 86 raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
  22. 86 value
  23. end
  24. end
  25. 8 module InstanceMethods
  26. 8 def fetch_response(request, selector, options)
  27. 244 response = super
  28. 244 if response
  29. 90 return response unless response.is_a?(Response)
  30. 90 return response unless response.headers.key?("upgrade")
  31. 35 upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
  32. 35 return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
  33. 35 protocol_handler = options.upgrade_handlers[upgrade_protocol]
  34. 35 return response unless protocol_handler
  35. 35 log { "upgrading to #{upgrade_protocol}..." }
  36. 35 connection = find_connection(request.uri, selector, options)
  37. # do not upgrade already upgraded connections
  38. 35 return if connection.upgrade_protocol == upgrade_protocol
  39. 31 protocol_handler.call(connection, request, response)
  40. # keep in the loop if the server is switching, unless
  41. # the connection has been hijacked, in which case you want
  42. # to terminante immediately
  43. 31 return if response.status == 101 && !connection.hijacked
  44. end
  45. 169 response
  46. end
  47. end
  48. 8 module ConnectionMethods
  49. 8 attr_reader :upgrade_protocol, :hijacked
  50. 8 def hijack_io
  51. 8 @hijacked = true
  52. # connection is taken away from selector and not given back to the pool.
  53. 8 @current_session.deselect_connection(self, @current_selector, true)
  54. end
  55. end
  56. end
  57. 8 register_plugin(:upgrade, Upgrade)
  58. end
  59. end

lib/httpx/plugins/upgrade/h2.rb

91.67% lines covered

24 relevant lines. 22 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin adds support for upgrading an HTTP/1.1 connection to HTTP/2
  6. # via an Upgrade: h2 response declaration
  7. #
  8. # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2
  9. #
  10. 8 module H2
  11. 8 class << self
  12. 8 def extra_options(options)
  13. 31 options.merge(upgrade_handlers: options.upgrade_handlers.merge("h2" => self))
  14. end
  15. 8 def call(connection, _request, _response)
  16. 7 connection.upgrade_to_h2
  17. end
  18. end
  19. 8 module ConnectionMethods
  20. 8 using URIExtensions
  21. 8 def upgrade_to_h2
  22. 7 prev_parser = @parser
  23. 7 if prev_parser
  24. 7 prev_parser.reset
  25. 6 @inflight -= prev_parser.requests.size
  26. end
  27. 7 @parser = Connection::HTTP2.new(@write_buffer, @options)
  28. 7 set_parser_callbacks(@parser)
  29. 7 @upgrade_protocol = "h2"
  30. # what's happening here:
  31. # a deviation from the state machine is done to perform the actions when a
  32. # connection is closed, without transitioning, so the connection is kept in the pool.
  33. # the state is reset to initial, so that the socket reconnect works out of the box,
  34. # while the parser is already here.
  35. 7 purge_after_closed
  36. 7 transition(:idle)
  37. 7 prev_parser.requests.each do |req|
  38. req.transition(:idle)
  39. send(req)
  40. end
  41. end
  42. end
  43. end
  44. 8 register_plugin(:"upgrade/h2", H2)
  45. end
  46. end

lib/httpx/plugins/webdav.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin implements convenience methods for performing WEBDAV requests.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/WebDav
  8. #
  9. 8 module WebDav
  10. 8 def self.configure(klass)
  11. 96 klass.plugin(:xml)
  12. end
  13. 8 module InstanceMethods
  14. 8 def copy(src, dest)
  15. 16 request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
  16. end
  17. 8 def move(src, dest)
  18. 16 request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
  19. end
  20. 8 def lock(path, timeout: nil, &blk)
  21. 48 headers = {}
  22. 42 headers["timeout"] = if timeout && timeout.positive?
  23. 16 "Second-#{timeout}"
  24. else
  25. 32 "Infinite, Second-4100000000"
  26. end
  27. 48 xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
  28. "<D:lockinfo xmlns:D=\"DAV:\">" \
  29. "<D:lockscope><D:exclusive/></D:lockscope>" \
  30. "<D:locktype><D:write/></D:locktype>" \
  31. "<D:owner>null</D:owner>" \
  32. "</D:lockinfo>"
  33. 48 response = request("LOCK", path, headers: headers, xml: xml)
  34. 48 return response unless response.is_a?(Response)
  35. 48 return response unless blk && response.status == 200
  36. 16 lock_token = response.headers["lock-token"]
  37. 1 begin
  38. 16 blk.call(response)
  39. ensure
  40. 16 unlock(path, lock_token)
  41. end
  42. 16 response
  43. end
  44. 8 def unlock(path, lock_token)
  45. 32 request("UNLOCK", path, headers: { "lock-token" => lock_token })
  46. end
  47. 8 def mkcol(dir)
  48. 16 request("MKCOL", dir)
  49. end
  50. 8 def propfind(path, xml = nil)
  51. 64 body = case xml
  52. when :acl
  53. 16 '<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
  54. "<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
  55. when nil
  56. 32 '<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
  57. else
  58. 16 xml
  59. end
  60. 64 request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
  61. end
  62. 8 def proppatch(path, xml)
  63. 6 body = "<?xml version=\"1.0\"?>" \
  64. 12 "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
  65. 16 request("PROPPATCH", path, xml: body)
  66. end
  67. # %i[ orderpatch acl report search]
  68. end
  69. end
  70. 8 register_plugin(:webdav, WebDav)
  71. end
  72. end

lib/httpx/plugins/xml.rb

100.0% lines covered

34 relevant lines. 34 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 8 module HTTPX
  3. 8 module Plugins
  4. #
  5. # This plugin supports request XML encoding/response decoding using the nokogiri gem.
  6. #
  7. # https://gitlab.com/os85/httpx/wikis/XML
  8. #
  9. 8 module XML
  10. 8 MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
  11. 8 module Transcoder
  12. 8 module_function
  13. 8 class Encoder
  14. 8 def initialize(xml)
  15. 160 @raw = xml
  16. end
  17. 8 def content_type
  18. 160 charset = @raw.respond_to?(:encoding) && @raw.encoding ? @raw.encoding.to_s.downcase : "utf-8"
  19. 160 "application/xml; charset=#{charset}"
  20. end
  21. 8 def bytesize
  22. 512 @raw.to_s.bytesize
  23. end
  24. 8 def to_s
  25. 160 @raw.to_s
  26. end
  27. end
  28. 8 def encode(xml)
  29. 160 Encoder.new(xml)
  30. end
  31. 8 def decode(response)
  32. 24 content_type = response.content_type.mime_type
  33. 24 raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
  34. 24 Nokogiri::XML.method(:parse)
  35. end
  36. end
  37. 8 class << self
  38. 8 def load_dependencies(*)
  39. 144 require "nokogiri"
  40. end
  41. end
  42. 8 module ResponseMethods
  43. # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
  44. # "application/xml" (requires the "nokogiri" gem).
  45. 8 def xml
  46. 16 decode(Transcoder)
  47. end
  48. end
  49. 8 module RequestBodyClassMethods
  50. # ..., xml: Nokogiri::XML::Node #=> xml encoder
  51. 8 def initialize_body(params)
  52. 592 if (xml = params.delete(:xml))
  53. # @type var xml: Nokogiri::XML::Node | String
  54. 140 return Transcoder.encode(xml)
  55. end
  56. 432 super
  57. end
  58. end
  59. end
  60. 8 register_plugin(:xml, XML)
  61. end
  62. end

lib/httpx/pmatch_extensions.rb

100.0% lines covered

17 relevant lines. 17 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module ResponsePatternMatchExtensions
  4. 27 def deconstruct
  5. 41 [@status, @headers, @body]
  6. end
  7. 27 def deconstruct_keys(_keys)
  8. 70 { status: @status, headers: @headers, body: @body }
  9. end
  10. end
  11. 27 module ErrorResponsePatternMatchExtensions
  12. 27 def deconstruct
  13. 11 [@error]
  14. end
  15. 27 def deconstruct_keys(_keys)
  16. 35 { error: @error }
  17. end
  18. end
  19. 27 module HeadersPatternMatchExtensions
  20. 27 def deconstruct
  21. 7 to_a
  22. end
  23. end
  24. 27 Headers.include HeadersPatternMatchExtensions
  25. 27 Response.include ResponsePatternMatchExtensions
  26. 27 ErrorResponse.include ErrorResponsePatternMatchExtensions
  27. end

lib/httpx/pool.rb

100.0% lines covered

89 relevant lines. 89 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "httpx/selector"
  3. 27 require "httpx/connection"
  4. 27 require "httpx/resolver"
  5. 27 module HTTPX
  6. 27 class Pool
  7. 27 using ArrayExtensions::FilterMap
  8. 27 using URIExtensions
  9. 27 POOL_TIMEOUT = 5
  10. # Sets up the connection pool with the given +options+, which can be the following:
  11. #
  12. # :max_connections:: the maximum number of connections held in the pool.
  13. # :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
  14. # :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
  15. #
  16. 27 def initialize(options)
  17. 11175 @max_connections = options.fetch(:max_connections, Float::INFINITY)
  18. 11175 @max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
  19. 11175 @pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
  20. 17322 @resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
  21. 11175 @resolver_mtx = Thread::Mutex.new
  22. 11175 @connections = []
  23. 11175 @connection_mtx = Thread::Mutex.new
  24. 11175 @connections_counter = 0
  25. 11175 @max_connections_cond = ConditionVariable.new
  26. 11175 @origin_counters = Hash.new(0)
  27. 16577 @origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
  28. end
  29. # connections returned by this function are not expected to return to the connection pool.
  30. 27 def pop_connection
  31. 11634 @connection_mtx.synchronize do
  32. 11634 drop_connection
  33. end
  34. end
  35. # opens a connection to the IP reachable through +uri+.
  36. # Many hostnames are reachable through the same IP, so we try to
  37. # maximize pipelining by opening as few connections as possible.
  38. #
  39. 27 def checkout_connection(uri, options)
  40. 8085 return checkout_new_connection(uri, options) if options.io
  41. 8019 @connection_mtx.synchronize do
  42. 8019 acquire_connection(uri, options) || begin
  43. 7383 if @connections_counter == @max_connections
  44. # this takes precedence over per-origin
  45. 16 @max_connections_cond.wait(@connection_mtx, @pool_timeout)
  46. 16 acquire_connection(uri, options) || begin
  47. 12 if @connections_counter == @max_connections
  48. # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
  49. # this means that all of them are persistent or being used, so raise a timeout error.
  50. 14 conn = @connections.find { |c| c.state == :closed }
  51. 1 raise PoolTimeoutError.new(@pool_timeout,
  52. 11 "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
  53. 3 drop_connection(conn)
  54. end
  55. end
  56. end
  57. 7375 if @origin_counters[uri.origin] == @max_connections_per_origin
  58. 16 @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
  59. 16 return acquire_connection(uri, options) ||
  60. raise(PoolTimeoutError.new(@pool_timeout,
  61. 1 "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
  62. end
  63. 6543 @connections_counter += 1
  64. 7359 @origin_counters[uri.origin] += 1
  65. 7359 checkout_new_connection(uri, options)
  66. end
  67. end
  68. end
  69. 27 def checkin_connection(connection)
  70. 7952 return if connection.options.io
  71. 7886 @connection_mtx.synchronize do
  72. 7886 @connections << connection
  73. 7886 @max_connections_cond.signal
  74. 7886 @origin_conds[connection.origin.to_s].signal
  75. end
  76. end
  77. 27 def checkout_mergeable_connection(connection)
  78. 7355 return if connection.options.io
  79. 7355 @connection_mtx.synchronize do
  80. 7355 idx = @connections.find_index do |ch|
  81. 239 ch != connection && ch.mergeable?(connection)
  82. end
  83. 7355 @connections.delete_at(idx) if idx
  84. end
  85. end
  86. 27 def reset_resolvers
  87. 14424 @resolver_mtx.synchronize { @resolvers.clear }
  88. end
  89. 27 def checkout_resolver(options)
  90. 7175 resolver_type = options.resolver_class
  91. 7175 resolver_type = Resolver.resolver_for(resolver_type)
  92. 7175 @resolver_mtx.synchronize do
  93. 7175 resolvers = @resolvers[resolver_type]
  94. 7175 idx = resolvers.find_index do |res|
  95. 32 res.options == options
  96. end
  97. 7175 resolvers.delete_at(idx) if idx
  98. end || checkout_new_resolver(resolver_type, options)
  99. end
  100. 27 def checkin_resolver(resolver)
  101. 366 @resolver_mtx.synchronize do
  102. 366 resolvers = @resolvers[resolver.class]
  103. 366 resolver = resolver.multi
  104. 366 resolvers << resolver unless resolvers.include?(resolver)
  105. end
  106. end
  107. skipped # :nocov:
  108. skipped def inspect
  109. skipped "#<#{self.class}:#{object_id} " \
  110. skipped "@max_connections_per_origin=#{@max_connections_per_origin} " \
  111. skipped "@pool_timeout=#{@pool_timeout} " \
  112. skipped "@connections=#{@connections.size}>"
  113. skipped end
  114. skipped # :nocov:
  115. 27 private
  116. 27 def acquire_connection(uri, options)
  117. 8051 idx = @connections.find_index do |connection|
  118. 864 connection.match?(uri, options)
  119. end
  120. 8051 return unless idx
  121. 648 @connections.delete_at(idx)
  122. end
  123. 27 def checkout_new_connection(uri, options)
  124. 7425 options.connection_class.new(uri, options)
  125. end
  126. 27 def checkout_new_resolver(resolver_type, options)
  127. 7147 if resolver_type.multi?
  128. 7122 Resolver::Multi.new(resolver_type, options)
  129. else
  130. 25 resolver_type.new(options)
  131. end
  132. end
  133. # drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),
  134. # the first available connection from the pool will be dropped.
  135. 27 def drop_connection(connection = nil)
  136. 11637 if connection
  137. 3 @connections.delete(connection)
  138. else
  139. 11634 connection = @connections.shift
  140. 11634 return unless connection
  141. end
  142. 3907 @connections_counter -= 1
  143. 4425 @origin_conds.delete(connection.origin) if (@origin_counters[connection.origin.to_s] -= 1).zero?
  144. 4425 connection
  145. end
  146. end
  147. end

lib/httpx/punycode.rb

100.0% lines covered

9 relevant lines. 9 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Punycode
  4. 27 module_function
  5. begin
  6. 27 require "idnx"
  7. 26 def encode_hostname(hostname)
  8. 32 Idnx.to_punycode(hostname)
  9. end
  10. rescue LoadError
  11. 1 def encode_hostname(hostname)
  12. 1 warn "#{hostname} cannot be converted to punycode. Install the " \
  13. "\"idnx\" gem: https://github.com/HoneyryderChuck/idnx"
  14. 1 hostname
  15. end
  16. end
  17. end
  18. end

lib/httpx/request.rb

100.0% lines covered

132 relevant lines. 132 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "delegate"
  3. 27 require "forwardable"
  4. 27 module HTTPX
  5. # Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
  6. # as well as maintaining the state machine which manages streaming the request onto the wire.
  7. 27 class Request
  8. 27 extend Forwardable
  9. 27 include Callbacks
  10. 27 using URIExtensions
  11. 27 ALLOWED_URI_SCHEMES = %w[https http].freeze
  12. # default value used for "user-agent" header, when not overridden.
  13. 27 USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
  14. # the upcased string HTTP verb for this request.
  15. 27 attr_reader :verb
  16. # the absolute URI object for this request.
  17. 27 attr_reader :uri
  18. # an HTTPX::Headers object containing the request HTTP headers.
  19. 27 attr_reader :headers
  20. # an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
  21. 27 attr_reader :body
  22. # a symbol describing which frame is currently being flushed.
  23. 27 attr_reader :state
  24. # an HTTPX::Options object containing request options.
  25. 27 attr_reader :options
  26. # the corresponding HTTPX::Response object, when there is one.
  27. 27 attr_reader :response
  28. # Exception raised during enumerable body writes.
  29. 27 attr_reader :drain_error
  30. # The IP address from the peer server.
  31. 27 attr_accessor :peer_address
  32. 27 attr_writer :persistent
  33. 27 attr_reader :active_timeouts
  34. # will be +true+ when request body has been completely flushed.
  35. 27 def_delegator :@body, :empty?
  36. # closes the body
  37. 27 def_delegator :@body, :close
  38. # initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
  39. # an absolute or relative +uri+ (either as String or URI::HTTP object), the
  40. # request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
  41. #
  42. # Besides any of the options documented in HTTPX::Options (which would override or merge with what
  43. # +options+ sets), it accepts also the following:
  44. #
  45. # :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
  46. # :body :: to be encoded in the request body payload. can be a String, an IO object (i.e. a File), or an Enumerable.
  47. # :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
  48. # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
  49. # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
  50. #
  51. # :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
  52. 27 def initialize(verb, uri, options, params = EMPTY_HASH)
  53. 10273 @verb = verb.to_s.upcase
  54. 10273 @uri = Utils.to_uri(uri)
  55. 10272 @headers = options.headers.dup
  56. 10272 merge_headers(params.delete(:headers)) if params.key?(:headers)
  57. 10272 @headers["user-agent"] ||= USER_AGENT
  58. 10272 @headers["accept"] ||= "*/*"
  59. # forego compression in the Range request case
  60. 10272 if @headers.key?("range")
  61. 8 @headers.delete("accept-encoding")
  62. else
  63. 10264 @headers["accept-encoding"] ||= options.supported_compression_formats
  64. end
  65. 10272 @query_params = params.delete(:params) if params.key?(:params)
  66. 10272 @body = options.request_body_class.new(@headers, options, **params)
  67. 10264 @options = @body.options
  68. 10264 if @uri.relative? || @uri.host.nil?
  69. 600 origin = @options.origin
  70. 600 raise(Error, "invalid URI: #{@uri}") unless origin
  71. 576 base_path = @options.base_path
  72. 576 @uri = origin.merge("#{base_path}#{@uri}")
  73. end
  74. 10240 raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
  75. 10228 @state = :idle
  76. 10228 @response = nil
  77. 10228 @peer_address = nil
  78. 10228 @ping = false
  79. 10228 @persistent = @options.persistent
  80. 10228 @active_timeouts = []
  81. end
  82. # whether request has been buffered with a ping
  83. 27 def ping?
  84. 489 @ping
  85. end
  86. # marks the request as having been buffered with a ping
  87. 27 def ping!
  88. 29 @ping = true
  89. end
  90. # the read timeout defined for this request.
  91. 27 def read_timeout
  92. 19145 @options.timeout[:read_timeout]
  93. end
  94. # the write timeout defined for this request.
  95. 27 def write_timeout
  96. 19145 @options.timeout[:write_timeout]
  97. end
  98. # the request timeout defined for this request.
  99. 27 def request_timeout
  100. 18861 @options.timeout[:request_timeout]
  101. end
  102. 27 def persistent?
  103. 4891 @persistent
  104. end
  105. # if the request contains trailer headers
  106. 27 def trailers?
  107. 3130 defined?(@trailers)
  108. end
  109. # returns an instance of HTTPX::Headers containing the trailer headers
  110. 27 def trailers
  111. 88 @trailers ||= @options.headers_class.new
  112. end
  113. # returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
  114. 27 def interests
  115. 25237 return :r if @state == :done || @state == :expect
  116. 2930 :w
  117. end
  118. 27 def can_buffer?
  119. 26091 @state != :done
  120. end
  121. # merges +h+ into the instance of HTTPX::Headers of the request.
  122. 27 def merge_headers(h)
  123. 1107 @headers = @headers.merge(h)
  124. end
  125. # the URI scheme of the request +uri+.
  126. 27 def scheme
  127. 3699 @uri.scheme
  128. end
  129. # sets the +response+ on this request.
  130. 27 def response=(response)
  131. 9581 return unless response
  132. 9581 if response.is_a?(Response) && response.status < 200
  133. # deal with informational responses
  134. 160 if response.status == 100 && @headers.key?("expect")
  135. 136 @informational_status = response.status
  136. 136 return
  137. end
  138. # 103 Early Hints advertises resources in document to browsers.
  139. # not very relevant for an HTTP client, discard.
  140. 24 return if response.status >= 103
  141. end
  142. 9445 @response = response
  143. 9445 emit(:response_started, response)
  144. end
  145. # returnns the URI path of the request +uri+.
  146. 27 def path
  147. 8740 path = uri.path.dup
  148. 8740 path = +"" if path.nil?
  149. 8740 path << "/" if path.empty?
  150. 8740 path << "?#{query}" unless query.empty?
  151. 8740 path
  152. end
  153. # returs the URI authority of the request.
  154. #
  155. # session.build_request("GET", "https://google.com/query").authority #=> "google.com"
  156. # session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
  157. 27 def authority
  158. 8830 @uri.authority
  159. end
  160. # returs the URI origin of the request.
  161. #
  162. # session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
  163. # session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
  164. 27 def origin
  165. 3980 @uri.origin
  166. end
  167. # returs the URI query string of the request (when available).
  168. #
  169. # session.build_request("GET", "https://search.com").query #=> ""
  170. # session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
  171. # session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
  172. # session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
  173. 27 def query
  174. 9703 return @query if defined?(@query)
  175. 8160 query = []
  176. 8160 if (q = @query_params) && !q.empty?
  177. 152 query << Transcoder::Form.encode(q)
  178. end
  179. 8160 query << @uri.query if @uri.query
  180. 8160 @query = query.join("&")
  181. end
  182. # consumes and returns the next available chunk of request body that can be sent
  183. 27 def drain_body
  184. 9298 return nil if @body.nil?
  185. 9298 @drainer ||= @body.each
  186. 9298 chunk = @drainer.next.dup
  187. 6115 emit(:body_chunk, chunk)
  188. 6115 chunk
  189. rescue StopIteration
  190. 3159 nil
  191. rescue StandardError => e
  192. 24 @drain_error = e
  193. 24 nil
  194. end
  195. skipped # :nocov:
  196. skipped def inspect
  197. skipped "#<#{self.class}:#{object_id} " \
  198. skipped "#{@verb} " \
  199. skipped "#{uri} " \
  200. skipped "@headers=#{@headers} " \
  201. skipped "@body=#{@body}>"
  202. skipped end
  203. skipped # :nocov:
  204. # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
  205. 27 def transition(nextstate)
  206. 37178 case nextstate
  207. when :idle
  208. 744 @body.rewind
  209. 744 @ping = false
  210. 744 @response = nil
  211. 744 @drainer = nil
  212. 744 @active_timeouts.clear
  213. when :headers
  214. 11290 return unless @state == :idle
  215. when :body
  216. 11354 return unless @state == :headers ||
  217. @state == :expect
  218. 9193 if @headers.key?("expect")
  219. 516 if @informational_status && @informational_status == 100
  220. # check for 100 Continue response, and deallocate the var
  221. # if @informational_status == 100
  222. # @response = nil
  223. # end
  224. else
  225. 389 return if @state == :expect # do not re-set it
  226. 144 nextstate = :expect
  227. end
  228. end
  229. when :trailers
  230. 9092 return unless @state == :body
  231. when :done
  232. 9100 return if @state == :expect
  233. end
  234. 35931 @state = nextstate
  235. 35931 emit(@state, self)
  236. 14426 nil
  237. end
  238. # whether the request supports the 100-continue handshake and already processed the 100 response.
  239. 27 def expects?
  240. 8246 @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
  241. end
  242. 27 def set_timeout_callback(event, &callback)
  243. 96147 clb = once(event, &callback)
  244. # reset timeout callbacks when requests get rerouted to a different connection
  245. 96147 once(:idle) do
  246. 3560 callbacks(event).delete(clb)
  247. end
  248. end
  249. end
  250. end
  251. 27 require_relative "request/body"

lib/httpx/request/body.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
  4. 27 class Request::Body < SimpleDelegator
  5. 27 class << self
  6. 27 def new(_, options, body: nil, **params)
  7. 10280 if body.is_a?(self)
  8. # request derives its options from body
  9. 16 body.options = options.merge(params)
  10. 14 return body
  11. end
  12. 10264 super
  13. end
  14. end
  15. 27 attr_accessor :options
  16. # inits the instance with the request +headers+, +options+ and +params+, which contain the payload definition.
  17. # it wraps the given body with the appropriate encoder on initialization.
  18. #
  19. # ..., json: { foo: "bar" }) #=> json encoder
  20. # ..., form: { foo: "bar" }) #=> form urlencoded encoder
  21. # ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
  22. # ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
  23. # ..., form: { body: "bla") }) #=> raw data encoder
  24. 27 def initialize(h, options, **params)
  25. 10264 @headers = h
  26. 10264 @body = self.class.initialize_body(params)
  27. 10264 @options = options.merge(params)
  28. 10264 if @body
  29. 3196 if @options.compress_request_body && @headers.key?("content-encoding")
  30. 96 @headers.get("content-encoding").each do |encoding|
  31. 96 @body = self.class.initialize_deflater_body(@body, encoding)
  32. end
  33. end
  34. 3196 @headers["content-type"] ||= @body.content_type
  35. 3196 @headers["content-length"] = @body.bytesize unless unbounded_body?
  36. end
  37. 10256 super(@body)
  38. end
  39. # consumes and yields the request payload in chunks.
  40. 27 def each(&block)
  41. 6610 return enum_for(__method__) unless block
  42. 3309 return if @body.nil?
  43. 3237 body = stream(@body)
  44. 3237 if body.respond_to?(:read)
  45. 5434 while (chunk = body.read(16_384))
  46. 3363 block.call(chunk)
  47. end
  48. # TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
  49. # ::IO.copy_stream(body, ProcIO.new(block))
  50. 1847 elsif body.respond_to?(:each)
  51. 510 body.each(&block)
  52. else
  53. 1338 block[body.to_s]
  54. end
  55. end
  56. 27 def close
  57. 457 @body.close if @body.respond_to?(:close)
  58. end
  59. # if the +@body+ is rewindable, it rewinnds it.
  60. 27 def rewind
  61. 808 return if empty?
  62. 172 @body.rewind if @body.respond_to?(:rewind)
  63. end
  64. # return +true+ if the +body+ has been fully drained (or does nnot exist).
  65. 27 def empty?
  66. 19523 return true if @body.nil?
  67. 8725 return false if chunked?
  68. 8629 @body.bytesize.zero?
  69. end
  70. # returns the +@body+ payload size in bytes.
  71. 27 def bytesize
  72. 3471 return 0 if @body.nil?
  73. 128 @body.bytesize
  74. end
  75. # sets the body to yield using chunked trannsfer encoding format.
  76. 27 def stream(body)
  77. 3237 return body unless chunked?
  78. 96 Transcoder::Chunker.encode(body.enum_for(:each))
  79. end
  80. # returns whether the body yields infinitely.
  81. 27 def unbounded_body?
  82. 3720 return @unbounded_body if defined?(@unbounded_body)
  83. 3268 @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
  84. end
  85. # returns whether the chunked transfer encoding header is set.
  86. 27 def chunked?
  87. 20270 @headers["transfer-encoding"] == "chunked"
  88. end
  89. # sets the chunked transfer encoding header.
  90. 27 def chunk!
  91. 32 @headers.add("transfer-encoding", "chunked")
  92. end
  93. skipped # :nocov:
  94. skipped def inspect
  95. skipped "#<#{self.class}:#{object_id} " \
  96. skipped "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
  97. skipped end
  98. skipped # :nocov:
  99. 27 class << self
  100. 27 def initialize_body(params)
  101. 10104 if (body = params.delete(:body))
  102. # @type var body: bodyIO
  103. 1448 Transcoder::Body.encode(body)
  104. 8656 elsif (form = params.delete(:form))
  105. # @type var form: Transcoder::urlencoded_input
  106. 1505 Transcoder::Form.encode(form)
  107. 7151 elsif (json = params.delete(:json))
  108. # @type var body: _ToJson
  109. 83 Transcoder::JSON.encode(json)
  110. end
  111. end
  112. # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
  113. 27 def initialize_deflater_body(body, encoding)
  114. 87 case encoding
  115. when "gzip"
  116. 48 Transcoder::GZIP.encode(body)
  117. when "deflate"
  118. 24 Transcoder::Deflate.encode(body)
  119. when "identity"
  120. 16 body
  121. else
  122. 8 body
  123. end
  124. end
  125. end
  126. end
  127. end

lib/httpx/resolver.rb

100.0% lines covered

82 relevant lines. 82 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "ipaddr"
  4. 27 module HTTPX
  5. 27 module Resolver
  6. 27 RESOLVE_TIMEOUT = [2, 3].freeze
  7. 27 require "httpx/resolver/resolver"
  8. 27 require "httpx/resolver/system"
  9. 27 require "httpx/resolver/native"
  10. 27 require "httpx/resolver/https"
  11. 27 require "httpx/resolver/multi"
  12. 27 @lookup_mutex = Thread::Mutex.new
  13. 194 @lookups = Hash.new { |h, k| h[k] = [] }
  14. 27 @identifier_mutex = Thread::Mutex.new
  15. 27 @identifier = 1
  16. 27 @system_resolver = Resolv::Hosts.new
  17. 27 module_function
  18. 27 def resolver_for(resolver_type)
  19. 6404 case resolver_type
  20. 7038 when :native then Native
  21. 33 when :system then System
  22. 74 when :https then HTTPS
  23. else
  24. 76 return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
  25. 8 raise Error, "unsupported resolver type (#{resolver_type})"
  26. end
  27. end
  28. 27 def nolookup_resolve(hostname)
  29. 7020 ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
  30. end
  31. 27 def ip_resolve(hostname)
  32. 7020 [IPAddr.new(hostname)]
  33. rescue ArgumentError
  34. end
  35. 27 def system_resolve(hostname)
  36. 606 ips = @system_resolver.getaddresses(hostname)
  37. 606 return if ips.empty?
  38. 822 ips.map { |ip| IPAddr.new(ip) }
  39. rescue IOError
  40. end
  41. 27 def cached_lookup(hostname)
  42. 6483 now = Utils.now
  43. 6483 lookup_synchronize do |lookups|
  44. 6483 lookup(hostname, lookups, now)
  45. end
  46. end
  47. 27 def cached_lookup_set(hostname, family, entries)
  48. 234 now = Utils.now
  49. 234 entries.each do |entry|
  50. 347 entry["TTL"] += now
  51. end
  52. 234 lookup_synchronize do |lookups|
  53. 206 case family
  54. when Socket::AF_INET6
  55. 40 lookups[hostname].concat(entries)
  56. when Socket::AF_INET
  57. 194 lookups[hostname].unshift(*entries)
  58. end
  59. 234 entries.each do |entry|
  60. 347 next unless entry["name"] != hostname
  61. 30 case family
  62. when Socket::AF_INET6
  63. 8 lookups[entry["name"]] << entry
  64. when Socket::AF_INET
  65. 24 lookups[entry["name"]].unshift(entry)
  66. end
  67. end
  68. end
  69. end
  70. # do not use directly!
  71. 27 def lookup(hostname, lookups, ttl)
  72. 6491 return unless lookups.key?(hostname)
  73. 5883 entries = lookups[hostname] = lookups[hostname].select do |address|
  74. 17820 address["TTL"] > ttl
  75. end
  76. 5883 ips = entries.flat_map do |address|
  77. 17783 if (als = address["alias"])
  78. 8 lookup(als, lookups, ttl)
  79. else
  80. 17775 IPAddr.new(address["data"])
  81. end
  82. end.compact
  83. 5883 ips unless ips.empty?
  84. end
  85. 27 def generate_id
  86. 1726 id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
  87. end
  88. 27 def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
  89. 800 Resolv::DNS::Message.new(message_id).tap do |query|
  90. 863 query.rd = 1
  91. 863 query.add_question(hostname, type)
  92. 125 end.encode
  93. end
  94. 27 def decode_dns_answer(payload)
  95. 62 begin
  96. 776 message = Resolv::DNS::Message.decode(payload)
  97. rescue Resolv::DNS::DecodeError => e
  98. 6 return :decode_error, e
  99. end
  100. # no domain was found
  101. 770 return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
  102. 262 return :message_truncated if message.tc == 1
  103. 250 return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
  104. 244 addresses = []
  105. 244 message.each_answer do |question, _, value|
  106. 1046 case value
  107. when Resolv::DNS::Resource::IN::CNAME
  108. 18 addresses << {
  109. "name" => question.to_s,
  110. "TTL" => value.ttl,
  111. "alias" => value.name.to_s,
  112. }
  113. when Resolv::DNS::Resource::IN::A,
  114. Resolv::DNS::Resource::IN::AAAA
  115. 1059 addresses << {
  116. 28 "name" => question.to_s,
  117. "TTL" => value.ttl,
  118. "data" => value.address.to_s,
  119. }
  120. end
  121. end
  122. 244 [:ok, addresses]
  123. end
  124. 27 def lookup_synchronize
  125. 13434 @lookup_mutex.synchronize { yield(@lookups) }
  126. end
  127. 27 def id_synchronize(&block)
  128. 863 @identifier_mutex.synchronize(&block)
  129. end
  130. end
  131. end

lib/httpx/resolver/https.rb

86.11% lines covered

144 relevant lines. 124 lines covered and 20 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "uri"
  4. 27 require "forwardable"
  5. 27 require "httpx/base64"
  6. 27 module HTTPX
  7. # Implementation of a DoH name resolver (https://www.youtube.com/watch?v=unMXvnY2FNM).
  8. # It wraps an HTTPX::Connection object which integrates with the main session in the
  9. # same manner as other performed HTTP requests.
  10. #
  11. 27 class Resolver::HTTPS < Resolver::Resolver
  12. 27 extend Forwardable
  13. 27 using URIExtensions
  14. 27 module DNSExtensions
  15. 27 refine Resolv::DNS do
  16. 27 def generate_candidates(name)
  17. 42 @config.generate_candidates(name)
  18. end
  19. end
  20. end
  21. 27 using DNSExtensions
  22. 27 NAMESERVER = "https://1.1.1.1/dns-query"
  23. 2 DEFAULTS = {
  24. 25 uri: NAMESERVER,
  25. use_get: false,
  26. }.freeze
  27. 27 def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?, :handle_socket_timeout
  28. 27 def initialize(_, options)
  29. 90 super
  30. 90 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  31. 90 @queries = {}
  32. 90 @requests = {}
  33. 90 @uri = URI(@resolver_options[:uri])
  34. 90 @uri_addresses = nil
  35. 90 @resolver = Resolv::DNS.new
  36. 90 @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
  37. 90 @resolver.lazy_initialize
  38. end
  39. 27 def <<(connection)
  40. 90 return if @uri.origin == connection.peer.to_s
  41. 48 @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
  42. 48 if @uri_addresses.empty?
  43. 6 ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
  44. 6 ex.set_backtrace(caller)
  45. 6 connection.force_reset
  46. 6 throw(:resolve_error, ex)
  47. end
  48. 42 resolve(connection)
  49. end
  50. 27 def closed?
  51. true
  52. end
  53. 27 def empty?
  54. 84 true
  55. end
  56. 27 def resolver_connection
  57. # TODO: leaks connection object into the pool
  58. 66 @resolver_connection ||= @current_session.find_connection(@uri, @current_selector,
  59. @options.merge(ssl: { alpn_protocols: %w[h2] })).tap do |conn|
  60. 42 emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
  61. end
  62. end
  63. 27 private
  64. 27 def resolve(connection = nil, hostname = nil)
  65. 66 @connections.shift until @connections.empty? || @connections.first.state != :closed
  66. 66 connection ||= @connections.first
  67. 66 return unless connection
  68. 66 hostname ||= @queries.key(connection)
  69. 66 if hostname.nil?
  70. 42 hostname = connection.peer.host
  71. log do
  72. "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
  73. 42 end if connection.peer.non_ascii_hostname
  74. 42 hostname = @resolver.generate_candidates(hostname).each do |name|
  75. 126 @queries[name.to_s] = connection
  76. end.first.to_s
  77. else
  78. 24 @queries[hostname] = connection
  79. end
  80. 66 log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
  81. begin
  82. 66 request = build_request(hostname)
  83. 66 request.on(:response, &method(:on_response).curry(2)[request])
  84. 66 request.on(:promise, &method(:on_promise))
  85. 66 @requests[request] = hostname
  86. 66 resolver_connection.send(request)
  87. 66 @connections << connection
  88. rescue ResolveError, Resolv::DNS::EncodeError => e
  89. reset_hostname(hostname)
  90. emit_resolve_error(connection, connection.peer.host, e)
  91. end
  92. end
  93. 27 def on_response(request, response)
  94. 66 response.raise_for_status
  95. rescue StandardError => e
  96. 6 hostname = @requests.delete(request)
  97. 6 connection = reset_hostname(hostname)
  98. 6 emit_resolve_error(connection, connection.peer.host, e)
  99. else
  100. # @type var response: HTTPX::Response
  101. 60 parse(request, response)
  102. ensure
  103. 66 @requests.delete(request)
  104. end
  105. 27 def on_promise(_, stream)
  106. log(level: 2) { "#{stream.id}: refusing stream!" }
  107. stream.refuse
  108. end
  109. 27 def parse(request, response)
  110. 60 code, result = decode_response_body(response)
  111. 60 case code
  112. when :ok
  113. 18 parse_addresses(result, request)
  114. when :no_domain_found
  115. # Indicates no such domain was found.
  116. 36 host = @requests.delete(request)
  117. 36 connection = reset_hostname(host, reset_candidates: false)
  118. 36 unless @queries.value?(connection)
  119. 12 emit_resolve_error(connection)
  120. 12 return
  121. end
  122. 24 resolve
  123. when :dns_error
  124. host = @requests.delete(request)
  125. connection = reset_hostname(host)
  126. emit_resolve_error(connection)
  127. when :decode_error
  128. 6 host = @requests.delete(request)
  129. 6 connection = reset_hostname(host)
  130. 6 emit_resolve_error(connection, connection.peer.host, result)
  131. end
  132. end
  133. 27 def parse_addresses(answers, request)
  134. 18 if answers.empty?
  135. # no address found, eliminate candidates
  136. host = @requests.delete(request)
  137. connection = reset_hostname(host)
  138. emit_resolve_error(connection)
  139. return
  140. else
  141. 42 answers = answers.group_by { |answer| answer["name"] }
  142. 18 answers.each do |hostname, addresses|
  143. 24 addresses = addresses.flat_map do |address|
  144. 24 if address.key?("alias")
  145. 6 alias_address = answers[address["alias"]]
  146. 6 if alias_address.nil?
  147. reset_hostname(address["name"])
  148. if early_resolve(connection, hostname: address["alias"])
  149. @connections.delete(connection)
  150. else
  151. resolve(connection, address["alias"])
  152. return # rubocop:disable Lint/NonLocalExitFromIterator
  153. end
  154. else
  155. 6 alias_address
  156. end
  157. else
  158. 18 address
  159. end
  160. end.compact
  161. 24 next if addresses.empty?
  162. 24 hostname.delete_suffix!(".") if hostname.end_with?(".")
  163. 24 connection = reset_hostname(hostname, reset_candidates: false)
  164. 24 next unless connection # probably a retried query for which there's an answer
  165. 18 @connections.delete(connection)
  166. # eliminate other candidates
  167. 54 @queries.delete_if { |_, conn| connection == conn }
  168. 18 Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
  169. 54 catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
  170. end
  171. end
  172. 18 return if @connections.empty?
  173. resolve
  174. end
  175. 27 def build_request(hostname)
  176. 60 uri = @uri.dup
  177. 60 rklass = @options.request_class
  178. 60 payload = Resolver.encode_dns_query(hostname, type: @record_type)
  179. 60 if @resolver_options[:use_get]
  180. 6 params = URI.decode_www_form(uri.query.to_s)
  181. 6 params << ["type", FAMILY_TYPES[@record_type]]
  182. 6 params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
  183. 6 uri.query = URI.encode_www_form(params)
  184. 6 request = rklass.new("GET", uri, @options)
  185. else
  186. 54 request = rklass.new("POST", uri, @options, body: [payload])
  187. 54 request.headers["content-type"] = "application/dns-message"
  188. end
  189. 60 request.headers["accept"] = "application/dns-message"
  190. 60 request
  191. end
  192. 27 def decode_response_body(response)
  193. 54 case response.headers["content-type"]
  194. when "application/dns-udpwireformat",
  195. "application/dns-message"
  196. 54 Resolver.decode_dns_answer(response.to_s)
  197. else
  198. raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
  199. end
  200. end
  201. 27 def reset_hostname(hostname, reset_candidates: true)
  202. 72 connection = @queries.delete(hostname)
  203. 72 return connection unless connection && reset_candidates
  204. # eliminate other candidates
  205. 36 candidates = @queries.select { |_, conn| connection == conn }.keys
  206. 36 @queries.delete_if { |h, _| candidates.include?(h) }
  207. 12 connection
  208. end
  209. end
  210. end

lib/httpx/resolver/multi.rb

88.24% lines covered

51 relevant lines. 45 lines covered and 6 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 require "resolv"
  4. 27 module HTTPX
  5. 27 class Resolver::Multi
  6. 27 include Callbacks
  7. 27 using ArrayExtensions::FilterMap
  8. 27 attr_reader :resolvers, :options
  9. 27 def initialize(resolver_type, options)
  10. 7122 @current_selector = nil
  11. 7122 @current_session = nil
  12. 7122 @options = options
  13. 7122 @resolver_options = @options.resolver_options
  14. 7122 @resolvers = options.ip_families.map do |ip_family|
  15. 7122 resolver = resolver_type.new(ip_family, options)
  16. 7122 resolver.multi = self
  17. 7122 resolver
  18. end
  19. 7122 @errors = Hash.new { |hs, k| hs[k] = [] }
  20. end
  21. 27 def current_selector=(s)
  22. 7150 @current_selector = s
  23. 14300 @resolvers.each { |r| r.__send__(__method__, s) }
  24. end
  25. 27 def current_session=(s)
  26. 7150 @current_session = s
  27. 14300 @resolvers.each { |r| r.__send__(__method__, s) }
  28. end
  29. 27 def closed?
  30. @resolvers.all?(&:closed?)
  31. end
  32. 27 def empty?
  33. @resolvers.all?(&:empty?)
  34. end
  35. 27 def inflight?
  36. @resolvers.any(&:inflight?)
  37. end
  38. 27 def timeout
  39. @resolvers.filter_map(&:timeout).min
  40. end
  41. 27 def close
  42. @resolvers.each(&:close)
  43. end
  44. 27 def connections
  45. @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
  46. end
  47. 27 def early_resolve(connection)
  48. 7152 hostname = connection.peer.host
  49. 7152 addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  50. 7152 return false unless addresses
  51. 6688 resolved = false
  52. 6962 addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
  53. # try to match the resolver by family. However, there are cases where that's not possible, as when
  54. # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
  55. 13892 resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
  56. 6946 next unless resolver # this should ever happen
  57. # it does not matter which resolver it is, as early-resolve code is shared.
  58. 6946 resolver.emit_addresses(connection, family, addrs, true)
  59. 6907 resolved = true
  60. end
  61. 6649 resolved
  62. end
  63. 27 def lazy_resolve(connection)
  64. 464 @resolvers.each do |resolver|
  65. 464 resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
  66. 452 next if resolver.empty?
  67. 368 @current_session.select_resolver(resolver, @current_selector)
  68. end
  69. end
  70. end
  71. end

lib/httpx/resolver/native.rb

89.0% lines covered

309 relevant lines. 275 lines covered and 34 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 require "resolv"
  4. 27 module HTTPX
  5. # Implements a pure ruby name resolver, which abides by the Selectable API.
  6. # It delegates DNS payload encoding/decoding to the +resolv+ stlid gem.
  7. #
  8. 27 class Resolver::Native < Resolver::Resolver
  9. 27 extend Forwardable
  10. 27 using URIExtensions
  11. 17 DEFAULTS = {
  12. 10 nameserver: nil,
  13. **Resolv::DNS::Config.default_config_hash,
  14. packet_size: 512,
  15. timeouts: Resolver::RESOLVE_TIMEOUT,
  16. }.freeze
  17. 27 DNS_PORT = 53
  18. 27 def_delegator :@connections, :empty?
  19. 27 attr_reader :state
  20. 27 def initialize(family, options)
  21. 7032 super
  22. 7032 @ns_index = 0
  23. 7032 @resolver_options = DEFAULTS.merge(@options.resolver_options)
  24. 7032 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  25. 7032 @nameserver = if (nameserver = @resolver_options[:nameserver])
  26. 7026 nameserver = nameserver[family] if nameserver.is_a?(Hash)
  27. 7026 Array(nameserver)
  28. end
  29. 7032 @ndots = @resolver_options.fetch(:ndots, 1)
  30. 21096 @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
  31. 7032 @_timeouts = Array(@resolver_options[:timeouts])
  32. 8448 @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
  33. 7032 @name = nil
  34. 7032 @queries = {}
  35. 7032 @read_buffer = "".b
  36. 7032 @write_buffer = Buffer.new(@resolver_options[:packet_size])
  37. 7032 @state = :idle
  38. end
  39. 27 def close
  40. 366 transition(:closed)
  41. end
  42. 27 def terminate
  43. 16 emit(:close, self)
  44. end
  45. 27 def closed?
  46. 766 @state == :closed
  47. end
  48. 27 def to_io
  49. 1126 @io.to_io
  50. end
  51. 27 def call
  52. 928 case @state
  53. when :open
  54. 993 consume
  55. end
  56. end
  57. 27 def interests
  58. 12957 case @state
  59. when :idle
  60. 12311 transition(:open)
  61. when :closed
  62. 12 transition(:idle)
  63. 12 transition(:open)
  64. end
  65. 13029 calculate_interests
  66. end
  67. 27 def <<(connection)
  68. 374 if @nameserver.nil?
  69. 6 ex = ResolveError.new("No available nameserver")
  70. 6 ex.set_backtrace(caller)
  71. 6 connection.force_reset
  72. 6 throw(:resolve_error, ex)
  73. else
  74. 368 @connections << connection
  75. 368 resolve
  76. end
  77. end
  78. 27 def timeout
  79. 13029 return if @connections.empty?
  80. 13029 @start_timeout = Utils.now
  81. 13029 hosts = @queries.keys
  82. 13029 @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
  83. end
  84. 27 def handle_socket_timeout(interval); end
  85. 27 private
  86. 27 def calculate_interests
  87. 17396 return :w unless @write_buffer.empty?
  88. 15669 return :r unless @queries.empty?
  89. 237 nil
  90. end
  91. 27 def consume
  92. 1005 loop do
  93. 1802 dread if calculate_interests == :r
  94. 1768 break unless calculate_interests == :w
  95. # do_retry
  96. 815 dwrite
  97. 797 break unless calculate_interests == :r
  98. end
  99. rescue Errno::EHOSTUNREACH => e
  100. 18 @ns_index += 1
  101. 18 nameserver = @nameserver
  102. 18 if nameserver && @ns_index < nameserver.size
  103. 12 log { "resolver #{FAMILY_TYPES[@record_type]}: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
  104. 12 transition(:idle)
  105. 12 @timeouts.clear
  106. 12 retry
  107. else
  108. 6 handle_error(e)
  109. 6 emit(:close, self)
  110. end
  111. rescue NativeResolveError => e
  112. 18 handle_error(e)
  113. 18 close_or_resolve
  114. 18 retry unless closed?
  115. end
  116. 27 def schedule_retry
  117. 797 h = @name
  118. 797 return unless h
  119. 797 connection = @queries[h]
  120. 797 timeouts = @timeouts[h]
  121. 797 timeout = timeouts.shift
  122. 797 @timer = @current_selector.after(timeout) do
  123. 75 next unless @connections.include?(connection)
  124. 75 do_retry(h, connection, timeout)
  125. end
  126. end
  127. 27 def do_retry(h, connection, interval)
  128. 75 timeouts = @timeouts[h]
  129. 75 if !timeouts.empty?
  130. 45 log { "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..." }
  131. # must downgrade to tcp AND retry on same host as last
  132. 45 downgrade_socket
  133. 45 resolve(connection, h)
  134. 30 elsif @ns_index + 1 < @nameserver.size
  135. # try on the next nameserver
  136. 6 @ns_index += 1
  137. 6 log do
  138. "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
  139. end
  140. 6 transition(:idle)
  141. 6 @timeouts.clear
  142. 6 resolve(connection, h)
  143. else
  144. 24 @timeouts.delete(h)
  145. 24 reset_hostname(h, reset_candidates: false)
  146. 24 unless @queries.empty?
  147. 18 resolve(connection)
  148. 18 return
  149. end
  150. 6 @connections.delete(connection)
  151. 6 host = connection.peer.host
  152. # This loop_time passed to the exception is bogus. Ideally we would pass the total
  153. # resolve timeout, including from the previous retries.
  154. 6 ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
  155. 6 ex.set_backtrace(ex ? ex.backtrace : caller)
  156. 6 emit_resolve_error(connection, host, ex)
  157. 6 close_or_resolve
  158. end
  159. end
  160. 27 def dread(wsize = @resolver_options[:packet_size])
  161. 1343 loop do
  162. 1355 wsize = @large_packet.capacity if @large_packet
  163. 1355 siz = @io.read(wsize, @read_buffer)
  164. 1355 unless siz
  165. ex = EOFError.new("descriptor closed")
  166. ex.set_backtrace(caller)
  167. raise ex
  168. end
  169. 1355 return unless siz.positive?
  170. 734 if @socket_type == :tcp
  171. # packet may be incomplete, need to keep draining from the socket
  172. 30 if @large_packet
  173. # large packet buffer already exists, continue pumping
  174. 12 @large_packet << @read_buffer
  175. 12 next unless @large_packet.full?
  176. 12 parse(@large_packet.to_s)
  177. 12 @large_packet = nil
  178. # downgrade to udp again
  179. 12 downgrade_socket
  180. 12 return
  181. else
  182. 18 size = @read_buffer[0, 2].unpack1("n")
  183. 18 buffer = @read_buffer.byteslice(2..-1)
  184. 18 if size > @read_buffer.bytesize
  185. # only do buffer logic if it's worth it, and the whole packet isn't here already
  186. 12 @large_packet = Buffer.new(size)
  187. 12 @large_packet << buffer
  188. 12 next
  189. else
  190. 6 parse(buffer)
  191. end
  192. end
  193. else # udp
  194. 704 parse(@read_buffer)
  195. end
  196. 676 return if @state == :closed || !@write_buffer.empty?
  197. end
  198. end
  199. 27 def dwrite
  200. 797 loop do
  201. 1594 return if @write_buffer.empty?
  202. 797 siz = @io.write(@write_buffer)
  203. 797 unless siz
  204. ex = EOFError.new("descriptor closed")
  205. ex.set_backtrace(caller)
  206. raise ex
  207. end
  208. 797 return unless siz.positive?
  209. 797 schedule_retry if @write_buffer.empty?
  210. 797 return if @state == :closed
  211. end
  212. end
  213. 27 def parse(buffer)
  214. 722 @timer.cancel
  215. 722 code, result = Resolver.decode_dns_answer(buffer)
  216. 657 case code
  217. when :ok
  218. 226 parse_addresses(result)
  219. when :no_domain_found
  220. # Indicates no such domain was found.
  221. 472 hostname, connection = @queries.first
  222. 472 reset_hostname(hostname, reset_candidates: false)
  223. 826 other_candidate, _ = @queries.find { |_, conn| conn == connection }
  224. 472 if other_candidate
  225. 354 resolve(connection, other_candidate)
  226. else
  227. 118 @connections.delete(connection)
  228. 118 ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
  229. 118 ex.set_backtrace(ex ? ex.backtrace : caller)
  230. 118 emit_resolve_error(connection, connection.peer.host, ex)
  231. 102 close_or_resolve
  232. end
  233. when :message_truncated
  234. # TODO: what to do if it's already tcp??
  235. 12 return if @socket_type == :tcp
  236. 12 @socket_type = :tcp
  237. 12 hostname, _ = @queries.first
  238. 12 reset_hostname(hostname)
  239. 12 transition(:closed)
  240. when :dns_error
  241. 6 hostname, connection = @queries.first
  242. 6 reset_hostname(hostname)
  243. 6 @connections.delete(connection)
  244. 6 ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
  245. 6 raise ex
  246. when :decode_error
  247. 6 hostname, connection = @queries.first
  248. 6 reset_hostname(hostname)
  249. 6 @connections.delete(connection)
  250. 6 ex = NativeResolveError.new(connection, connection.peer.host, result.message)
  251. 6 ex.set_backtrace(result.backtrace)
  252. 6 raise ex
  253. end
  254. end
  255. 27 def parse_addresses(addresses)
  256. 226 if addresses.empty?
  257. # no address found, eliminate candidates
  258. 6 hostname, connection = @queries.first
  259. 6 reset_hostname(hostname)
  260. 6 @connections.delete(connection)
  261. 6 raise NativeResolveError.new(connection, connection.peer.host)
  262. else
  263. 220 address = addresses.first
  264. 220 name = address["name"]
  265. 220 connection = @queries.delete(name)
  266. 220 unless connection
  267. orig_name = name
  268. # absolute name
  269. name_labels = Resolv::DNS::Name.create(name).to_a
  270. name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
  271. # probably a retried query for which there's an answer
  272. unless name
  273. @timeouts.delete(orig_name)
  274. return
  275. end
  276. address["name"] = name
  277. connection = @queries.delete(name)
  278. end
  279. 1279 alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
  280. 220 if addresses.empty? && !alias_addresses.empty? # CNAME
  281. hostname_alias = alias_addresses.first["alias"]
  282. # clean up intermediate queries
  283. @timeouts.delete(name) unless connection.peer.host == name
  284. if early_resolve(connection, hostname: hostname_alias)
  285. @connections.delete(connection)
  286. else
  287. if @socket_type == :tcp
  288. # must downgrade to udp if tcp
  289. @socket_type = @resolver_options.fetch(:socket_type, :udp)
  290. transition(:idle)
  291. transition(:open)
  292. end
  293. log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
  294. resolve(connection, hostname_alias)
  295. return
  296. end
  297. else
  298. 220 reset_hostname(name, connection: connection)
  299. 220 @timeouts.delete(connection.peer.host)
  300. 220 @connections.delete(connection)
  301. 220 Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
  302. 1481 catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
  303. end
  304. end
  305. 220 close_or_resolve
  306. end
  307. 27 def resolve(connection = nil, hostname = nil)
  308. 805 @connections.shift until @connections.empty? || @connections.first.state != :closed
  309. 1189 connection ||= @connections.find { |c| !@queries.value?(c) }
  310. 805 raise Error, "no URI to resolve" unless connection
  311. 805 return unless @write_buffer.empty?
  312. 803 hostname ||= @queries.key(connection)
  313. 803 if hostname.nil?
  314. 380 hostname = connection.peer.host
  315. 380 if connection.peer.non_ascii_hostname
  316. log { "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" }
  317. end
  318. 380 hostname = generate_candidates(hostname).each do |name|
  319. 1392 @queries[name] = connection
  320. end.first
  321. else
  322. 389 @queries[hostname] = connection
  323. end
  324. 803 @name = hostname
  325. 803 log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
  326. 62 begin
  327. 803 @write_buffer << encode_dns_query(hostname)
  328. rescue Resolv::DNS::EncodeError => e
  329. reset_hostname(hostname, connection: connection)
  330. @connections.delete(connection)
  331. emit_resolve_error(connection, hostname, e)
  332. close_or_resolve
  333. end
  334. end
  335. 27 def encode_dns_query(hostname)
  336. 803 message_id = Resolver.generate_id
  337. 803 msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
  338. 803 msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
  339. 803 msg
  340. end
  341. 27 def generate_candidates(name)
  342. 380 return [name] if name.end_with?(".")
  343. 380 candidates = []
  344. 380 name_parts = name.scan(/[^.]+/)
  345. 380 candidates = [name] if @ndots <= name_parts.size - 1
  346. 1140 candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
  347. 380 fname = "#{name}."
  348. 380 candidates << fname unless candidates.include?(fname)
  349. 380 candidates
  350. end
  351. 27 def build_socket
  352. 396 ip, port = @nameserver[@ns_index]
  353. 396 port ||= DNS_PORT
  354. 364 case @socket_type
  355. when :udp
  356. 378 log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
  357. 378 UDP.new(ip, port, @options)
  358. when :tcp
  359. 18 log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
  360. 18 origin = URI("tcp://#{ip}:#{port}")
  361. 18 TCP.new(origin, [ip], @options)
  362. end
  363. end
  364. 27 def downgrade_socket
  365. 57 return unless @socket_type == :tcp
  366. 12 @socket_type = @resolver_options.fetch(:socket_type, :udp)
  367. 12 transition(:idle)
  368. 12 transition(:open)
  369. end
  370. 27 def transition(nextstate)
  371. 12691 case nextstate
  372. when :idle
  373. 42 if @io
  374. 36 @io.close
  375. 36 @io = nil
  376. end
  377. when :open
  378. 12335 return unless @state == :idle
  379. 12335 @io ||= build_socket
  380. 12335 @io.connect
  381. 12335 return unless @io.connected?
  382. 396 resolve if @queries.empty? && !@connections.empty?
  383. when :closed
  384. 378 return unless @state == :open
  385. 372 @io.close if @io
  386. 372 @start_timeout = nil
  387. 372 @write_buffer.clear
  388. 372 @read_buffer.clear
  389. end
  390. 810 @state = nextstate
  391. rescue Errno::ECONNREFUSED,
  392. Errno::EADDRNOTAVAIL,
  393. Errno::EHOSTUNREACH,
  394. SocketError,
  395. IOError,
  396. ConnectTimeoutError => e
  397. # these errors may happen during TCP handshake
  398. # treat them as resolve errors.
  399. handle_error(e)
  400. emit(:close, self)
  401. end
  402. 27 def handle_error(error)
  403. 24 if error.respond_to?(:connection) &&
  404. error.respond_to?(:host)
  405. 18 reset_hostname(error.host, connection: error.connection)
  406. 18 @connections.delete(error.connection)
  407. 18 emit_resolve_error(error.connection, error.host, error)
  408. else
  409. 6 @queries.each do |host, connection|
  410. 6 reset_hostname(host, connection: connection)
  411. 6 @connections.delete(connection)
  412. 6 emit_resolve_error(connection, host, error)
  413. end
  414. 12 while (connection = @connections.shift)
  415. emit_resolve_error(connection, connection.peer.host, error)
  416. end
  417. end
  418. end
  419. 27 def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
  420. 770 @timeouts.delete(hostname)
  421. 770 return unless connection && reset_candidates
  422. # eliminate other candidates
  423. 1048 candidates = @queries.select { |_, conn| connection == conn }.keys
  424. 1048 @queries.delete_if { |h, _| candidates.include?(h) }
  425. # reset timeouts
  426. 1024 @timeouts.delete_if { |h, _| candidates.include?(h) }
  427. end
  428. 27 def close_or_resolve
  429. # drop already closed connections
  430. 346 @connections.shift until @connections.empty? || @connections.first.state != :closed
  431. 346 if (@connections - @queries.values).empty?
  432. 344 emit(:close, self)
  433. else
  434. 2 resolve
  435. end
  436. end
  437. end
  438. end

lib/httpx/resolver/resolver.rb

84.52% lines covered

84 relevant lines. 71 lines covered and 13 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 require "ipaddr"
  4. 27 module HTTPX
  5. # Base class for all internal internet name resolvers. It handles basic blocks
  6. # from the Selectable API.
  7. #
  8. 27 class Resolver::Resolver
  9. 27 include Callbacks
  10. 27 include Loggable
  11. 27 using ArrayExtensions::Intersect
  12. 2 RECORD_TYPES = {
  13. 25 Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
  14. Socket::AF_INET => Resolv::DNS::Resource::IN::A,
  15. }.freeze
  16. 2 FAMILY_TYPES = {
  17. 25 Resolv::DNS::Resource::IN::AAAA => "AAAA",
  18. Resolv::DNS::Resource::IN::A => "A",
  19. }.freeze
  20. 27 class << self
  21. 27 def multi?
  22. 7122 true
  23. end
  24. end
  25. 27 attr_reader :family, :options
  26. 27 attr_writer :current_selector, :current_session
  27. 27 attr_accessor :multi
  28. 27 def initialize(family, options)
  29. 7147 @family = family
  30. 7147 @record_type = RECORD_TYPES[family]
  31. 7147 @options = options
  32. 7147 @connections = []
  33. 7147 set_resolver_callbacks
  34. end
  35. 27 def each_connection(&block)
  36. 236 enum_for(__method__) unless block
  37. 236 return unless @connections
  38. 236 @connections.each(&block)
  39. end
  40. 27 def close; end
  41. 27 alias_method :terminate, :close
  42. 27 def closed?
  43. true
  44. end
  45. 27 def empty?
  46. true
  47. end
  48. 27 def inflight?
  49. 16 false
  50. end
  51. 27 def emit_addresses(connection, family, addresses, early_resolve = false)
  52. 7238 addresses.map! do |address|
  53. 19933 address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
  54. end
  55. # double emission check, but allow early resolution to work
  56. 7238 return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
  57. 7238 log do
  58. 72 "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
  59. 6 "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
  60. end
  61. 7238 if !early_resolve && # do not apply resolution delay for non-dns name resolution
  62. @current_selector && # just in case...
  63. family == Socket::AF_INET && # resolution delay only applies to IPv4
  64. !connection.io && # connection already has addresses and initiated/ended handshake
  65. connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
  66. addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
  67. log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
  68. @current_selector.after(0.05) do
  69. # double emission check
  70. unless connection.addresses && addresses.intersect?(connection.addresses)
  71. emit_resolved_connection(connection, addresses, early_resolve)
  72. end
  73. end
  74. else
  75. 7238 emit_resolved_connection(connection, addresses, early_resolve)
  76. end
  77. end
  78. 27 private
  79. 27 def emit_resolved_connection(connection, addresses, early_resolve)
  80. begin
  81. 7238 connection.addresses = addresses
  82. 7192 return if connection.state == :closed
  83. 7192 emit(:resolve, connection)
  84. 24 rescue StandardError => e
  85. 46 if early_resolve
  86. 39 connection.force_reset
  87. 39 throw(:resolve_error, e)
  88. else
  89. 7 emit(:error, connection, e)
  90. end
  91. end
  92. end
  93. 27 def early_resolve(connection, hostname: connection.peer.host)
  94. addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
  95. return false unless addresses
  96. addresses = addresses.select { |addr| addr.family == @family }
  97. return false if addresses.empty?
  98. emit_addresses(connection, @family, addresses, true)
  99. true
  100. end
  101. 27 def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
  102. 185 emit_connection_error(connection, resolve_error(hostname, ex))
  103. end
  104. 27 def resolve_error(hostname, ex = nil)
  105. 185 return ex if ex.is_a?(ResolveError) || ex.is_a?(ResolveTimeoutError)
  106. 42 message = ex ? ex.message : "Can't resolve #{hostname}"
  107. 42 error = ResolveError.new(message)
  108. 42 error.set_backtrace(ex ? ex.backtrace : caller)
  109. 42 error
  110. end
  111. 27 def set_resolver_callbacks
  112. 7147 on(:resolve, &method(:resolve_connection))
  113. 7147 on(:error, &method(:emit_connection_error))
  114. 7147 on(:close, &method(:close_resolver))
  115. end
  116. 27 def resolve_connection(connection)
  117. 7192 @current_session.__send__(:on_resolver_connection, connection, @current_selector)
  118. end
  119. 27 def emit_connection_error(connection, error)
  120. 179 return connection.handle_connect_error(error) if connection.connecting?
  121. connection.emit(:error, error)
  122. end
  123. 27 def close_resolver(resolver)
  124. 366 @current_session.__send__(:on_resolver_close, resolver, @current_selector)
  125. end
  126. end
  127. end

lib/httpx/resolver/system.rb

78.99% lines covered

138 relevant lines. 109 lines covered and 29 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "resolv"
  3. 27 module HTTPX
  4. # Implementation of a synchronous name resolver which relies on the system resolver,
  5. # which is lib'c getaddrinfo function (abstracted in ruby via Addrinfo.getaddrinfo).
  6. #
  7. # Its main advantage is relying on the reference implementation for name resolution
  8. # across most/all OSs which deploy ruby (it's what TCPSocket also uses), its main
  9. # disadvantage is the inability to set timeouts / check socket for readiness events,
  10. # hence why it relies on using the Timeout module, which poses a lot of problems for
  11. # the selector loop, specially when network is unstable.
  12. #
  13. 27 class Resolver::System < Resolver::Resolver
  14. 27 using URIExtensions
  15. 27 RESOLV_ERRORS = [Resolv::ResolvError,
  16. Resolv::DNS::Requester::RequestError,
  17. Resolv::DNS::EncodeError,
  18. Resolv::DNS::DecodeError].freeze
  19. 27 DONE = 1
  20. 27 ERROR = 2
  21. 27 class << self
  22. 27 def multi?
  23. 25 false
  24. end
  25. end
  26. 27 attr_reader :state
  27. 27 def initialize(options)
  28. 25 super(0, options)
  29. 25 @resolver_options = @options.resolver_options
  30. 25 resolv_options = @resolver_options.dup
  31. 25 timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
  32. 25 @_timeouts = Array(timeouts)
  33. 50 @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
  34. 25 resolv_options.delete(:cache)
  35. 25 @queries = []
  36. 25 @ips = []
  37. 25 @pipe_mutex = Thread::Mutex.new
  38. 25 @state = :idle
  39. end
  40. 27 def resolvers
  41. return enum_for(__method__) unless block_given?
  42. yield self
  43. end
  44. 27 def multi
  45. self
  46. end
  47. 27 def empty?
  48. true
  49. end
  50. 27 def close
  51. transition(:closed)
  52. end
  53. 27 def closed?
  54. @state == :closed
  55. end
  56. 27 def to_io
  57. @pipe_read.to_io
  58. end
  59. 27 def call
  60. case @state
  61. when :open
  62. consume
  63. end
  64. nil
  65. end
  66. 27 def interests
  67. return if @queries.empty?
  68. :r
  69. end
  70. 27 def timeout
  71. return unless @queries.empty?
  72. _, connection = @queries.first
  73. return unless connection
  74. @timeouts[connection.peer.host].first
  75. end
  76. 27 def <<(connection)
  77. 25 @connections << connection
  78. 25 resolve
  79. end
  80. 27 def early_resolve(connection, **)
  81. 25 self << connection
  82. 12 true
  83. end
  84. 27 def handle_socket_timeout(interval)
  85. error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
  86. error.set_backtrace(caller)
  87. @queries.each do |host, connection|
  88. @connections.delete(connection)
  89. emit_resolve_error(connection, host, error)
  90. end
  91. while (connection = @connections.shift)
  92. emit_resolve_error(connection, connection.peer.host, error)
  93. end
  94. end
  95. 27 private
  96. 27 def transition(nextstate)
  97. 25 case nextstate
  98. when :idle
  99. @timeouts.clear
  100. when :open
  101. 25 return unless @state == :idle
  102. 25 @pipe_read, @pipe_write = ::IO.pipe
  103. when :closed
  104. return unless @state == :open
  105. @pipe_write.close
  106. @pipe_read.close
  107. end
  108. 25 @state = nextstate
  109. end
  110. 27 def consume
  111. 25 return if @connections.empty?
  112. 25 if @pipe_read.wait_readable
  113. 25 event = @pipe_read.getbyte
  114. 25 case event
  115. when DONE
  116. 24 *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
  117. 12 if pair
  118. 12 @queries.delete(pair)
  119. 12 family, connection = pair
  120. 12 @connections.delete(connection)
  121. 24 catch(:coalesced) { emit_addresses(connection, family, addrs) }
  122. end
  123. when ERROR
  124. 26 *pair, error = @pipe_mutex.synchronize { @ips.pop }
  125. 13 if pair && error
  126. 13 @queries.delete(pair)
  127. 13 @connections.delete(connection)
  128. 13 _, connection = pair
  129. 13 emit_resolve_error(connection, connection.peer.host, error)
  130. end
  131. end
  132. end
  133. 12 return emit(:close, self) if @connections.empty?
  134. resolve
  135. end
  136. 27 def resolve(connection = nil, hostname = nil)
  137. 25 @connections.shift until @connections.empty? || @connections.first.state != :closed
  138. 25 connection ||= @connections.first
  139. 25 raise Error, "no URI to resolve" unless connection
  140. 25 return unless @queries.empty?
  141. 25 hostname ||= connection.peer.host
  142. 25 scheme = connection.origin.scheme
  143. log do
  144. "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
  145. 25 end if connection.peer.non_ascii_hostname
  146. 25 transition(:open)
  147. 25 connection.options.ip_families.each do |family|
  148. 25 @queries << [family, connection]
  149. end
  150. 25 async_resolve(connection, hostname, scheme)
  151. 25 consume
  152. end
  153. 27 def async_resolve(connection, hostname, scheme)
  154. 25 families = connection.options.ip_families
  155. 25 log { "resolver: query for #{hostname}" }
  156. 25 timeouts = @timeouts[connection.peer.host]
  157. 25 resolve_timeout = timeouts.first
  158. 25 Thread.start do
  159. 25 Thread.current.report_on_exception = false
  160. begin
  161. 25 addrs = if resolve_timeout
  162. 25 Timeout.timeout(resolve_timeout) do
  163. 25 __addrinfo_resolve(hostname, scheme)
  164. end
  165. else
  166. __addrinfo_resolve(hostname, scheme)
  167. end
  168. 12 addrs = addrs.sort_by(&:afamily).group_by(&:afamily)
  169. 12 families.each do |family|
  170. 12 addresses = addrs[family]
  171. 12 next unless addresses
  172. 12 addresses.map!(&:ip_address)
  173. 12 addresses.uniq!
  174. 12 @pipe_mutex.synchronize do
  175. 12 @ips.unshift([family, connection, addresses])
  176. 12 @pipe_write.putc(DONE) unless @pipe_write.closed?
  177. end
  178. end
  179. rescue StandardError => e
  180. 13 if e.is_a?(Timeout::Error)
  181. 1 timeouts.shift
  182. 1 retry unless timeouts.empty?
  183. 1 e = ResolveTimeoutError.new(resolve_timeout, e.message)
  184. 1 e.set_backtrace(e.backtrace)
  185. end
  186. 13 @pipe_mutex.synchronize do
  187. 13 families.each do |family|
  188. 13 @ips.unshift([family, connection, e])
  189. 13 @pipe_write.putc(ERROR) unless @pipe_write.closed?
  190. end
  191. end
  192. end
  193. end
  194. end
  195. 27 def __addrinfo_resolve(host, scheme)
  196. 25 Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
  197. end
  198. 27 def emit_connection_error(_, error)
  199. 13 throw(:resolve_error, error)
  200. end
  201. 27 def close_resolver(resolver); end
  202. end
  203. end

lib/httpx/response.rb

100.0% lines covered

115 relevant lines. 115 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "objspace"
  3. 27 require "stringio"
  4. 27 require "tempfile"
  5. 27 require "fileutils"
  6. 27 require "forwardable"
  7. 27 module HTTPX
  8. # Defines a HTTP response is handled internally, with a few properties exposed as attributes.
  9. #
  10. # It delegates the following methods to the corresponding HTTPX::Request:
  11. #
  12. # * HTTPX::Request#uri
  13. # * HTTPX::Request#peer_address
  14. #
  15. # It implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads.
  16. #
  17. # It implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
  18. # (of payload chunks).
  19. #
  20. 27 class Response
  21. 27 extend Forwardable
  22. 27 include Callbacks
  23. # the HTTP response status code
  24. 27 attr_reader :status
  25. # an HTTPX::Headers object containing the response HTTP headers.
  26. 27 attr_reader :headers
  27. # a HTTPX::Response::Body object wrapping the response body. The following methods are delegated to it:
  28. #
  29. # * HTTPX::Response::Body#to_s
  30. # * HTTPX::Response::Body#to_str
  31. # * HTTPX::Response::Body#read
  32. # * HTTPX::Response::Body#copy_to
  33. # * HTTPX::Response::Body#close
  34. 27 attr_reader :body
  35. # The HTTP protocol version used to fetch the response.
  36. 27 attr_reader :version
  37. # returns the response body buffered in a string.
  38. 27 def_delegator :@body, :to_s
  39. 27 def_delegator :@body, :to_str
  40. # implements the IO reader +#read+ interface.
  41. 27 def_delegator :@body, :read
  42. # copies the response body to a different location.
  43. 27 def_delegator :@body, :copy_to
  44. # the corresponding request uri.
  45. 27 def_delegator :@request, :uri
  46. # the IP address of the peer server.
  47. 27 def_delegator :@request, :peer_address
  48. # inits the instance with the corresponding +request+ to this response, an the
  49. # response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
  50. 27 def initialize(request, status, version, headers)
  51. 9332 @request = request
  52. 9332 @options = request.options
  53. 9332 @version = version
  54. 9332 @status = Integer(status)
  55. 9332 @headers = @options.headers_class.new(headers)
  56. 9332 @body = @options.response_body_class.new(self, @options)
  57. 9332 @finished = complete?
  58. 9332 @content_type = nil
  59. end
  60. # dupped initialization
  61. 27 def initialize_dup(orig)
  62. 64 super
  63. # if a response gets dupped, the body handle must also get dupped to prevent
  64. # two responses from using the same file handle to read.
  65. 64 @body = orig.body.dup
  66. end
  67. # closes the respective +@request+ and +@body+.
  68. 27 def close
  69. 457 @request.close
  70. 457 @body.close
  71. end
  72. # merges headers defined in +h+ into the response headers.
  73. 27 def merge_headers(h)
  74. 193 @headers = @headers.merge(h)
  75. end
  76. # writes +data+ chunk into the response body.
  77. 27 def <<(data)
  78. 12339 @body.write(data)
  79. end
  80. # returns the HTTPX::ContentType for the response, as per what's declared in the content-type header.
  81. #
  82. # response.content_type #=> #<HTTPX::ContentType:xxx @header_value="text/plain">
  83. # response.content_type.mime_type #=> "text/plain"
  84. 27 def content_type
  85. 9701 @content_type ||= ContentType.new(@headers["content-type"])
  86. end
  87. # returns whether the response has been fully fetched.
  88. 27 def finished?
  89. 14526 @finished
  90. end
  91. # marks the response as finished, freezes the headers.
  92. 27 def finish!
  93. 8261 @finished = true
  94. 8261 @headers.freeze
  95. end
  96. # returns whether the response contains body payload.
  97. 27 def bodyless?
  98. 9332 @request.verb == "HEAD" ||
  99. @status < 200 || # informational response
  100. @status == 204 ||
  101. @status == 205 ||
  102. @status == 304 || begin
  103. 8886 content_length = @headers["content-length"]
  104. 8886 return false if content_length.nil?
  105. 7529 content_length == "0"
  106. end
  107. end
  108. 27 def complete?
  109. 9332 bodyless? || (@request.verb == "CONNECT" && @status == 200)
  110. end
  111. skipped # :nocov:
  112. skipped def inspect
  113. skipped "#<#{self.class}:#{object_id} " \
  114. skipped "HTTP/#{version} " \
  115. skipped "@status=#{@status} " \
  116. skipped "@headers=#{@headers} " \
  117. skipped "@body=#{@body.bytesize}>"
  118. skipped end
  119. skipped # :nocov:
  120. # returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
  121. # status code, or nothing.
  122. #
  123. # ok_response.error #=> nil
  124. # not_found_response.error #=> HTTPX::HTTPError instance, status 404
  125. 27 def error
  126. 578 return if @status < 400
  127. 54 HTTPError.new(self)
  128. end
  129. # it raises the exception returned by +error+, or itself otherwise.
  130. #
  131. # ok_response.raise_for_status #=> ok_response
  132. # not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
  133. 27 def raise_for_status
  134. 538 return self unless (err = error)
  135. 38 raise err
  136. end
  137. # decodes the response payload into a ruby object **if** the payload is valid json.
  138. #
  139. # response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
  140. # response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
  141. 27 def json(*args)
  142. 129 decode(Transcoder::JSON, *args)
  143. end
  144. # decodes the response payload into a ruby object **if** the payload is valid
  145. # "application/x-www-urlencoded" or "multipart/form-data".
  146. 27 def form
  147. 64 decode(Transcoder::Form)
  148. end
  149. 27 def xml
  150. # TODO: remove at next major version.
  151. 8 warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
  152. 1 "Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
  153. 8 require "httpx/plugins/xml"
  154. 8 decode(Plugins::XML::Transcoder)
  155. end
  156. 27 private
  157. # decodes the response payload using the given +transcoder+, which implements the decoding logic.
  158. #
  159. # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
  160. # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
  161. 27 def decode(transcoder, *args)
  162. # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
  163. 217 decoder = transcoder.decode(self)
  164. 193 raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
  165. 193 @body.rewind
  166. 193 decoder.call(self, *args)
  167. end
  168. end
  169. # Helper class which decodes the HTTP "content-type" header.
  170. 27 class ContentType
  171. 27 MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
  172. 27 CHARSET_RE = /;\s*charset=([^;]+)/i.freeze
  173. 27 def initialize(header_value)
  174. 9663 @header_value = header_value
  175. end
  176. # returns the mime type declared in the header.
  177. #
  178. # ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
  179. 27 def mime_type
  180. 217 return @mime_type if defined?(@mime_type)
  181. 179 m = @header_value.to_s[MIME_TYPE_RE, 1]
  182. 179 m && @mime_type = m.strip.downcase
  183. end
  184. # returns the charset declared in the header.
  185. #
  186. # ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
  187. # ContentType.new("text/plain").charset #=> nil
  188. 27 def charset
  189. 9484 return @charset if defined?(@charset)
  190. 9484 m = @header_value.to_s[CHARSET_RE, 1]
  191. 9484 m && @charset = m.strip.delete('"')
  192. end
  193. end
  194. # Wraps an error which has happened while processing an HTTP Request. It has partial
  195. # public API parity with HTTPX::Response, so users should rely on it to infer whether
  196. # the returned response is one or the other.
  197. #
  198. # response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
  199. # response.raise_for_status #=> raises if it wraps an error
  200. 27 class ErrorResponse
  201. 27 include Loggable
  202. 27 extend Forwardable
  203. # the corresponding HTTPX::Request instance.
  204. 27 attr_reader :request
  205. # the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
  206. 27 attr_reader :response
  207. # the wrapped exception.
  208. 27 attr_reader :error
  209. # the request uri
  210. 27 def_delegator :@request, :uri
  211. # the IP address of the peer server.
  212. 27 def_delegator :@request, :peer_address
  213. 27 def initialize(request, error)
  214. 1210 @request = request
  215. 1210 @response = request.response if request.response.is_a?(Response)
  216. 1210 @error = error
  217. 1210 @options = request.options
  218. 1210 log_exception(@error)
  219. end
  220. # returns the exception full message.
  221. 27 def to_s
  222. 9 @error.full_message(highlight: false)
  223. end
  224. # closes the error resources.
  225. 27 def close
  226. 40 @response.close if @response
  227. end
  228. # always true for error responses.
  229. 27 def finished?
  230. 1076 true
  231. end
  232. 27 def finish!; end
  233. # raises the wrapped exception.
  234. 27 def raise_for_status
  235. 82 raise @error
  236. end
  237. # buffers lost chunks to error response
  238. 27 def <<(data)
  239. 8 return unless @response
  240. 8 @response << data
  241. end
  242. end
  243. end
  244. 27 require_relative "response/body"
  245. 27 require_relative "response/buffer"
  246. 27 require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"

lib/httpx/response/body.rb

100.0% lines covered

109 relevant lines. 109 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
  4. # (for buffering the response payload), the IO reader protocol (for consuming the response payload),
  5. # and can be iterated over (via #each, which yields the payload in chunks).
  6. 27 class Response::Body
  7. # the payload encoding (i.e. "utf-8", "ASCII-8BIT")
  8. 27 attr_reader :encoding
  9. # Array of encodings contained in the response "content-encoding" header.
  10. 27 attr_reader :encodings
  11. 27 attr_reader :buffer
  12. 27 protected :buffer
  13. # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
  14. 27 def initialize(response, options)
  15. 9484 @response = response
  16. 9484 @headers = response.headers
  17. 9484 @options = options
  18. 9484 @window_size = options.window_size
  19. 9484 @encodings = []
  20. 9484 @length = 0
  21. 9484 @buffer = nil
  22. 9484 @reader = nil
  23. 9484 @state = :idle
  24. # initialize response encoding
  25. 9484 @encoding = if (enc = response.content_type.charset)
  26. 180 begin
  27. 1598 Encoding.find(enc)
  28. rescue ArgumentError
  29. 32 Encoding::BINARY
  30. end
  31. else
  32. 7886 Encoding::BINARY
  33. end
  34. 9484 initialize_inflaters
  35. end
  36. 27 def initialize_dup(other)
  37. 96 super
  38. 96 @buffer = other.instance_variable_get(:@buffer).dup
  39. end
  40. 27 def closed?
  41. 52 @state == :closed
  42. end
  43. # write the response payload +chunk+ into the buffer. Inflates the chunk when required
  44. # and supported.
  45. 27 def write(chunk)
  46. 12359 return if @state == :closed
  47. 12359 return 0 if chunk.empty?
  48. 11941 chunk = decode_chunk(chunk)
  49. 11941 size = chunk.bytesize
  50. 10624 @length += size
  51. 11941 transition(:open)
  52. 11941 @buffer.write(chunk)
  53. 11941 @response.emit(:chunk_received, chunk)
  54. 11925 size
  55. end
  56. # reads a chunk from the payload (implementation of the IO reader protocol).
  57. 27 def read(*args)
  58. 409 return unless @buffer
  59. 409 unless @reader
  60. 187 rewind
  61. 187 @reader = @buffer
  62. end
  63. 409 @reader.read(*args)
  64. end
  65. # size of the decoded response payload. May differ from "content-length" header if
  66. # response was encoded over-the-wire.
  67. 27 def bytesize
  68. 224 @length
  69. end
  70. # yields the payload in chunks.
  71. 27 def each
  72. 48 return enum_for(__method__) unless block_given?
  73. 3 begin
  74. 32 if @buffer
  75. 32 rewind
  76. 84 while (chunk = @buffer.read(@window_size))
  77. 32 yield(chunk.force_encoding(@encoding))
  78. end
  79. end
  80. ensure
  81. 32 close
  82. end
  83. end
  84. # returns the declared filename in the "contennt-disposition" header, when present.
  85. 27 def filename
  86. 48 return unless @headers.key?("content-disposition")
  87. 40 Utils.get_filename(@headers["content-disposition"])
  88. end
  89. # returns the full response payload as a string.
  90. 27 def to_s
  91. 4634 return "".b unless @buffer
  92. 4304 @buffer.to_s
  93. end
  94. 27 alias_method :to_str, :to_s
  95. # whether the payload is empty.
  96. 27 def empty?
  97. 32 @length.zero?
  98. end
  99. # copies the payload to +dest+.
  100. #
  101. # body.copy_to("path/to/file")
  102. # body.copy_to(Pathname.new("path/to/file"))
  103. # body.copy_to(File.new("path/to/file"))
  104. 27 def copy_to(dest)
  105. 48 return unless @buffer
  106. 48 rewind
  107. 48 if dest.respond_to?(:path) && @buffer.respond_to?(:path)
  108. 8 FileUtils.mv(@buffer.path, dest.path)
  109. else
  110. 40 ::IO.copy_stream(@buffer, dest)
  111. end
  112. end
  113. # closes/cleans the buffer, resets everything
  114. 27 def close
  115. 714 if @buffer
  116. 550 @buffer.close
  117. 550 @buffer = nil
  118. end
  119. 714 @length = 0
  120. 714 transition(:closed)
  121. end
  122. 27 def ==(other)
  123. 294 super || case other
  124. when Response::Body
  125. 152 @buffer == other.buffer
  126. else
  127. 86 @buffer = other
  128. end
  129. end
  130. skipped # :nocov:
  131. skipped def inspect
  132. skipped "#<#{self.class}:#{object_id} " \
  133. skipped "@state=#{@state} " \
  134. skipped "@length=#{@length}>"
  135. skipped end
  136. skipped # :nocov:
  137. # rewinds the response payload buffer.
  138. 27 def rewind
  139. 900 return unless @buffer
  140. # in case there's some reading going on
  141. 900 @reader = nil
  142. 900 @buffer.rewind
  143. end
  144. 27 private
  145. # prepares inflaters for the advertised encodings in "content-encoding" header.
  146. 27 def initialize_inflaters
  147. 9484 @inflaters = nil
  148. 9484 return unless @headers.key?("content-encoding")
  149. 163 return unless @options.decompress_response_body
  150. 147 @inflaters = @headers.get("content-encoding").filter_map do |encoding|
  151. 147 next if encoding == "identity"
  152. 147 inflater = self.class.initialize_inflater_by_encoding(encoding, @response)
  153. # do not uncompress if there is no decoder available. In fact, we can't reliably
  154. # continue decompressing beyond that, so ignore.
  155. 147 break unless inflater
  156. 147 @encodings << encoding
  157. 147 inflater
  158. end
  159. end
  160. # passes the +chunk+ through all inflaters to decode it.
  161. 27 def decode_chunk(chunk)
  162. 38 @inflaters.reverse_each do |inflater|
  163. 457 chunk = inflater.call(chunk)
  164. 12234 end if @inflaters
  165. 12235 chunk
  166. end
  167. # tries transitioning the body STM to the +nextstate+.
  168. 27 def transition(nextstate)
  169. 11260 case nextstate
  170. when :open
  171. 11941 return unless @state == :idle
  172. 7513 @buffer = Response::Buffer.new(
  173. threshold_size: @options.body_threshold_size,
  174. bytesize: @length,
  175. encoding: @encoding
  176. )
  177. when :closed
  178. 714 return if @state == :closed
  179. end
  180. 8227 @state = nextstate
  181. end
  182. 27 class << self
  183. 27 def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
  184. 133 case encoding
  185. when "gzip"
  186. 131 Transcoder::GZIP.decode(response, **kwargs)
  187. when "deflate"
  188. 16 Transcoder::Deflate.decode(response, **kwargs)
  189. end
  190. end
  191. end
  192. end
  193. end

lib/httpx/response/buffer.rb

92.19% lines covered

64 relevant lines. 59 lines covered and 5 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "delegate"
  3. 27 require "stringio"
  4. 27 require "tempfile"
  5. 27 module HTTPX
  6. # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
  7. 27 class Response::Buffer < SimpleDelegator
  8. 27 attr_reader :buffer
  9. 27 protected :buffer
  10. # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
  11. # the initial +bytesize+, and the +encoding+.
  12. 27 def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
  13. 7709 @threshold_size = threshold_size
  14. 7709 @bytesize = bytesize
  15. 7709 @encoding = encoding
  16. 7709 @buffer = StringIO.new("".b)
  17. 7709 super(@buffer)
  18. end
  19. 27 def initialize_dup(other)
  20. 96 super
  21. # create new descriptor in READ-ONLY mode
  22. 12 @buffer =
  23. 83 case other.buffer
  24. when StringIO
  25. 96 StringIO.new(other.buffer.string, mode: File::RDONLY)
  26. else
  27. other.buffer.class.new(other.buffer.path, encoding: Encoding::BINARY, mode: File::RDONLY)
  28. end
  29. end
  30. # size in bytes of the buffered content.
  31. 27 def size
  32. 324 @bytesize
  33. end
  34. # writes the +chunk+ into the buffer.
  35. 27 def write(chunk)
  36. 10963 @bytesize += chunk.bytesize
  37. 12320 try_upgrade_buffer
  38. 12320 @buffer.write(chunk)
  39. end
  40. # returns the buffered content as a string.
  41. 27 def to_s
  42. 3880 case @buffer
  43. when StringIO
  44. 494 begin
  45. 4320 @buffer.string.force_encoding(@encoding)
  46. rescue ArgumentError
  47. @buffer.string
  48. end
  49. when Tempfile
  50. 72 rewind
  51. 72 content = @buffer.read
  52. 8 begin
  53. 72 content.force_encoding(@encoding)
  54. rescue ArgumentError # ex: unknown encoding name - utf
  55. content
  56. end
  57. end
  58. end
  59. # closes the buffer.
  60. 27 def close
  61. 662 @buffer.close
  62. 662 @buffer.unlink if @buffer.respond_to?(:unlink)
  63. end
  64. 27 def ==(other)
  65. 152 super || begin
  66. 152 return false unless other.is_a?(Response::Buffer)
  67. 152 if @buffer.nil?
  68. other.buffer.nil?
  69. 151 elsif @buffer.respond_to?(:read) &&
  70. other.respond_to?(:read)
  71. 152 buffer_pos = @buffer.pos
  72. 152 other_pos = other.buffer.pos
  73. 152 @buffer.rewind
  74. 152 other.buffer.rewind
  75. 18 begin
  76. 152 FileUtils.compare_stream(@buffer, other.buffer)
  77. ensure
  78. 152 @buffer.pos = buffer_pos
  79. 152 other.buffer.pos = other_pos
  80. end
  81. else
  82. to_s == other.to_s
  83. end
  84. end
  85. end
  86. 27 private
  87. # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
  88. # has been reached.
  89. 27 def try_upgrade_buffer
  90. 12320 return unless @bytesize > @threshold_size
  91. 474 return if @buffer.is_a?(Tempfile)
  92. 147 aux = @buffer
  93. 147 @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  94. 147 if aux
  95. 147 aux.rewind
  96. 147 ::IO.copy_stream(aux, @buffer)
  97. 147 aux.close
  98. end
  99. 147 __setobj__(@buffer)
  100. end
  101. end
  102. end

lib/httpx/selector.rb

93.64% lines covered

110 relevant lines. 103 lines covered and 7 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "io/wait"
  3. 27 module HTTPX
  4. 27 class Selector
  5. 27 extend Forwardable
  6. 27 READABLE = %i[rw r].freeze
  7. 27 WRITABLE = %i[rw w].freeze
  8. 27 private_constant :READABLE
  9. 27 private_constant :WRITABLE
  10. 27 def_delegator :@timers, :after
  11. 27 def_delegator :@selectables, :empty?
  12. 27 def initialize
  13. 7708 @timers = Timers.new
  14. 7708 @selectables = []
  15. 7708 @is_timer_interval = false
  16. end
  17. 27 def each(&blk)
  18. @selectables.each(&blk)
  19. end
  20. 27 def next_tick
  21. 10689117 catch(:jump_tick) do
  22. 10689117 timeout = next_timeout
  23. 10689117 if timeout && timeout.negative?
  24. @timers.fire
  25. throw(:jump_tick)
  26. end
  27. 758844 begin
  28. 10689117 select(timeout) do |c|
  29. 23448 c.log(level: 2) { "[#{c.state}] selected#{" after #{timeout} secs" unless timeout.nil?}..." }
  30. 23331 c.call
  31. end
  32. 10688960 @timers.fire
  33. rescue TimeoutError => e
  34. @timers.fire(e)
  35. end
  36. end
  37. rescue StandardError => e
  38. 21 each_connection do |c|
  39. c.emit(:error, e)
  40. end
  41. rescue Exception # rubocop:disable Lint/RescueException
  42. 120 each_connection do |conn|
  43. 96 conn.force_reset
  44. 96 conn.disconnect
  45. end
  46. 120 raise
  47. end
  48. 27 def terminate
  49. # array may change during iteration
  50. 7212 selectables = @selectables.reject(&:inflight?)
  51. 7212 selectables.delete_if do |sel|
  52. 2922 sel.terminate
  53. 2914 sel.state == :closed
  54. end
  55. 7205 until selectables.empty?
  56. 9 next_tick
  57. 8 selectables &= @selectables
  58. end
  59. end
  60. 27 def find_resolver(options)
  61. 7177 res = @selectables.find do |c|
  62. 53 c.is_a?(Resolver::Resolver) && options == c.options
  63. end
  64. 7177 res.multi if res
  65. end
  66. 27 def each_connection(&block)
  67. 33934 return enum_for(__method__) unless block
  68. 17285 @selectables.each do |c|
  69. 2123 case c
  70. when Resolver::Resolver
  71. 236 c.each_connection(&block)
  72. when Connection
  73. 2020 yield c
  74. end
  75. end
  76. end
  77. 27 def find_connection(request_uri, options)
  78. 9267 each_connection.find do |connection|
  79. 1236 connection.match?(request_uri, options)
  80. end
  81. end
  82. 27 def find_mergeable_connection(connection)
  83. 7382 each_connection.find do |ch|
  84. 337 ch != connection && ch.mergeable?(connection)
  85. end
  86. end
  87. # deregisters +io+ from selectables.
  88. 27 def deregister(io)
  89. 8374 @selectables.delete(io)
  90. end
  91. # register +io+.
  92. 27 def register(io)
  93. 8871 return if @selectables.include?(io)
  94. 8416 @selectables << io
  95. end
  96. 27 private
  97. 27 def select(interval, &block)
  98. # do not cause an infinite loop here.
  99. #
  100. # this may happen if timeout calculation actually triggered an error which causes
  101. # the connections to be reaped (such as the total timeout error) before #select
  102. # gets called.
  103. 10689117 return if interval.nil? && @selectables.empty?
  104. 10689108 return select_one(interval, &block) if @selectables.size == 1
  105. 603 select_many(interval, &block)
  106. end
  107. 27 def select_many(interval, &block)
  108. 603 r, w = nil
  109. # first, we group IOs based on interest type. On call to #interests however,
  110. # things might already happen, and new IOs might be registered, so we might
  111. # have to start all over again. We do this until we group all selectables
  112. 603 @selectables.delete_if do |io|
  113. 887 interests = io.interests
  114. 887 io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
  115. 887 (r ||= []) << io if READABLE.include?(interests)
  116. 887 (w ||= []) << io if WRITABLE.include?(interests)
  117. 887 io.state == :closed
  118. end
  119. # TODO: what to do if there are no selectables?
  120. 603 readers, writers = IO.select(r, w, nil, interval)
  121. 603 if readers.nil? && writers.nil? && interval
  122. 169 [*r, *w].each { |io| io.handle_socket_timeout(interval) }
  123. 154 return
  124. end
  125. 434 if writers
  126. 35 readers.each do |io|
  127. 302 yield io
  128. # so that we don't yield 2 times
  129. 302 writers.delete(io)
  130. 433 end if readers
  131. 434 writers.each(&block)
  132. else
  133. readers.each(&block) if readers
  134. end
  135. end
  136. 27 def select_one(interval)
  137. 10688505 io = @selectables.first
  138. 10688505 return unless io
  139. 10688505 interests = io.interests
  140. 10688632 io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
  141. 10688503 result = case interests
  142. 13563 when :r then io.to_io.wait_readable(interval)
  143. 9870 when :w then io.to_io.wait_writable(interval)
  144. when :rw then io.to_io.wait(interval, :read_write)
  145. 10665070 when nil then return
  146. end
  147. 23433 unless result || interval.nil?
  148. 543 io.handle_socket_timeout(interval) unless @is_timer_interval
  149. 485 return
  150. end
  151. # raise TimeoutError.new(interval, "timed out while waiting on select")
  152. 22890 yield io
  153. # rescue IOError, SystemCallError
  154. # @selectables.reject!(&:closed?)
  155. # raise unless @selectables.empty?
  156. end
  157. 27 def next_timeout
  158. 10689117 @is_timer_interval = false
  159. 10689117 timer_interval = @timers.wait_interval
  160. 10689117 connection_interval = @selectables.filter_map(&:timeout).min
  161. 10689117 return connection_interval unless timer_interval
  162. 10663621 if connection_interval.nil? || timer_interval <= connection_interval
  163. 10663579 @is_timer_interval = true
  164. 10231949 return timer_interval
  165. end
  166. 42 connection_interval
  167. end
  168. end
  169. end

lib/httpx/session.rb

95.6% lines covered

273 relevant lines. 261 lines covered and 12 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. # Class implementing the APIs being used publicly.
  4. #
  5. # HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
  6. # HTTPX.plugin(..).get(..) #=> creating an intermediate HTTPX::Session with plugin, then sending the GET request
  7. 27 class Session
  8. 27 include Loggable
  9. 27 include Chainable
  10. # initializes the session with a set of +options+, which will be shared by all
  11. # requests sent from it.
  12. #
  13. # When pass a block, it'll yield itself to it, then closes after the block is evaluated.
  14. 27 def initialize(options = EMPTY_HASH, &blk)
  15. 11143 @options = self.class.default_options.merge(options)
  16. 11143 @persistent = @options.persistent
  17. 11143 @pool = @options.pool_class.new(@options.pool_options)
  18. 11143 @wrapped = false
  19. 11143 @closing = false
  20. 11143 INSTANCES[self] = self if @persistent && @options.close_on_fork && INSTANCES
  21. 11143 wrap(&blk) if blk
  22. end
  23. # Yields itself the block, then closes it after the block is evaluated.
  24. #
  25. # session.wrap do |http|
  26. # http.get("https://wikipedia.com")
  27. # end # wikipedia connection closes here
  28. 27 def wrap
  29. 582 prev_wrapped = @wrapped
  30. 582 @wrapped = true
  31. 582 was_initialized = false
  32. 582 current_selector = get_current_selector do
  33. 582 selector = Selector.new
  34. 582 set_current_selector(selector)
  35. 582 was_initialized = true
  36. 582 selector
  37. end
  38. 57 begin
  39. 582 yield self
  40. ensure
  41. 582 unless prev_wrapped
  42. 582 if @persistent
  43. 1 deactivate(current_selector)
  44. else
  45. 581 close(current_selector)
  46. end
  47. end
  48. 582 @wrapped = prev_wrapped
  49. 582 set_current_selector(nil) if was_initialized
  50. end
  51. end
  52. # closes all the active connections from the session.
  53. #
  54. # when called directly without specifying +selector+, all available connections
  55. # will be picked up from the connection pool and closed. Connections in use
  56. # by other sessions, or same session in a different thread, will not be reaped.
  57. 27 def close(selector = Selector.new)
  58. # throw resolvers away from the pool
  59. 7212 @pool.reset_resolvers
  60. # preparing to throw away connections
  61. 16698 while (connection = @pool.pop_connection)
  62. 4422 next if connection.state == :closed
  63. 213 select_connection(connection, selector)
  64. end
  65. 812 begin
  66. 7212 @closing = true
  67. 7212 selector.terminate
  68. ensure
  69. 7212 @closing = false
  70. end
  71. end
  72. # performs one, or multple requests; it accepts:
  73. #
  74. # 1. one or multiple HTTPX::Request objects;
  75. # 2. an HTTP verb, then a sequence of URIs or URI/options tuples;
  76. # 3. one or multiple HTTP verb / uri / (optional) options tuples;
  77. #
  78. # when present, the set of +options+ kwargs is applied to all of the
  79. # sent requests.
  80. #
  81. # respectively returns a single HTTPX::Response response, or all of them in an Array, in the same order.
  82. #
  83. # resp1 = session.request(req1)
  84. # resp1, resp2 = session.request(req1, req2)
  85. # resp1 = session.request("GET", "https://server.org/a")
  86. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"])
  87. # resp1, resp2 = session.request(["GET", "https://server.org/a"], ["GET", "https://server.org/b"])
  88. # resp1 = session.request("POST", "https://server.org/a", form: { "foo" => "bar" })
  89. # resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
  90. # resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
  91. #
  92. 27 def request(*args, **params)
  93. 7505 raise ArgumentError, "must perform at least one request" if args.empty?
  94. 7505 requests = args.first.is_a?(Request) ? args : build_requests(*args, params)
  95. 7468 responses = send_requests(*requests)
  96. 7318 return responses.first if responses.size == 1
  97. 199 responses
  98. end
  99. # returns a HTTP::Request instance built from the HTTP +verb+, the request +uri+, and
  100. # the optional set of request-specific +options+. This request **must** be sent through
  101. # the same session it was built from.
  102. #
  103. # req = session.build_request("GET", "https://server.com")
  104. # resp = session.request(req)
  105. 27 def build_request(verb, uri, params = EMPTY_HASH, options = @options)
  106. 9195 rklass = options.request_class
  107. 9195 request = rklass.new(verb, uri, options, params)
  108. 9158 request.persistent = @persistent
  109. 9158 set_request_callbacks(request)
  110. 9158 request
  111. end
  112. 27 def select_connection(connection, selector)
  113. 8839 pin_connection(connection, selector)
  114. 8839 selector.register(connection)
  115. end
  116. 27 def pin_connection(connection, selector)
  117. 8860 connection.current_session = self
  118. 8860 connection.current_selector = selector
  119. end
  120. 27 alias_method :select_resolver, :select_connection
  121. 27 def deselect_connection(connection, selector, cloned = false)
  122. 7992 selector.deregister(connection)
  123. # when connections coalesce
  124. 7992 return if connection.state == :idle
  125. 7968 return if cloned
  126. 7960 return if @closing && connection.state == :closed
  127. 7952 @pool.checkin_connection(connection)
  128. end
  129. 27 def deselect_resolver(resolver, selector)
  130. 366 selector.deregister(resolver)
  131. 366 return if @closing && resolver.closed?
  132. 366 @pool.checkin_resolver(resolver)
  133. end
  134. 27 def try_clone_connection(connection, selector, family)
  135. 464 connection.family ||= family
  136. 464 return connection if connection.family == family
  137. new_connection = connection.class.new(connection.origin, connection.options)
  138. new_connection.family = family
  139. connection.sibling = new_connection
  140. do_init_connection(new_connection, selector)
  141. new_connection
  142. end
  143. # returns the HTTPX::Connection through which the +request+ should be sent through.
  144. 27 def find_connection(request_uri, selector, options)
  145. 9267 if (connection = selector.find_connection(request_uri, options))
  146. 1182 connection.idling if connection.state == :closed
  147. 1182 connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
  148. 1115 return connection
  149. end
  150. 8085 connection = @pool.checkout_connection(request_uri, options)
  151. 8113 connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
  152. 7154 case connection.state
  153. when :idle
  154. 7343 do_init_connection(connection, selector)
  155. when :open
  156. 66 if options.io
  157. 66 select_connection(connection, selector)
  158. else
  159. pin_connection(connection, selector)
  160. end
  161. when :closing, :closed
  162. 623 connection.idling
  163. 623 select_connection(connection, selector)
  164. else
  165. 21 pin_connection(connection, selector)
  166. end
  167. 7989 connection
  168. end
  169. 27 private
  170. 27 def deactivate(selector)
  171. 495 selector.each_connection do |connection|
  172. 369 connection.deactivate
  173. 369 deselect_connection(connection, selector) if connection.state == :inactive
  174. end
  175. end
  176. # callback executed when an HTTP/2 promise frame has been received.
  177. 27 def on_promise(_, stream)
  178. 8 log(level: 2) { "#{stream.id}: refusing stream!" }
  179. 8 stream.refuse
  180. end
  181. # returns the corresponding HTTP::Response to the given +request+ if it has been received.
  182. 27 def fetch_response(request, _selector, _options)
  183. 10697353 response = request.response
  184. 10697353 response if response && response.finished?
  185. end
  186. # sends the +request+ to the corresponding HTTPX::Connection
  187. 27 def send_request(request, selector, options = request.options)
  188. 1917 error = begin
  189. 9190 catch(:resolve_error) do
  190. 9190 connection = find_connection(request.uri, selector, options)
  191. 9070 connection.send(request)
  192. end
  193. rescue StandardError => e
  194. 40 e
  195. end
  196. 9184 return unless error && error.is_a?(Exception)
  197. 120 raise error unless error.is_a?(Error)
  198. 120 response = ErrorResponse.new(request, error)
  199. 120 request.response = response
  200. 120 request.emit(:response, response)
  201. end
  202. # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
  203. 27 def build_requests(*args, params)
  204. 6905 requests = if args.size == 1
  205. 78 reqs = args.first
  206. 78 reqs.map do |verb, uri, ps = EMPTY_HASH|
  207. 156 request_params = params
  208. 156 request_params = request_params.merge(ps) unless ps.empty?
  209. 156 build_request(verb, uri, request_params)
  210. end
  211. else
  212. 6827 verb, uris = args
  213. 6827 if uris.respond_to?(:each)
  214. 6587 uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
  215. 7345 request_params = params
  216. 7345 request_params = request_params.merge(ps) unless ps.empty?
  217. 7345 build_request(verb, uri, request_params)
  218. end
  219. else
  220. 240 [build_request(verb, uris, params)]
  221. end
  222. end
  223. 6868 raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
  224. 6868 requests
  225. end
  226. 27 def set_request_callbacks(request)
  227. 9031 request.on(:promise, &method(:on_promise))
  228. end
  229. 27 def do_init_connection(connection, selector)
  230. 7343 resolve_connection(connection, selector) unless connection.family
  231. end
  232. # sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
  233. 27 def send_requests(*requests)
  234. 14411 selector = get_current_selector { Selector.new }
  235. 833 begin
  236. 7566 _send_requests(requests, selector)
  237. 7560 receive_requests(requests, selector)
  238. ensure
  239. 7550 unless @wrapped
  240. 6844 if @persistent
  241. 494 deactivate(selector)
  242. else
  243. 6350 close(selector)
  244. end
  245. end
  246. end
  247. end
  248. # sends an array of HTTPX::Request objects
  249. 27 def _send_requests(requests, selector)
  250. 7566 requests.each do |request|
  251. 8387 send_request(request, selector)
  252. end
  253. end
  254. # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
  255. 27 def receive_requests(requests, selector)
  256. # @type var responses: Array[response]
  257. 7560 responses = []
  258. # guarantee ordered responses
  259. 7560 loop do
  260. 8389 request = requests.first
  261. 8389 return responses unless request
  262. 11129305 catch(:coalesced) { selector.next_tick } until (response = fetch_response(request, selector, request.options))
  263. 8245 request.emit(:complete, response)
  264. 8245 responses << response
  265. 8245 requests.shift
  266. 8245 break if requests.empty?
  267. 829 next unless selector.empty?
  268. # in some cases, the pool of connections might have been drained because there was some
  269. # handshake error, and the error responses have already been emitted, but there was no
  270. # opportunity to traverse the requests, hence we're returning only a fraction of the errors
  271. # we were supposed to. This effectively fetches the existing responses and return them.
  272. while (request = requests.shift)
  273. response = fetch_response(request, selector, request.options)
  274. request.emit(:complete, response) if response
  275. responses << response
  276. end
  277. break
  278. end
  279. 7416 responses
  280. end
  281. 27 def resolve_connection(connection, selector)
  282. 7367 if connection.addresses || connection.open?
  283. #
  284. # there are two cases in which we want to activate initialization of
  285. # connection immediately:
  286. #
  287. # 1. when the connection already has addresses, i.e. it doesn't need to
  288. # resolve a name (not the same as name being an IP, yet)
  289. # 2. when the connection is initialized with an external already open IO.
  290. #
  291. 190 on_resolver_connection(connection, selector)
  292. 188 return
  293. end
  294. 7177 resolver = find_resolver_for(connection, selector)
  295. 7177 resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
  296. end
  297. 27 def on_resolver_connection(connection, selector)
  298. 7382 from_pool = false
  299. 7382 found_connection = selector.find_mergeable_connection(connection) || begin
  300. 7355 from_pool = true
  301. 7355 @pool.checkout_mergeable_connection(connection)
  302. end
  303. 7382 return select_connection(connection, selector) unless found_connection
  304. 27 connection.log(level: 2) do
  305. "try coalescing from #{from_pool ? "pool##{@pool.object_id}" : "selector##{selector.object_id}"} " \
  306. "(conn##{found_connection.object_id}[#{found_connection.origin}])"
  307. end
  308. 27 coalesce_connections(found_connection, connection, selector, from_pool)
  309. end
  310. 27 def on_resolver_close(resolver, selector)
  311. 366 return if resolver.closed?
  312. 366 deselect_resolver(resolver, selector)
  313. 366 resolver.close unless resolver.closed?
  314. end
  315. 27 def find_resolver_for(connection, selector)
  316. 7177 resolver = selector.find_resolver(connection.options)
  317. 7177 unless resolver
  318. 7175 resolver = @pool.checkout_resolver(connection.options)
  319. 7175 resolver.current_session = self
  320. 7175 resolver.current_selector = selector
  321. end
  322. 7177 resolver
  323. end
  324. # coalesces +conn2+ into +conn1+. if +conn1+ was loaded from the connection pool
  325. # (it is known via +from_pool+), then it adds its to the +selector+.
  326. 27 def coalesce_connections(conn1, conn2, selector, from_pool)
  327. 27 unless conn1.coalescable?(conn2)
  328. 14 conn2.log(level: 2) { "not coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
  329. 14 select_connection(conn2, selector)
  330. 14 @pool.checkin_connection(conn1) if from_pool
  331. 14 return false
  332. end
  333. 13 conn2.log(level: 2) { "coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
  334. 13 conn2.coalesce!(conn1)
  335. 13 select_connection(conn1, selector) if from_pool
  336. 13 conn2.disconnect
  337. 13 true
  338. end
  339. 27 def get_current_selector
  340. 8148 selector_store[self] || (yield if block_given?)
  341. end
  342. 27 def set_current_selector(selector)
  343. 1642 if selector
  344. 958 selector_store[self] = selector
  345. else
  346. 582 selector_store.delete(self)
  347. end
  348. end
  349. 27 def selector_store
  350. 9790 th_current = Thread.current
  351. 9790 th_current.thread_variable_get(:httpx_persistent_selector_store) || begin
  352. 165 {}.compare_by_identity.tap do |store|
  353. 165 th_current.thread_variable_set(:httpx_persistent_selector_store, store)
  354. end
  355. end
  356. end
  357. 27 @default_options = Options.new
  358. 27 @default_options.freeze
  359. 27 @plugins = []
  360. 27 class << self
  361. 27 attr_reader :default_options
  362. 27 def inherited(klass)
  363. 5615 super
  364. 5615 klass.instance_variable_set(:@default_options, @default_options)
  365. 5615 klass.instance_variable_set(:@plugins, @plugins.dup)
  366. 5615 klass.instance_variable_set(:@callbacks, @callbacks.dup)
  367. end
  368. # returns a new HTTPX::Session instance, with the plugin pointed by +pl+ loaded.
  369. #
  370. # session_with_retries = session.plugin(:retries)
  371. # session_with_custom = session.plugin(CustomPlugin)
  372. #
  373. 27 def plugin(pl, options = nil, &block)
  374. 7641 label = pl
  375. # raise Error, "Cannot add a plugin to a frozen config" if frozen?
  376. 7641 pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
  377. 7641 if !@plugins.include?(pl)
  378. 7411 @plugins << pl
  379. 7411 pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
  380. 7411 @default_options = @default_options.dup
  381. 7411 include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
  382. 7411 extend(pl::ClassMethods) if defined?(pl::ClassMethods)
  383. 7411 opts = @default_options
  384. 7411 opts.extend_with_plugin_classes(pl)
  385. 7411 if defined?(pl::OptionsMethods)
  386. 2997 (pl::OptionsMethods.instance_methods - Object.instance_methods).each do |meth|
  387. 8915 opts.options_class.method_added(meth)
  388. end
  389. 2997 @default_options = opts.options_class.new(opts)
  390. end
  391. 7411 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  392. 7411 @default_options = @default_options.merge(options) if options
  393. 7411 if pl.respond_to?(:subplugins)
  394. 32 pl.subplugins.transform_keys(&Plugins.method(:load_plugin)).each do |main_pl, sub_pl|
  395. # in case the main plugin has already been loaded, then apply subplugin functionality
  396. # immediately
  397. 32 next unless @plugins.include?(main_pl)
  398. 8 plugin(sub_pl, options, &block)
  399. end
  400. end
  401. 7411 pl.configure(self, &block) if pl.respond_to?(:configure)
  402. 7411 if label.is_a?(Symbol)
  403. # in case an already-loaded plugin complements functionality of
  404. # the plugin currently being loaded, loaded it now
  405. 5656 @plugins.each do |registered_pl|
  406. 13737 next if registered_pl == pl
  407. 8081 next unless registered_pl.respond_to?(:subplugins)
  408. 16 sub_pl = registered_pl.subplugins[label]
  409. 16 next unless sub_pl
  410. 16 plugin(sub_pl, options, &block)
  411. end
  412. end
  413. 7411 @default_options.freeze
  414. 7411 set_temporary_name("#{superclass}/#{pl}") if respond_to?(:set_temporary_name) # ruby 3.4 only
  415. 229 elsif options
  416. # this can happen when two plugins are loaded, an one of them calls the other under the hood,
  417. # albeit changing some default.
  418. 16 @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
  419. 16 @default_options = @default_options.merge(options) if options
  420. 16 @default_options.freeze
  421. end
  422. 7641 self
  423. end
  424. end
  425. # setup of the support for close_on_fork sessions.
  426. # adapted from https://github.com/mperham/connection_pool/blob/main/lib/connection_pool.rb#L48
  427. 27 if Process.respond_to?(:fork)
  428. 25 INSTANCES = ObjectSpace::WeakMap.new
  429. 25 private_constant :INSTANCES
  430. 25 def self.after_fork
  431. 1 INSTANCES.each_value(&:close)
  432. 1 nil
  433. end
  434. 25 if ::Process.respond_to?(:_fork)
  435. 21 module ForkTracker
  436. 21 def _fork
  437. 1 pid = super
  438. 1 Session.after_fork if pid.zero?
  439. 1 pid
  440. end
  441. end
  442. 21 Process.singleton_class.prepend(ForkTracker)
  443. end
  444. else
  445. 2 INSTANCES = nil
  446. 2 private_constant :INSTANCES
  447. 2 def self.after_fork
  448. # noop
  449. end
  450. end
  451. end
  452. # session may be overridden by certain adapters.
  453. 27 S = Session
  454. end

lib/httpx/session_extensions.rb

100.0% lines covered

14 relevant lines. 14 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 unless ENV.keys.grep(/\Ahttps?_proxy\z/i).empty?
  4. 1 proxy_session = plugin(:proxy)
  5. 1 remove_const(:Session)
  6. 1 const_set(:Session, proxy_session.class)
  7. # redefine the default options static var, which needs to
  8. # refresh options_class
  9. 1 options = proxy_session.class.default_options.to_hash
  10. 1 original_verbosity = $VERBOSE
  11. 1 $VERBOSE = nil
  12. 1 const_set(:Options, proxy_session.class.default_options.options_class)
  13. 1 options[:options_class] = Class.new(options[:options_class])
  14. 1 options.freeze
  15. 1 Options.send(:const_set, :DEFAULT_OPTIONS, options)
  16. 1 Session.instance_variable_set(:@default_options, Options.new(options))
  17. 1 $VERBOSE = original_verbosity
  18. end
  19. skipped # :nocov:
  20. skipped if Session.default_options.debug_level > 2
  21. skipped proxy_session = plugin(:internal_telemetry)
  22. skipped remove_const(:Session)
  23. skipped const_set(:Session, proxy_session.class)
  24. skipped end
  25. skipped # :nocov:
  26. end

lib/httpx/timers.rb

93.94% lines covered

66 relevant lines. 62 lines covered and 4 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 class Timers
  4. 27 def initialize
  5. 7708 @intervals = []
  6. end
  7. 27 def after(interval_in_secs, cb = nil, &blk)
  8. 39329 callback = cb || blk
  9. 39329 raise Error, "timer must have a callback" unless callback
  10. # I'm assuming here that most requests will have the same
  11. # request timeout, as in most cases they share common set of
  12. # options. A user setting different request timeouts for 100s of
  13. # requests will already have a hard time dealing with that.
  14. 70737 unless (interval = @intervals.bsearch { |t| t.interval == interval_in_secs })
  15. 9453 interval = Interval.new(interval_in_secs)
  16. 9453 @intervals << interval
  17. 9453 @intervals.sort!
  18. end
  19. 39329 interval << callback
  20. 39329 @next_interval_at = nil
  21. 39329 Timer.new(interval, callback)
  22. end
  23. 27 def wait_interval
  24. 10689117 drop_elapsed!
  25. 10689117 return if @intervals.empty?
  26. 10663621 @next_interval_at = Utils.now
  27. 10663621 @intervals.first.interval
  28. end
  29. 27 def fire(error = nil)
  30. 10688960 raise error if error && error.timeout != @intervals.first
  31. 10688960 return if @intervals.empty? || !@next_interval_at
  32. 10662959 elapsed_time = Utils.elapsed_time(@next_interval_at)
  33. 10662959 drop_elapsed!(elapsed_time)
  34. 21318515 @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
  35. 10662959 @next_interval_at = nil if @intervals.empty?
  36. end
  37. 27 private
  38. 27 def drop_elapsed!(elapsed_time = 0)
  39. # check first, if not elapsed, then return
  40. 21352076 first_interval = @intervals.first
  41. 21352076 return unless first_interval && first_interval.elapsed?(elapsed_time)
  42. # TODO: would be nice to have a drop_while!
  43. 16545 @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
  44. end
  45. 27 class Timer
  46. 27 def initialize(interval, callback)
  47. 39329 @interval = interval
  48. 39329 @callback = callback
  49. end
  50. 27 def cancel
  51. 57806 @interval.delete(@callback)
  52. end
  53. end
  54. 27 class Interval
  55. 27 include Comparable
  56. 27 attr_reader :interval
  57. 27 def initialize(interval)
  58. 9453 @interval = interval
  59. 9453 @callbacks = []
  60. end
  61. 27 def <=>(other)
  62. 822 @interval <=> other.interval
  63. end
  64. 27 def ==(other)
  65. return @interval == other if other.is_a?(Numeric)
  66. @interval == other.to_f # rubocop:disable Lint/FloatComparison
  67. end
  68. 27 def to_f
  69. Float(@interval)
  70. end
  71. 27 def <<(callback)
  72. 39329 @callbacks << callback
  73. end
  74. 27 def delete(callback)
  75. 57806 @callbacks.delete(callback)
  76. end
  77. 27 def no_callbacks?
  78. @callbacks.empty?
  79. end
  80. 27 def elapsed?(elapsed = 0)
  81. 21327034 (@interval - elapsed) <= 0 || @callbacks.empty?
  82. end
  83. 27 def elapse(elapsed)
  84. # same as elapsing
  85. 10664112 return 0 if @callbacks.empty?
  86. 10225304 @interval -= elapsed
  87. 10656318 if @interval <= 0
  88. 633 cb = @callbacks.dup
  89. 633 cb.each(&:call)
  90. end
  91. 10656318 @interval
  92. end
  93. end
  94. 27 private_constant :Interval
  95. end
  96. end

lib/httpx/transcoder.rb

100.0% lines covered

52 relevant lines. 52 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder
  4. 27 module_function
  5. 27 def normalize_keys(key, value, cond = nil, &block)
  6. 3415 if cond && cond.call(value)
  7. 1065 block.call(key.to_s, value)
  8. 2349 elsif value.respond_to?(:to_ary)
  9. 454 if value.empty?
  10. 128 block.call("#{key}[]")
  11. else
  12. 326 value.to_ary.each do |element|
  13. 524 normalize_keys("#{key}[]", element, cond, &block)
  14. end
  15. end
  16. 1895 elsif value.respond_to?(:to_hash)
  17. 512 value.to_hash.each do |child_key, child_value|
  18. 512 normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
  19. end
  20. else
  21. 1384 block.call(key.to_s, value)
  22. end
  23. end
  24. # based on https://github.com/rack/rack/blob/d15dd728440710cfc35ed155d66a98dc2c07ae42/lib/rack/query_parser.rb#L82
  25. 27 def normalize_query(params, name, v, depth)
  26. 184 raise Error, "params depth surpasses what's supported" if depth <= 0
  27. 184 name =~ /\A[\[\]]*([^\[\]]+)\]*/
  28. 184 k = Regexp.last_match(1) || ""
  29. 184 after = Regexp.last_match ? Regexp.last_match.post_match : ""
  30. 184 if k.empty?
  31. 16 return Array(v) if !v.empty? && name == "[]"
  32. 7 return
  33. end
  34. 147 case after
  35. when ""
  36. 49 params[k] = v
  37. when "["
  38. 7 params[name] = v
  39. when "[]"
  40. 16 params[k] ||= []
  41. 16 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  42. 16 params[k] << v
  43. when /^\[\]\[([^\[\]]+)\]$/, /^\[\](.+)$/
  44. 32 child_key = Regexp.last_match(1)
  45. 32 params[k] ||= []
  46. 32 raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
  47. 32 if params[k].last.is_a?(Hash) && !params_hash_has_key?(params[k].last, child_key)
  48. 8 normalize_query(params[k].last, child_key, v, depth - 1)
  49. else
  50. 24 params[k] << normalize_query({}, child_key, v, depth - 1)
  51. end
  52. else
  53. 56 params[k] ||= {}
  54. 56 raise Error, "expected Hash (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Hash)
  55. 49 params[k] = normalize_query(params[k], after, v, depth - 1)
  56. end
  57. 168 params
  58. end
  59. 27 def params_hash_has_key?(hash, key)
  60. 16 return false if key.include?("[]")
  61. 16 key.split(/[\[\]]+/).inject(hash) do |h, part|
  62. 16 next h if part == ""
  63. 16 return false unless h.is_a?(Hash) && h.key?(part)
  64. 8 h[part]
  65. end
  66. 8 true
  67. end
  68. end
  69. end
  70. 27 require "httpx/transcoder/body"
  71. 27 require "httpx/transcoder/form"
  72. 27 require "httpx/transcoder/json"
  73. 27 require "httpx/transcoder/chunker"
  74. 27 require "httpx/transcoder/deflate"
  75. 27 require "httpx/transcoder/gzip"

lib/httpx/transcoder/body.rb

100.0% lines covered

26 relevant lines. 26 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "delegate"
  3. 27 module HTTPX::Transcoder
  4. 27 module Body
  5. 27 class Error < HTTPX::Error; end
  6. 27 module_function
  7. 27 class Encoder < SimpleDelegator
  8. 27 def initialize(body)
  9. 1448 body = body.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && body.is_a?(Pathname)
  10. 1448 @body = body
  11. 1448 super(body)
  12. end
  13. 27 def bytesize
  14. 5704 if @body.respond_to?(:bytesize)
  15. 2550 @body.bytesize
  16. 3153 elsif @body.respond_to?(:to_ary)
  17. 998 @body.sum(&:bytesize)
  18. 2155 elsif @body.respond_to?(:size)
  19. 1500 @body.size || Float::INFINITY
  20. 655 elsif @body.respond_to?(:length)
  21. 360 @body.length || Float::INFINITY
  22. 295 elsif @body.respond_to?(:each)
  23. 288 Float::INFINITY
  24. else
  25. 8 raise Error, "cannot determine size of body: #{@body.inspect}"
  26. end
  27. end
  28. 27 def content_type
  29. 1370 "application/octet-stream"
  30. end
  31. end
  32. 27 def encode(body)
  33. 1448 Encoder.new(body)
  34. end
  35. end
  36. end

lib/httpx/transcoder/chunker.rb

100.0% lines covered

66 relevant lines. 66 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 module HTTPX::Transcoder
  4. 27 module Chunker
  5. 27 class Error < HTTPX::Error; end
  6. 27 CRLF = "\r\n".b
  7. 27 class Encoder
  8. 27 extend Forwardable
  9. 27 def initialize(body)
  10. 96 @raw = body
  11. end
  12. 27 def each
  13. 96 return enum_for(__method__) unless block_given?
  14. 96 @raw.each do |chunk|
  15. 448 yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
  16. end
  17. 96 yield "0#{CRLF}"
  18. end
  19. 27 def respond_to_missing?(meth, *args)
  20. 108 @raw.respond_to?(meth, *args) || super
  21. end
  22. end
  23. 27 class Decoder
  24. 27 extend Forwardable
  25. 27 def_delegator :@buffer, :empty?
  26. 27 def_delegator :@buffer, :<<
  27. 27 def_delegator :@buffer, :clear
  28. 27 def initialize(buffer, trailers = false)
  29. 114 @buffer = buffer
  30. 114 @chunk_buffer = "".b
  31. 114 @finished = false
  32. 114 @state = :length
  33. 114 @trailers = trailers
  34. end
  35. 27 def to_s
  36. 106 @buffer
  37. end
  38. 27 def each
  39. 159 loop do
  40. 999 case @state
  41. when :length
  42. 332 index = @buffer.index(CRLF)
  43. 332 return unless index && index.positive?
  44. # Read hex-length
  45. 332 hexlen = @buffer.byteslice(0, index)
  46. 332 @buffer = @buffer.byteslice(index..-1) || "".b
  47. 332 hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
  48. 332 @chunk_length = hexlen.hex
  49. # check if is last chunk
  50. 332 @finished = @chunk_length.zero?
  51. 332 nextstate(:crlf)
  52. when :crlf
  53. 550 crlf_size = @finished && !@trailers ? 4 : 2
  54. # consume CRLF
  55. 550 return if @buffer.bytesize < crlf_size
  56. 550 raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
  57. 550 @buffer = @buffer.byteslice(crlf_size..-1)
  58. 550 if @chunk_length.nil?
  59. 218 nextstate(:length)
  60. else
  61. 332 return if @finished
  62. 226 nextstate(:data)
  63. end
  64. when :data
  65. 258 chunk = @buffer.byteslice(0, @chunk_length)
  66. 258 @buffer = @buffer.byteslice(@chunk_length..-1) || "".b
  67. 258 @chunk_buffer << chunk
  68. 226 @chunk_length -= chunk.bytesize
  69. 258 if @chunk_length.zero?
  70. 226 yield @chunk_buffer unless @chunk_buffer.empty?
  71. 218 @chunk_buffer.clear
  72. 218 @chunk_length = nil
  73. 218 nextstate(:crlf)
  74. end
  75. end
  76. 1026 break if @buffer.empty?
  77. end
  78. end
  79. 27 def finished?
  80. 151 @finished
  81. end
  82. 27 private
  83. 27 def nextstate(state)
  84. 994 @state = state
  85. end
  86. end
  87. 27 module_function
  88. 27 def encode(chunks)
  89. 96 Encoder.new(chunks)
  90. end
  91. end
  92. end

lib/httpx/transcoder/deflate.rb

100.0% lines covered

20 relevant lines. 20 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "zlib"
  3. 27 require_relative "utils/deflater"
  4. 27 module HTTPX
  5. 27 module Transcoder
  6. 27 module Deflate
  7. 27 class Deflater < Transcoder::Deflater
  8. 27 def deflate(chunk)
  9. 72 @deflater ||= Zlib::Deflate.new
  10. 72 if chunk.nil?
  11. 48 unless @deflater.closed?
  12. 24 last = @deflater.finish
  13. 24 @deflater.close
  14. 24 last.empty? ? nil : last
  15. end
  16. else
  17. 24 @deflater.deflate(chunk)
  18. end
  19. end
  20. end
  21. 27 module_function
  22. 27 def encode(body)
  23. 24 Deflater.new(body)
  24. end
  25. 27 def decode(response, bytesize: nil)
  26. 16 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  27. 16 GZIP::Inflater.new(bytesize)
  28. end
  29. end
  30. end
  31. end

lib/httpx/transcoder/form.rb

100.0% lines covered

42 relevant lines. 42 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 require "uri"
  4. 27 require_relative "multipart"
  5. 27 module HTTPX
  6. 27 module Transcoder
  7. 27 module Form
  8. 27 module_function
  9. 27 PARAM_DEPTH_LIMIT = 32
  10. 27 class Encoder
  11. 27 extend Forwardable
  12. 27 def_delegator :@raw, :to_s
  13. 27 def_delegator :@raw, :to_str
  14. 27 def_delegator :@raw, :bytesize
  15. 27 def_delegator :@raw, :==
  16. 27 def initialize(form)
  17. 706 @raw = form.each_with_object("".b) do |(key, val), buf|
  18. 1186 HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
  19. 1384 buf << "&" unless buf.empty?
  20. 1384 buf << URI.encode_www_form_component(k)
  21. 1384 buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
  22. end
  23. end
  24. end
  25. 27 def content_type
  26. 554 "application/x-www-form-urlencoded"
  27. end
  28. end
  29. 27 module Decoder
  30. 27 module_function
  31. 27 def call(response, *)
  32. 40 URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params|
  33. 96 HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT)
  34. end
  35. end
  36. end
  37. 27 def encode(form)
  38. 1657 if multipart?(form)
  39. 951 Multipart::Encoder.new(form)
  40. else
  41. 706 Encoder.new(form)
  42. end
  43. end
  44. 27 def decode(response)
  45. 64 content_type = response.content_type.mime_type
  46. 56 case content_type
  47. when "application/x-www-form-urlencoded"
  48. 40 Decoder
  49. when "multipart/form-data"
  50. 16 Multipart::Decoder.new(response)
  51. else
  52. 8 raise Error, "invalid form mime type (#{content_type})"
  53. end
  54. end
  55. 27 def multipart?(data)
  56. 1657 data.any? do |_, v|
  57. 2201 Multipart::MULTIPART_VALUE_COND.call(v) ||
  58. 1698 (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
  59. 2082 (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
  60. end
  61. end
  62. end
  63. end
  64. end

lib/httpx/transcoder/gzip.rb

100.0% lines covered

40 relevant lines. 40 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "zlib"
  3. 27 module HTTPX
  4. 27 module Transcoder
  5. 27 module GZIP
  6. 27 class Deflater < Transcoder::Deflater
  7. 27 def initialize(body)
  8. 48 @compressed_chunk = "".b
  9. 48 super
  10. end
  11. 27 def deflate(chunk)
  12. 96 @deflater ||= Zlib::GzipWriter.new(self)
  13. 96 if chunk.nil?
  14. 48 unless @deflater.closed?
  15. 48 @deflater.flush
  16. 48 @deflater.close
  17. 48 compressed_chunk
  18. end
  19. else
  20. 48 @deflater.write(chunk)
  21. 48 compressed_chunk
  22. end
  23. end
  24. 27 private
  25. 27 def write(chunk)
  26. 144 @compressed_chunk << chunk
  27. end
  28. 27 def compressed_chunk
  29. 96 @compressed_chunk.dup
  30. ensure
  31. 96 @compressed_chunk.clear
  32. end
  33. end
  34. 27 class Inflater
  35. 27 def initialize(bytesize)
  36. 147 @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
  37. 147 @bytesize = bytesize
  38. end
  39. 27 def call(chunk)
  40. 457 buffer = @inflater.inflate(chunk)
  41. 419 @bytesize -= chunk.bytesize
  42. 457 if @bytesize <= 0
  43. 100 buffer << @inflater.finish
  44. 100 @inflater.close
  45. end
  46. 457 buffer
  47. end
  48. end
  49. 27 module_function
  50. 27 def encode(body)
  51. 48 Deflater.new(body)
  52. end
  53. 27 def decode(response, bytesize: nil)
  54. 131 bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
  55. 131 Inflater.new(bytesize)
  56. end
  57. end
  58. end
  59. end

lib/httpx/transcoder/json.rb

100.0% lines covered

33 relevant lines. 33 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "forwardable"
  3. 27 module HTTPX::Transcoder
  4. 27 module JSON
  5. 27 module_function
  6. 27 JSON_REGEX = %r{
  7. \b
  8. application/
  9. # optional vendor specific type
  10. (?:
  11. # token as per https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6
  12. [!#$%&'*+\-.^_`|~0-9a-z]+
  13. # literal plus sign
  14. \+
  15. )?
  16. json
  17. \b
  18. }ix.freeze
  19. 27 class Encoder
  20. 27 extend Forwardable
  21. 27 def_delegator :@raw, :to_s
  22. 27 def_delegator :@raw, :bytesize
  23. 27 def_delegator :@raw, :==
  24. 27 def initialize(json)
  25. 83 @raw = JSON.json_dump(json)
  26. 83 @charset = @raw.encoding.name.downcase
  27. end
  28. 27 def content_type
  29. 83 "application/json; charset=#{@charset}"
  30. end
  31. end
  32. 27 def encode(json)
  33. 83 Encoder.new(json)
  34. end
  35. 27 def decode(response)
  36. 129 content_type = response.content_type.mime_type
  37. 129 raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
  38. 113 method(:json_load)
  39. end
  40. # rubocop:disable Style/SingleLineMethods
  41. 27 if defined?(MultiJson)
  42. 4 def json_load(*args); MultiJson.load(*args); end
  43. 2 def json_dump(*args); MultiJson.dump(*args); end
  44. 25 elsif defined?(Oj)
  45. 5 def json_load(response, *args); Oj.load(response.to_s, *args); end
  46. 3 def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
  47. 23 elsif defined?(Yajl)
  48. 4 def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
  49. 2 def json_dump(*args); Yajl::Encoder.encode(*args); end
  50. else
  51. 23 require "json"
  52. 114 def json_load(*args); ::JSON.parse(*args); end
  53. 93 def json_dump(*args); ::JSON.dump(*args); end
  54. end
  55. # rubocop:enable Style/SingleLineMethods
  56. end
  57. end

lib/httpx/transcoder/multipart.rb

100.0% lines covered

10 relevant lines. 10 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require_relative "multipart/encoder"
  3. 27 require_relative "multipart/decoder"
  4. 27 require_relative "multipart/part"
  5. 27 require_relative "multipart/mime_type_detector"
  6. 27 module HTTPX::Transcoder
  7. 27 module Multipart
  8. 27 MULTIPART_VALUE_COND = lambda do |value|
  9. 4878 value.respond_to?(:read) ||
  10. 3506 (value.respond_to?(:to_hash) &&
  11. value.key?(:body) &&
  12. 644 (value.key?(:filename) || value.key?(:content_type)))
  13. end
  14. end
  15. end

lib/httpx/transcoder/multipart/decoder.rb

94.05% lines covered

84 relevant lines. 79 lines covered and 5 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "tempfile"
  3. 27 require "delegate"
  4. 27 module HTTPX
  5. 27 module Transcoder
  6. 27 module Multipart
  7. 27 class FilePart < SimpleDelegator
  8. 27 attr_reader :original_filename, :content_type
  9. 27 def initialize(filename, content_type)
  10. 32 @original_filename = filename
  11. 32 @content_type = content_type
  12. 32 @current = nil
  13. 32 @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
  14. 32 super(@file)
  15. end
  16. end
  17. 27 class Decoder
  18. 27 include HTTPX::Utils
  19. 27 CRLF = "\r\n"
  20. 27 BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
  21. 27 MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
  22. 27 MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
  23. 27 MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
  24. 27 WINDOW_SIZE = 2 << 14
  25. 27 def initialize(response)
  26. 2 @boundary = begin
  27. 16 m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
  28. 16 raise Error, "no boundary declared in content-type header" unless m
  29. 16 m.strip
  30. end
  31. 16 @buffer = "".b
  32. 16 @parts = {}
  33. 16 @intermediate_boundary = "--#{@boundary}"
  34. 16 @state = :idle
  35. end
  36. 27 def call(response, *)
  37. 16 response.body.each do |chunk|
  38. 16 @buffer << chunk
  39. 16 parse
  40. end
  41. 16 raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
  42. 16 @parts
  43. end
  44. 27 private
  45. 27 def parse
  46. 14 case @state
  47. when :idle
  48. 16 raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
  49. 16 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
  50. 16 @state = :part_header
  51. when :part_header
  52. 48 idx = @buffer.index("#{CRLF}#{CRLF}")
  53. # raise Error, "couldn't parse part headers" unless idx
  54. 48 return unless idx
  55. # @type var head: String
  56. 48 head = @buffer.byteslice(0..idx + 4 - 1)
  57. 48 @buffer = @buffer.byteslice(head.bytesize..-1)
  58. 48 content_type = head[MULTIPART_CONTENT_TYPE, 1] || "text/plain"
  59. 84 if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
  60. 48 name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
  61. 48 name.gsub!(/\\(.)/, "\\1")
  62. 12 name
  63. else
  64. name = head[MULTIPART_CONTENT_ID, 1]
  65. end
  66. 48 filename = HTTPX::Utils.get_filename(head)
  67. 48 name = filename || +"#{content_type}[]" if name.nil? || name.empty?
  68. 48 @current = name
  69. 42 @parts[name] = if filename
  70. 32 FilePart.new(filename, content_type)
  71. else
  72. 16 "".b
  73. end
  74. 48 @state = :part_body
  75. when :part_body
  76. 48 part = @parts[@current]
  77. 48 body_separator = if part.is_a?(FilePart)
  78. 28 "#{CRLF}#{CRLF}"
  79. else
  80. 16 CRLF
  81. end
  82. 48 idx = @buffer.index(body_separator)
  83. 48 if idx
  84. 48 payload = @buffer.byteslice(0..idx - 1)
  85. 48 @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
  86. 48 part << payload
  87. 48 part.rewind if part.respond_to?(:rewind)
  88. 48 @state = :parse_boundary
  89. else
  90. part << @buffer
  91. @buffer.clear
  92. end
  93. when :parse_boundary
  94. 48 raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
  95. 48 @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
  96. 48 if @buffer == "--"
  97. 16 @buffer.clear
  98. 16 @state = :done
  99. 16 return
  100. 31 elsif @buffer.start_with?(CRLF)
  101. 32 @buffer = @buffer.byteslice(2..-1)
  102. 32 @state = :part_header
  103. else
  104. return
  105. end
  106. when :done
  107. raise Error, "parsing should have been over by now"
  108. 20 end until @buffer.empty?
  109. end
  110. end
  111. end
  112. end
  113. end

lib/httpx/transcoder/multipart/encoder.rb

100.0% lines covered

70 relevant lines. 70 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder::Multipart
  4. 27 class Encoder
  5. 27 attr_reader :bytesize
  6. 27 def initialize(form)
  7. 951 @boundary = ("-" * 21) << SecureRandom.hex(21)
  8. 951 @part_index = 0
  9. 951 @buffer = "".b
  10. 951 @form = form
  11. 951 @bytesize = 0
  12. 951 @parts = to_parts(form)
  13. end
  14. 27 def content_type
  15. 951 "multipart/form-data; boundary=#{@boundary}"
  16. end
  17. 27 def to_s
  18. 18 read || ""
  19. ensure
  20. 18 rewind
  21. end
  22. 27 def read(length = nil, outbuf = nil)
  23. 3518 data = String(outbuf).clear.force_encoding(Encoding::BINARY) if outbuf
  24. 3518 data ||= "".b
  25. 3518 read_chunks(data, length)
  26. 3518 data unless length && data.empty?
  27. end
  28. 27 def rewind
  29. 50 form = @form.each_with_object([]) do |(key, val), aux|
  30. 50 if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
  31. # @type var val: File
  32. 50 val = val.reopen(val.path, File::RDONLY)
  33. end
  34. 50 val.rewind if val.respond_to?(:rewind)
  35. 50 aux << [key, val]
  36. end
  37. 50 @form = form
  38. 50 @bytesize = 0
  39. 50 @parts = to_parts(form)
  40. 50 @part_index = 0
  41. end
  42. 27 private
  43. 27 def to_parts(form)
  44. 1001 params = form.each_with_object([]) do |(key, val), aux|
  45. 1193 Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
  46. 1193 next if v.nil?
  47. 1193 value, content_type, filename = Part.call(v)
  48. 1193 header = header_part(k, content_type, filename)
  49. 1049 @bytesize += header.size
  50. 1193 aux << header
  51. 1049 @bytesize += value.size
  52. 1193 aux << value
  53. 1193 delimiter = StringIO.new("\r\n")
  54. 1049 @bytesize += delimiter.size
  55. 1193 aux << delimiter
  56. end
  57. end
  58. 1001 final_delimiter = StringIO.new("--#{@boundary}--\r\n")
  59. 881 @bytesize += final_delimiter.size
  60. 1001 params << final_delimiter
  61. 1001 params
  62. end
  63. 27 def header_part(key, content_type, filename)
  64. 1193 header = "--#{@boundary}\r\n".b
  65. 1193 header << "Content-Disposition: form-data; name=#{key.inspect}".b
  66. 1193 header << "; filename=#{filename.inspect}" if filename
  67. 1193 header << "\r\nContent-Type: #{content_type}\r\n\r\n"
  68. 1193 StringIO.new(header)
  69. end
  70. 27 def read_chunks(buffer, length = nil)
  71. 4618 while @part_index < @parts.size
  72. 10584 chunk = read_from_part(length)
  73. 10584 next unless chunk
  74. 6052 buffer << chunk.force_encoding(Encoding::BINARY)
  75. 6052 next unless length
  76. 5246 length -= chunk.bytesize
  77. 5986 break if length.zero?
  78. end
  79. end
  80. # if there's a current part to read from, tries to read a chunk.
  81. 27 def read_from_part(max_length = nil)
  82. 10584 part = @parts[@part_index]
  83. 10584 chunk = part.read(max_length, @buffer)
  84. 10584 return chunk if chunk && !chunk.empty?
  85. 4532 part.close if part.respond_to?(:close)
  86. 3980 @part_index += 1
  87. 1692 nil
  88. end
  89. end
  90. end
  91. end

lib/httpx/transcoder/multipart/mime_type_detector.rb

92.11% lines covered

38 relevant lines. 35 lines covered and 3 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder::Multipart
  4. 27 module MimeTypeDetector
  5. 27 module_function
  6. 27 DEFAULT_MIMETYPE = "application/octet-stream"
  7. # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
  8. 27 if defined?(FileMagic)
  9. 1 MAGIC_NUMBER = 256 * 1024
  10. 1 def call(file, _)
  11. 1 return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
  12. 1 mime = FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
  13. 1 filemagic.buffer(file.read(MAGIC_NUMBER))
  14. end
  15. 1 file.rewind
  16. 1 mime
  17. end
  18. 25 elsif defined?(Marcel)
  19. 1 def call(file, filename)
  20. 1 return nil if file.eof? # marcel returns "application/octet-stream" for empty files
  21. 1 Marcel::MimeType.for(file, name: filename)
  22. end
  23. 24 elsif defined?(MimeMagic)
  24. 1 def call(file, _)
  25. 1 mime = MimeMagic.by_magic(file)
  26. 1 mime.type if mime
  27. end
  28. 23 elsif system("which file", out: File::NULL)
  29. 24 require "open3"
  30. 24 def call(file, _)
  31. 677 return if file.eof? # file command returns "application/x-empty" for empty files
  32. 633 Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
  33. 75 begin
  34. 633 ::IO.copy_stream(file, stdin.binmode)
  35. rescue Errno::EPIPE
  36. end
  37. 633 file.rewind
  38. 633 stdin.close
  39. 633 status = thread.value
  40. # call to file command failed
  41. 633 if status.nil? || !status.success?
  42. $stderr.print(stderr.read)
  43. else
  44. 633 output = stdout.read.strip
  45. 633 if output.include?("cannot open")
  46. $stderr.print(output)
  47. else
  48. 633 output
  49. end
  50. end
  51. end
  52. end
  53. else
  54. def call(_, _); end
  55. end
  56. end
  57. end
  58. end

lib/httpx/transcoder/multipart/part.rb

100.0% lines covered

18 relevant lines. 18 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Transcoder::Multipart
  4. 27 module Part
  5. 27 module_function
  6. 27 def call(value)
  7. # take out specialized objects of the way
  8. 1193 if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
  9. 112 return value, value.content_type, value.filename
  10. end
  11. 1065 content_type = filename = nil
  12. 1065 if value.is_a?(Hash)
  13. 322 content_type = value[:content_type]
  14. 322 filename = value[:filename]
  15. 322 value = value[:body]
  16. end
  17. 1065 value = value.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
  18. 1065 if value.respond_to?(:path) && value.respond_to?(:read)
  19. # either a File, a Tempfile, or something else which has to quack like a file
  20. 681 filename ||= File.basename(value.path)
  21. 681 content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
  22. 681 [value, content_type, filename]
  23. else
  24. 384 [StringIO.new(value.to_s), content_type || "text/plain", filename]
  25. end
  26. end
  27. end
  28. end
  29. end

lib/httpx/transcoder/utils/body_reader.rb

92.0% lines covered

25 relevant lines. 23 lines covered and 2 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require "stringio"
  3. 27 module HTTPX
  4. 27 module Transcoder
  5. 27 class BodyReader
  6. 27 def initialize(body)
  7. 210 @body = if body.respond_to?(:read)
  8. 20 body.rewind if body.respond_to?(:rewind)
  9. 20 body
  10. 189 elsif body.respond_to?(:each)
  11. 36 body.enum_for(:each)
  12. else
  13. 154 StringIO.new(body.to_s)
  14. end
  15. end
  16. 27 def bytesize
  17. 450 return @body.bytesize if @body.respond_to?(:bytesize)
  18. 414 Float::INFINITY
  19. end
  20. 27 def read(length = nil, outbuf = nil)
  21. 468 return @body.read(length, outbuf) if @body.respond_to?(:read)
  22. begin
  23. 96 chunk = @body.next
  24. 48 if outbuf
  25. outbuf.clear.force_encoding(Encoding::BINARY)
  26. outbuf << chunk
  27. else
  28. 48 outbuf = chunk
  29. end
  30. 48 outbuf unless length && outbuf.empty?
  31. 32 rescue StopIteration
  32. end
  33. end
  34. 27 def close
  35. 48 @body.close if @body.respond_to?(:close)
  36. end
  37. end
  38. end
  39. end

lib/httpx/transcoder/utils/deflater.rb

100.0% lines covered

37 relevant lines. 37 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 require_relative "body_reader"
  3. 27 module HTTPX
  4. 27 module Transcoder
  5. 27 class Deflater
  6. 27 attr_reader :content_type
  7. 27 def initialize(body)
  8. 84 @content_type = body.content_type
  9. 84 @body = BodyReader.new(body)
  10. 84 @closed = false
  11. end
  12. 27 def bytesize
  13. 324 buffer_deflate!
  14. 324 @buffer.size
  15. end
  16. 27 def read(length = nil, outbuf = nil)
  17. 416 return @buffer.read(length, outbuf) if @buffer
  18. 240 return if @closed
  19. 192 chunk = @body.read(length)
  20. 192 compressed_chunk = deflate(chunk)
  21. 192 return unless compressed_chunk
  22. 156 if outbuf
  23. 144 outbuf.clear.force_encoding(Encoding::BINARY)
  24. 144 outbuf << compressed_chunk
  25. else
  26. 12 compressed_chunk
  27. end
  28. end
  29. 27 def close
  30. 48 return if @closed
  31. 48 @buffer.close if @buffer
  32. 48 @body.close
  33. 48 @closed = true
  34. end
  35. 27 def rewind
  36. 28 return unless @buffer
  37. 16 @buffer.rewind
  38. end
  39. 27 private
  40. # rubocop:disable Naming/MemoizedInstanceVariableName
  41. 27 def buffer_deflate!
  42. 324 return @buffer if defined?(@buffer)
  43. 84 buffer = Response::Buffer.new(
  44. threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
  45. )
  46. 84 ::IO.copy_stream(self, buffer)
  47. 84 buffer.rewind if buffer.respond_to?(:rewind)
  48. 84 @buffer = buffer
  49. end
  50. # rubocop:enable Naming/MemoizedInstanceVariableName
  51. end
  52. end
  53. end

lib/httpx/utils.rb

100.0% lines covered

39 relevant lines. 39 lines covered and 0 lines missed.
    
  1. # frozen_string_literal: true
  2. 27 module HTTPX
  3. 27 module Utils
  4. 27 using URIExtensions
  5. 27 TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
  6. 27 VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
  7. 27 FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
  8. 27 FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
  9. 27 module_function
  10. 27 def now
  11. 10699233 Process.clock_gettime(Process::CLOCK_MONOTONIC)
  12. end
  13. 27 def elapsed_time(monotonic_timestamp)
  14. 10663440 Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
  15. end
  16. # The value of this field can be either an HTTP-date or a number of
  17. # seconds to delay after the response is received.
  18. 27 def parse_retry_after(retry_after)
  19. # first: bet on it being an integer
  20. 63 Integer(retry_after)
  21. rescue ArgumentError
  22. # Then it's a datetime
  23. 16 time = Time.httpdate(retry_after)
  24. 16 time - Time.now
  25. end
  26. 27 def get_filename(header, _prefix_regex = nil)
  27. 88 filename = nil
  28. 77 case header
  29. when FILENAME_REGEX
  30. 56 filename = Regexp.last_match(1)
  31. 56 filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
  32. when FILENAME_EXTENSION_REGEX
  33. 16 filename = Regexp.last_match(1)
  34. 16 encoding, _, filename = filename.split("'", 3)
  35. end
  36. 88 return unless filename
  37. 136 filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
  38. 72 filename.scrub!
  39. 72 filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
  40. 72 filename.force_encoding ::Encoding.find(encoding) if encoding
  41. 72 filename
  42. end
  43. 27 URIParser = URI::RFC2396_Parser.new
  44. 27 def to_uri(uri)
  45. 18011 return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
  46. 33 uri = URI(URIParser.escape(uri))
  47. 33 non_ascii_hostname = URIParser.unescape(uri.host)
  48. 33 non_ascii_hostname.force_encoding(Encoding::UTF_8)
  49. 33 idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
  50. 33 uri.host = idna_hostname
  51. 32 uri.non_ascii_hostname = non_ascii_hostname
  52. 32 uri
  53. end
  54. end
  55. end